Update mq-perf benchmark for the refactored queue API

- start poll threads via q.backend/start! (start-transports was removed).
- stale-recovery bench calls the recover-stale! protocol method.
- encode payloads before direct publish! and seed the :payload column
  (encoding was centralized / column renamed upstream).
- read the keyword :publish-ns key (decode keywordizes message keys), so
  latency stats are populated again.

.clj-kondo/config.edn

@@ -592,6 +592,9 @@
    metabase.model-persistence.api-test/with-persistence-setup!                                                               hooks.common/with-one-top-level-binding
    metabase.model-persistence.api-test/with-setup!                                                                           hooks.common/with-one-top-level-binding
    metabase.model-persistence.test-util/with-persistence-enabled!                                                            hooks.metabase.model-persistence.test-util/with-persistence-enabled!
+   metabase.mq.core/with-queue                                                                                               hooks.common/let-second
+   metabase.mq.test-util/with-test-mq                                                                                        hooks.common/with-one-binding
+   metabase.mq.core/def-listener!                                                                                            hooks.metabase.mq.impl/def-listener
    metabase.models.interface/define-batched-hydration-method                                                                 hooks.metabase.models.interface/define-hydration-method
    metabase.models.interface/define-simple-hydration-method                                                                  hooks.metabase.models.interface/define-hydration-method
    metabase.permissions.core/allow-changing-all-users-group-members                                                          hooks.common/do*

.clj-kondo/config/modules/config.edn

@@ -1355,6 +1355,16 @@
                     :model/Table
                     :model/User}}
 
+  mq
+  {:team "UX West"
+   :api  #{metabase.mq.init}
+   :uses #{analytics-interface
+           app-db
+           models
+           settings
+           startup
+           util}}
+
   native-query-snippets
   {:team "Graphy"
    :api  #{metabase.native-query-snippets.api

.clj-kondo/src/hooks/metabase/mq/impl.clj

@@ -0,0 +1,18 @@
+(ns hooks.metabase.mq.impl
+  (:require
+   [clj-kondo.hooks-api :as hooks]))
+
+(defn def-listener
+  "Kondo hook for `mq/def-listener`. Rewrites to a `fn` form so kondo can analyze bindings.
+
+   Form:
+     (mq/def-listener! :queue/bar [msgs] body...)"
+  [{:keys [node]}]
+  (let [[_ _channel-name bindings & body] (:children node)
+        ;; rewrite as: (fn [bindings...] body...)
+        new-node (hooks/list-node
+                  (list*
+                   (hooks/token-node 'fn)
+                   bindings
+                   body))]
+    {:node new-node}))

dev/src/dev/mq_perf.clj

@@ -0,0 +1,835 @@
+(ns dev.mq-perf
+  "Performance benchmarks for the appdb-backed MQ queue system.
+
+  Measures publishing throughput, end-to-end latency, and sustained throughput
+  under different load levels. Benchmarks exercise the full realistic pipeline:
+  publish buffer → DB → poll thread → worker pool → listener.
+
+  Self-contained — starts the MQ subsystem without the full Metabase server.
+  Supports multi-node testing: run multiple JVM instances against the same DB.
+
+  Example usage:
+    ;; Start MQ and run all benchmarks
+    (start-mq!)
+    (run-all-benchmarks!)
+
+    ;; Individual benchmarks
+    (bench-publish-throughput! {:n 1000 :batch-sizes [1 10 100]})
+    (bench-e2e-latency! {:n 50 :rate :medium})
+    (bench-sustained-throughput! {:duration-sec 15 :rate :medium})
+    (bench-batch-listener! {:n 200 :batch-sizes [1 10 50]})
+    (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})"
+  (:require
+   [clojure.string :as str]
+   [metabase.app-db.connection :as mdb.conn]
+   [metabase.mq.core :as mq]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.payload :as payload]
+   [metabase.mq.polling :as polling]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.registry :as q.registry]
+   [toucan2.core :as t2])
+  (:import
+   (java.lang.reflect InvocationHandler Method Proxy)
+   (java.sql Connection PreparedStatement Statement)
+   (java.util.concurrent CountDownLatch TimeUnit)
+   (javax.sql DataSource)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------------ RTT injection ------------------------------------------------
+;;;
+;;; Wraps the application DB's `DataSource` so every JDBC `execute*` call sleeps a randomized
+;;; interval before hitting Postgres. This lets us see what the perf picture looks like with
+;;; cloud-realistic latency (~1-5 ms RTT) without leaving the JVM. The wrapper preserves the
+;;; pooled DataSource underneath, so connection acquisition is unaffected — only query
+;;; round-trips pay the simulated cost.
+
+(defonce ^:private rtt-ms
+  ;; Use an atom rather than a dynamic var so the injected RTT applies to *every* thread —
+  ;; including the backend poll threads and the worker pool — without needing per-thread
+  ;; bindings to be conveyed.
+  (atom nil))
+
+(defn- sleep-rtt! []
+  (when-let [[mn mx] @rtt-ms]
+    (let [delay (long (if (= mn mx) mn (+ mn (rand-int (inc (- mx mn))))))]
+      (when (pos? delay) (Thread/sleep delay)))))
+
+(defn- proxy-statement
+  "Wrap a Statement / PreparedStatement so every `execute*` call sleeps the injected RTT
+  before delegating. Returns a proxy that implements both `Statement` and
+  `PreparedStatement` (PreparedStatement extends Statement, so a single proxy covers both)."
+  [^Statement stmt]
+  (Proxy/newProxyInstance
+   (.getClassLoader Statement)
+   (into-array Class [Statement PreparedStatement])
+   (reify InvocationHandler
+     (invoke [_ _proxy method args]
+       (when (.startsWith (.getName ^Method method) "execute")
+         (sleep-rtt!))
+       (.invoke method stmt args)))))
+
+(defn- proxy-connection
+  "Wrap a Connection so prepareStatement / createStatement return latency-injecting Statements."
+  [^Connection conn]
+  (Proxy/newProxyInstance
+   (.getClassLoader Connection)
+   (into-array Class [Connection])
+   (reify InvocationHandler
+     (invoke [_ _proxy method args]
+       (let [n (.getName ^Method method)
+             result (.invoke method conn args)]
+         (if (and (or (= n "prepareStatement") (= n "createStatement"))
+                  (instance? Statement result))
+           (proxy-statement result)
+           result))))))
+
+(defn- latent-datasource
+  "Returns a wrapped DataSource whose connections inject RTT before every `execute*`.
+  Delegates to the underlying pooled DataSource for connection acquisition itself."
+  ^DataSource [^DataSource inner]
+  (reify DataSource
+    (getConnection [_]            (proxy-connection (.getConnection inner)))
+    (getConnection [_ user pw]    (proxy-connection (.getConnection inner user pw)))
+    (getLoginTimeout [_]          (.getLoginTimeout inner))
+    (setLoginTimeout [_ s]        (.setLoginTimeout inner s))
+    (getLogWriter [_]             (.getLogWriter inner))
+    (setLogWriter [_ pw]          (.setLogWriter inner pw))
+    (getParentLogger [_]          (.getParentLogger inner))
+    (^boolean isWrapperFor [_ ^Class iface] (.isWrapperFor inner iface))
+    (^Object unwrap [_ ^Class iface]        (.unwrap inner iface))))
+
+(defonce ^:private wrapped-datasource? (atom false))
+
+(defn install-rtt!
+  "Wraps the app DB's DataSource so every JDBC `execute*` call sleeps `[min-ms max-ms]`
+  before delegating. Idempotent — calling this twice keeps a single wrapper. Use
+  `(install-rtt! nil nil)` (or `(uninstall-rtt!)`) to disable injection without unwrapping.
+  Returns the new effective `[min max]`."
+  [min-ms max-ms]
+  (reset! rtt-ms (when (and min-ms max-ms) [min-ms max-ms]))
+  (when-not @wrapped-datasource?
+    (let [orig    mdb.conn/*application-db*
+          wrapped (mdb.conn/application-db (.db-type orig)
+                                           (latent-datasource (.data-source orig))
+                                           :create-pool? false)]
+      (alter-var-root #'mdb.conn/*application-db* (constantly wrapped))
+      (reset! wrapped-datasource? true)))
+  @rtt-ms)
+
+(defn uninstall-rtt!
+  "Disables RTT injection. Leaves the DataSource wrapper in place (no-op when rtt-ms is nil)
+  to avoid surprises if other code captured the wrapped instance."
+  []
+  (reset! rtt-ms nil))
+
+(defmacro with-rtt-ms
+  "Runs `body` with `[min-ms max-ms]` RTT injected on every JDBC execute, then restores the
+  prior setting."
+  [[min-ms max-ms] & body]
+  `(let [prev# @rtt-ms]
+     (install-rtt! ~min-ms ~max-ms)
+     (try ~@body
+          (finally
+            (reset! rtt-ms prev#)))))
+
+;;; ------------------------------------------------ Startup ------------------------------------------------
+
+(defonce ^:private started? (atom false))
+
+(declare declare-shared-queues!)
+
+(defn start-mq!
+  "Minimal startup: connects to app DB and starts the MQ subsystem.
+  No Jetty, no scheduler, no sample data. Idempotent."
+  []
+  (let [setup-db! (requiring-resolve 'metabase.app-db.core/setup-db!)]
+    (setup-db! :create-sample-content? false))
+  (let [setup-prometheus! (requiring-resolve 'metabase.analytics.prometheus/setup!)]
+    (setup-prometheus!))
+  (require 'metabase.mq.init)
+  (let [startup! (requiring-resolve 'metabase.startup.core/def-startup-logic!)]
+    (startup! :metabase.mq.init/MqStart))
+  (reset! started? true)
+  (println "MQ system started."))
+
+(defn- ensure-started!
+  "Starts the MQ system if not yet started, and restarts any dead poll threads."
+  []
+  (when-not @started?
+    (start-mq!))
+  ;; Restart poll threads if they died (e.g., from InterruptedException during a previous benchmark).
+  ;; start! is idempotent and the shared poll driver detects dead futures.
+  (q.backend/start! q.appdb/backend)
+  ;; Benchmarks register listeners on the shared queues, which now requires the queues to be
+  ;; declared first.
+  (declare-shared-queues!))
+
+;;; ------------------------------------------------ Utilities ------------------------------------------------
+
+(def ^:private shared-queues
+  "Fixed set of 10 shared queue names used across all nodes for cross-node contention testing."
+  (mapv #(keyword "queue" (str "bench-shared-" %)) (range 10)))
+
+(defn- ensure-queue!
+  "Declares `ch` in the queue registry (default config unless given) if not already declared.
+  Listeners require their queue to be declared first, so benchmarks call this before
+  `batch-listen!`. Idempotent for identical config."
+  ([ch] (ensure-queue! ch {}))
+  ([ch config]
+   (when-not (q.registry/get-queue ch)
+     (q.registry/register-queue! ch config))))
+
+(defn- declare-shared-queues!
+  "Declares all shared benchmark queues with default config."
+  []
+  (run! ensure-queue! shared-queues))
+
+(defn- run-id
+  "Generates an 8-character unique identifier for a benchmark run."
+  []
+  (subs (str (random-uuid)) 0 8))
+
+(defn- make-channel
+  "Creates a unique channel keyword for benchmarking."
+  [transport id suffix]
+  (keyword (name transport) (str "bench-" id "-" suffix)))
+
+(defn- random-shared-queue
+  "Returns a random shared queue name."
+  []
+  (nth shared-queues (rand-int (count shared-queues))))
+
+(defn- percentile
+  "Returns the p-th percentile from a sorted vector (p in [0,1])."
+  [sorted-vec p]
+  (if (empty? sorted-vec)
+    0.0
+    (let [idx (min (dec (count sorted-vec))
+                   (int (* p (count sorted-vec))))]
+      (double (nth sorted-vec idx)))))
+
+(defn- collect-stats
+  "Computes summary statistics from a collection of numeric values.
+  Returns {:count :min :max :avg :p50 :p95 :p99}."
+  [values]
+  (if (empty? values)
+    {:count 0 :min 0.0 :max 0.0 :avg 0.0 :p50 0.0 :p95 0.0 :p99 0.0}
+    (let [sorted (vec (sort values))
+          n      (count sorted)]
+      {:count n
+       :min   (double (first sorted))
+       :max   (double (last sorted))
+       :avg   (/ (reduce + 0.0 sorted) n)
+       :p50   (percentile sorted 0.50)
+       :p95   (percentile sorted 0.95)
+       :p99   (percentile sorted 0.99)})))
+
+(defn- fmt
+  "Formats a number for display."
+  [x]
+  (if (integer? x)
+    (str x)
+    (format "%.2f" (double x))))
+
+(defn- print-table!
+  "Prints a formatted ASCII table."
+  [title headers rows]
+  (let [col-widths (mapv (fn [i]
+                           (apply max
+                                  (count (nth headers i))
+                                  (map #(count (str (nth % i))) rows)))
+                         (range (count headers)))
+        separator  (str "+-" (str/join "-+-" (map #(apply str (repeat % \-)) col-widths)) "-+")
+        fmt-row    (fn [vals]
+                     (str "| "
+                          (str/join " | "
+                                    (map-indexed (fn [i v]
+                                                   (format (str "%-" (nth col-widths i) "s") (str v)))
+                                                 vals))
+                          " |"))]
+    (println)
+    (println title)
+    (println separator)
+    (println (fmt-row headers))
+    (println separator)
+    (doseq [row rows]
+      (println (fmt-row row)))
+    (println separator)
+    (println)))
+
+(defn- cleanup!
+  "Removes all benchmark data for the given run-id."
+  [id]
+  (t2/delete! :queue_message_batch :queue_name [:like (str "bench-" id "%")])
+  (doseq [[ch _] @listener/*listeners*]
+    (when (str/includes? (name ch) (str "bench-" id))
+      (mq/unlisten! ch))))
+
+(defn- publish-at-rate!
+  "Publishes `n` messages at the given rate (msgs/sec). `publish-fn` is called with each seq num.
+  If rate is nil, publishes as fast as possible (burst mode)."
+  [n rate publish-fn]
+  (if (nil? rate)
+    (dotimes [i n]
+      (publish-fn i))
+    (let [interval-ms (long (/ 1000 rate))]
+      (dotimes [i n]
+        (let [start (System/currentTimeMillis)]
+          (publish-fn i)
+          (let [elapsed (- (System/currentTimeMillis) start)
+                sleep-ms (- interval-ms elapsed)]
+            (when (pos? sleep-ms)
+              (Thread/sleep (long sleep-ms)))))))))
+
+(def ^:private rate-map
+  {:low    10
+   :medium 100
+   :high   1000
+   :burst  nil})
+
+(defn- rate->msgs-sec [rate]
+  (get rate-map rate rate))
+
+;;; ---------------------------------------- Benchmark 1: Publishing Throughput ----------------------------------------
+
+(defn bench-publish-throughput!
+  "Measures raw publishing speed for different batch sizes.
+  This benchmark bypasses the publish buffer to measure direct DB insert speed.
+  Options:
+    :n           - total messages to publish per batch-size test (default 1000)
+    :batch-sizes - vector of batch sizes to test (default [1 10 100])"
+  ([] (bench-publish-throughput! {}))
+  ([{:keys [n batch-sizes]
+     :or   {n 1000 batch-sizes [1 10 100]}}]
+   (ensure-started!)
+   (let [id      (run-id)
+         results (atom [])]
+     (try
+       (doseq [bs batch-sizes]
+         (let [channel    (make-channel :queue id (str "pub-" bs))
+               iterations (/ n bs)
+               start      (System/nanoTime)]
+           (binding [publish-buffer/*publish-buffer-ms* 0]
+             (dotimes [i iterations]
+               (let [msgs (vec (for [j (range bs)]
+                                 {:seq (+ (* i bs) j) :ts (System/currentTimeMillis)}))]
+                 (q.backend/publish! q.appdb/backend channel (payload/encode msgs)))))
+           (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+             (swap! results conj
+                    [(str bs) (fmt (/ n (/ elapsed-ms 1000.0))) (fmt elapsed-ms)]))))
+       (print-table!
+        (str "Publishing Throughput (queue, n=" n ")")
+        ["Batch Size" "Msgs/sec" "Total ms"]
+        @results)
+       (finally
+         (cleanup! id))))))
+
+;;; ---------------------------------------- Benchmark 2: End-to-End Latency ----------------------------------------
+
+(defn bench-e2e-latency!
+  "Measures end-to-end latency from publish to listener receipt via the full async pipeline.
+  Uses the real publish API including the publish buffer.
+  Publishes randomly to the 10 shared queues so multiple nodes compete for the same messages.
+  Options:
+    :n         - number of messages to publish (default 50)
+    :rate      - :low, :medium, :high, or :burst (default :medium)"
+  ([] (bench-e2e-latency! {}))
+  ([{:keys [n rate]
+     :or   {n 50 rate :medium}}]
+   (ensure-started!)
+   (let [latencies (atom [])
+         received  (atom 0)
+         channels  shared-queues]
+     (try
+       ;; Register listeners on all shared channels
+       (doseq [ch channels]
+         (when-not (listener/get-listener ch)
+           (listener/batch-listen! ch
+                                   (fn [msgs]
+                                     (doseq [msg msgs]
+                                       (let [receive-ns (System/nanoTime)
+                                             publish-ns (get msg :publish-ns)]
+                                         (when publish-ns
+                                           (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6))))
+                                       (swap! received inc))))))
+       (let [msgs-sec (rate->msgs-sec rate)
+             start    (System/nanoTime)]
+         ;; Publish to random shared channels
+         (publish-at-rate!
+          n msgs-sec
+          (fn [i]
+            (let [ch (random-shared-queue)]
+              (mq/with-queue ch [q]
+                (mq/put q {:seq i :publish-ns (System/nanoTime)})))))
+         ;; Force flush the publish buffer then wait for delivery
+         (publish-buffer/flush-publish-buffer!)
+         ;; Wait until we've received at least n messages or timeout
+         (let [deadline (+ (System/currentTimeMillis)
+                           (long (* 1000 (+ 30 (/ n (max 1 (or msgs-sec 1000)))))))]
+           (while (and (< @received n)
+                       (< (System/currentTimeMillis) deadline))
+             (Thread/sleep (long 100))))
+         (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)
+               stats      (collect-stats @latencies)]
+           (print-table!
+            (str "End-to-End Latency (queue, rate=" (name rate) ", n=" n ")")
+            ["Metric" "Value"]
+            [["Published" (str n)]
+             ["Received" (str @received)]
+             ["Other nodes" (str (max 0 (- n @received)))]
+             ["From others" (str (max 0 (- @received n)))]
+             ["Total ms" (fmt elapsed-ms)]
+             ["P50 ms" (fmt (:p50 stats))]
+             ["P95 ms" (fmt (:p95 stats))]
+             ["P99 ms" (fmt (:p99 stats))]
+             ["Min ms" (fmt (:min stats))]
+             ["Max ms" (fmt (:max stats))]
+             ["Avg ms" (fmt (:avg stats))]])))
+       (finally
+         (doseq [ch channels]
+           (when (listener/get-listener ch)
+             (mq/unlisten! ch))))))))
+
+;;; ---------------------------------------- Benchmark 3: Sustained Throughput ----------------------------------------
+
+(defn bench-sustained-throughput!
+  "Publishes continuously while consuming for a given duration via the full async pipeline.
+  Publishes randomly to the 10 shared queues.
+  Options:
+    :duration-sec - how long to publish (default 15)
+    :rate         - :low, :medium, :high, or :burst (default :medium)"
+  ([] (bench-sustained-throughput! {}))
+  ([{:keys [duration-sec rate]
+     :or   {duration-sec 15 rate :medium}}]
+   (ensure-started!)
+   (let [channels  shared-queues
+         received  (atom 0)
+         latencies (atom [])
+         published (atom 0)
+         stop?     (atom false)]
+     (try
+       ;; Register listeners on all shared channels
+       (doseq [ch channels]
+         (when-not (listener/get-listener ch)
+           (listener/batch-listen! ch
+                                   (fn [msgs]
+                                     (doseq [msg msgs]
+                                       (let [receive-ns (System/nanoTime)
+                                             publish-ns (get msg :publish-ns)]
+                                         (when publish-ns
+                                           (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6)))
+                                         (swap! received inc)))))))
+       (let [msgs-sec   (rate->msgs-sec rate)
+             start      (System/nanoTime)
+             pub-thread (Thread.
+                         (fn []
+                           (loop []
+                             (when-not @stop?
+                               (try
+                                 (let [ch (random-shared-queue)]
+                                   (mq/with-queue ch [q]
+                                     (mq/put q {:seq @published :publish-ns (System/nanoTime)})))
+                                 (swap! published inc)
+                                 (catch Exception _))
+                               (when msgs-sec
+                                 (Thread/sleep (long (/ 1000 msgs-sec))))
+                               (recur)))))]
+         (.start pub-thread)
+         (Thread/sleep (long (* duration-sec 1000)))
+         (reset! stop? true)
+         (.join pub-thread 5000)
+         ;; Flush publish buffer and wait for remaining messages to drain
+         (publish-buffer/flush-publish-buffer!)
+         (let [drain-start (System/currentTimeMillis)]
+           (while (and (< @received @published)
+                       (< (- (System/currentTimeMillis) drain-start) 30000))
+             (publish-buffer/flush-publish-buffer!)
+             (Thread/sleep (long 500))))
+         (let [elapsed-ms  (/ (- (System/nanoTime) start) 1e6)
+               stats       (collect-stats @latencies)]
+           (print-table!
+            (str "Sustained Throughput (queue, rate=" (name rate)
+                 ", duration=" duration-sec "s)")
+            ["Metric" "Value"]
+            [["Published" (str @published)]
+             ["Received" (str @received)]
+             ["Other nodes" (str (max 0 (- @published @received)))]
+             ["From others" (str (max 0 (- @received @published)))]
+             ["Duration ms" (fmt elapsed-ms)]
+             ["Actual pub/sec" (fmt (/ @published (/ elapsed-ms 1000.0)))]
+             ["Actual recv/sec" (fmt (/ @received (/ elapsed-ms 1000.0)))]
+             ["P50 latency ms" (fmt (:p50 stats))]
+             ["P95 latency ms" (fmt (:p95 stats))]
+             ["P99 latency ms" (fmt (:p99 stats))]
+             ["Max latency ms" (fmt (:max stats))]])))
+       (finally
+         (doseq [ch channels]
+           (when (listener/get-listener ch)
+             (mq/unlisten! ch))))))))
+
+;;; ---------------------------------------- Benchmark 4: Batch Listener ----------------------------------------
+
+(defn bench-batch-listener!
+  "Compares single-message vs batch listener processing rates.
+  Uses the shared queues so multiple nodes compete.
+  Options:
+    :n           - total messages to publish (default 200)
+    :batch-sizes - vector of max-batch-messages values to test (default [1 10 50])"
+  ([] (bench-batch-listener! {}))
+  ([{:keys [n batch-sizes]
+     :or   {n 200 batch-sizes [1 10 50]}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [bs batch-sizes]
+       (let [received (atom 0)
+             ;; `:max-batch-messages` is now an immutable per-queue property, so each batch
+             ;; size gets its own dedicated set of queues declared with that config.
+             channels (mapv #(keyword "queue" (str "bench-batch-" bs "-" %))
+                            (range (count shared-queues)))]
+         (try
+           (doseq [ch channels]
+             (ensure-queue! ch {:max-batch-messages bs})
+             (when-not (listener/get-listener ch)
+               (listener/batch-listen! ch
+                                       (fn [msgs]
+                                         (swap! received + (count msgs))))))
+           ;; Publish all messages randomly across this batch size's queues
+           (dotimes [i n]
+             (mq/with-queue (nth channels (rand-int (count channels))) [q]
+               (mq/put q {:seq i})))
+           (publish-buffer/flush-publish-buffer!)
+           ;; Wait for messages to be consumed
+           (let [start   (System/nanoTime)
+                 deadline (+ (System/currentTimeMillis) 30000)]
+             (while (and (< @received n)
+                         (< (System/currentTimeMillis) deadline))
+               (Thread/sleep (long 100)))
+             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+               (swap! results conj
+                      [(str bs) (str @received) (fmt elapsed-ms)
+                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
+           (finally
+             (doseq [ch channels]
+               (when (listener/get-listener ch)
+                 (mq/unlisten! ch))
+               (t2/delete! :queue_message_batch :queue_name (name ch)))))))
+     (print-table!
+      (str "Batch Listener Comparison (n=" n ")")
+      ["Max Batch" "Received" "Process ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Benchmark 5: Multiple Queues ----------------------------------------
+
+(defn bench-multi-queue!
+  "Measures performance with varying numbers of the shared queues.
+  Tests with subsets of the 10 shared queues to show scaling.
+  Options:
+    :queue-counts   - vector of queue counts to test (default [1 5 10])
+    :msgs-per-queue - messages per queue (default 50)"
+  ([] (bench-multi-queue! {}))
+  ([{:keys [queue-counts msgs-per-queue]
+     :or   {queue-counts [1 5 10] msgs-per-queue 50}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [qc queue-counts]
+       (let [channels (subvec shared-queues 0 (min qc (count shared-queues)))
+             total    (* (count channels) msgs-per-queue)
+             received (atom 0)]
+         (try
+           ;; Register listeners
+           (doseq [ch channels]
+             (when-not (listener/get-listener ch)
+               (listener/batch-listen! ch
+                                       (fn [msgs]
+                                         (swap! received + (count msgs))))))
+           ;; Publish to the queues
+           (let [start (System/nanoTime)]
+             (doseq [ch channels]
+               (dotimes [i msgs-per-queue]
+                 (mq/with-queue ch [q]
+                   (mq/put q {:seq i}))))
+             (publish-buffer/flush-publish-buffer!)
+             ;; Wait for messages
+             (let [deadline (+ (System/currentTimeMillis) 60000)]
+               (while (and (< @received total)
+                           (< (System/currentTimeMillis) deadline))
+                 (Thread/sleep (long 100))))
+             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+               (swap! results conj
+                      [(str (count channels)) (str total) (str @received) (fmt elapsed-ms)
+                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
+           (finally
+             (doseq [ch channels]
+               (when (listener/get-listener ch)
+                 (mq/unlisten! ch)))))))
+     (print-table!
+      (str "Multiple Queues (msgs-per-queue=" msgs-per-queue ")")
+      ["Queues" "Total Msgs" "Received" "Total ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Benchmark 6: Concurrent Consumers (LATERAL/SKIP LOCKED) ----------------------------------------
+
+(defn bench-concurrent-consumers!
+  "Validates the LATERAL/SKIP LOCKED fetch and exactly-once delivery under multi-consumer
+  contention by spinning up multiple `AppDbQueueBackend` instances in the same JVM, each
+  with its own listener atom and poll thread. All instances share the queue and DB.
+
+  What this actually measures:
+   - **Correctness**: every row is processed exactly once (`Dups` should always be 0; the
+     `Imbalance` shows how the work spread but the total received must equal `n`).
+   - **Per-row contention**: `mq.impl/active-handlers` is a JVM-global atom, so at most ONE
+     consumer can be actively handling the channel at a time within a single JVM. The other
+     consumers see `channel-busy?` and skip the queue. So this test does *not* validate
+     cross-node throughput scaling — for that you need separate JVMs.
+   - **Buffer-vs-row behavior**: `*publish-buffer-ms*` is forced to 0 so each `put` becomes
+     its own row; otherwise the buffer collapses all `n` messages into a single row that
+     only one consumer can grab.
+
+  Options:
+    :consumer-counts - vector of consumer counts to test (default [1 2 4 8])
+    :n               - total messages per run (default 2000)
+    :queue-name      - queue to use (default :queue/bench-concurrent)"
+  ([] (bench-concurrent-consumers! {}))
+  ([{:keys [consumer-counts n queue-name]
+     :or   {consumer-counts [1 2 4 8] n 2000 queue-name :queue/bench-concurrent}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [c consumer-counts]
+       (let [per-consumer-received (vec (repeatedly c #(atom 0)))
+             seen-batch-ids        (atom #{})
+             duplicates            (atom 0)
+             backends              (vec (repeatedly c q.appdb/make-backend))
+             listener-atoms        (vec (repeatedly c #(atom {})))]
+         (ensure-queue! queue-name)
+         (try
+           (doseq [i (range c)]
+             (binding [listener/*listeners* (nth listener-atoms i)]
+               (listener/batch-listen! queue-name
+                                       (fn [msgs]
+                                         (doseq [m msgs]
+                                           (let [bid (get m "batch-id" m)]
+                                             (when (contains? @seen-batch-ids bid)
+                                               (swap! duplicates inc))
+                                             (swap! seen-batch-ids conj bid)))
+                                         (swap! (nth per-consumer-received i) + (count msgs))))
+               (q.backend/start! (nth backends i))))
+           (let [start (System/nanoTime)]
+             ;; Bypass the time-windowed publish buffer (set *publish-buffer-ms* to 0) so each
+             ;; `put` becomes its own DB row — otherwise all `n` messages collapse into one row
+             ;; and there's nothing for multiple consumers to fetch in parallel.
+             (binding [publish-buffer/*publish-buffer-ms* 0]
+               (dotimes [i n]
+                 (mq/with-queue queue-name [q]
+                   (mq/put q {:id i})))
+               (publish-buffer/flush-publish-buffer!))
+             (let [total (atom 0)
+                   deadline (+ (System/currentTimeMillis) 60000)]
+               (while (and (< @total n)
+                           (< (System/currentTimeMillis) deadline))
+                 ;; Notify all poll threads so consumer backends drain promptly. In production
+                 ;; the publish-side notify reaches the same node; multi-instance benchmarks
+                 ;; sit between nodes where no cross-node wakeup exists.
+                 (polling/notify-all!)
+                 (reset! total (reduce + (map deref per-consumer-received)))
+                 (Thread/sleep 50))
+               (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)
+                     received-per-consumer (mapv deref per-consumer-received)
+                     min-c   (apply min received-per-consumer)
+                     max-c   (apply max received-per-consumer)
+                     spread  (when (pos? max-c)
+                               (* 100.0 (- 1.0 (/ (double min-c) max-c))))]
+                 (swap! results conj
+                        [(str c)
+                         (str @total)
+                         (str (apply max received-per-consumer))
+                         (str (apply min received-per-consumer))
+                         (if spread (format "%.1f%%" spread) "n/a")
+                         (str @duplicates)
+                         (fmt elapsed-ms)
+                         (fmt (/ @total (/ elapsed-ms 1000.0)))]))))
+           (finally
+             ;; Shut each backend's polling thread (per-instance, doesn't touch production)
+             (doseq [be backends] (q.backend/shutdown! be))
+             ;; Clean DB rows for this queue
+             (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+     (print-table!
+      (str "Concurrent Consumers (n=" n " per run, queue=" (name queue-name) ")")
+      ["Consumers" "Total" "Max/c" "Min/c" "Imbalance" "Dups" "Total ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Benchmark 7: Stale-Recovery Race ----------------------------------------
+
+(defn bench-stale-recovery-race!
+  "Pre-populates `:n` rows as `status=processing` with a stale heartbeat, then runs the
+  recovery function from `:workers` threads simultaneously. Validates that the guarded
+  bulk UPDATE delivers each row to exactly one winner regardless of concurrency.
+
+  Options:
+    :n       - number of stale rows to seed (default 500)
+    :workers - concurrent recovery threads (default 8)"
+  ([] (bench-stale-recovery-race! {}))
+  ([{:keys [n workers] :or {n 500 workers 8}}]
+   (ensure-started!)
+   (let [queue-name (keyword "queue" (str "bench-stale-" (run-id)))
+         stale-ts   (java.sql.Timestamp/from
+                     (.minusMillis (java.time.Instant/now) (* 20 60 1000)))]
+     (try
+       (let [rows (vec (for [_ (range n)]
+                         {:queue_name (name queue-name)
+                          :payload    "[]"
+                          :status     "processing"
+                          :status_heartbeat stale-ts
+                          :owner      "dead-node"}))]
+         (t2/insert! :queue_message_batch rows))
+       (let [recoveries (atom [])
+             be         (q.appdb/make-backend)
+             latch      (CountDownLatch. 1)
+             threads    (mapv (fn [_]
+                                (let [r (bound-fn []
+                                          (.await latch)
+                                          ;; Large max-retries so every stale row is recovered to 'pending'
+                                          ;; rather than dropped, matching the original "all recovered" check.
+                                          (let [results     (q.backend/recover-stale! be (* 10 60 1000) Integer/MAX_VALUE)
+                                                n-recovered  (reduce + 0 (map #(+ (:recovered % 0) (:failed % 0)) results))]
+                                            (swap! recoveries conj n-recovered)))]
+                                  (Thread. ^Runnable r)))
+                              (range workers))
+             start      (System/nanoTime)]
+         (run! #(.start ^Thread %) threads)
+         (.countDown latch)
+         (run! #(.join ^Thread % 30000) threads)
+         (let [elapsed-ms  (/ (- (System/nanoTime) start) 1e6)
+               total-rec   (reduce + @recoveries)
+               final-state (frequencies (map :status (t2/select :queue_message_batch
+                                                                :queue_name (name queue-name))))]
+           (print-table!
+            (str "Stale Recovery Race (n=" n " stale, workers=" workers ")")
+            ["Metric" "Value"]
+            [["Sum of returned counts" (str total-rec)]
+             ["Stale rows pre-populated" (str n)]
+             ["Sum == n? (correctness)" (if (= total-rec n) "YES" (str "NO  diff=" (- total-rec n)))]
+             ["Final 'pending'" (str (get final-state "pending" 0))]
+             ["Final 'failed'" (str (get final-state "failed" 0))]
+             ["Final 'processing'" (str (get final-state "processing" 0))]
+             ["Total wall-clock ms" (fmt elapsed-ms)]
+             ["Per-row ms (effective)" (fmt (/ elapsed-ms (max 1 n)))]])))
+       (finally
+         (t2/delete! :queue_message_batch :queue_name (name queue-name)))))))
+
+;;; ---------------------------------------- Benchmark 8: Concurrent Publish + Consume ----------------------------------------
+
+(defn bench-concurrent-pub-consume!
+  "Runs a publisher thread and a consumer simultaneously against one queue, measuring
+  end-to-end latency under write contention. Unlike `bench-e2e-latency!`, the publisher and
+  the wait-for-receive happen on separate threads, so we exercise the actual
+  publish-while-consume code path (buffer atom contention, fetch contending with INSERT,
+  the per-channel `active-handlers` gate).
+
+  Each published message carries a `publish-ns` timestamp; the listener measures
+  `receive-ns - publish-ns` and records it as the per-message latency.
+
+  Options:
+    :duration-sec - how long to publish for (default 10)
+    :pub-rate-msgs-sec - target publish rate (default 50)
+    :queue-name   - queue to use (default :queue/bench-pub-consume)"
+  ([] (bench-concurrent-pub-consume! {}))
+  ([{:keys [duration-sec pub-rate-msgs-sec queue-name]
+     :or   {duration-sec 10 pub-rate-msgs-sec 50 queue-name :queue/bench-pub-consume}}]
+   (ensure-started!)
+   (let [received    (atom 0)
+         latencies   (atom [])
+         stop?       (atom false)
+         publisher-published (atom 0)]
+     (ensure-queue! queue-name)
+     (try
+       (listener/batch-listen! queue-name
+                               (fn [msgs]
+                                 (let [now (System/nanoTime)]
+                                   (doseq [m msgs]
+                                     (when-let [pub-ns (get m :publish-ns)]
+                                       (swap! latencies conj (/ (- now (double pub-ns)) 1e6))))
+                                   (swap! received + (count msgs)))))
+       (let [pub-runnable (bound-fn []
+                            (let [interval-ns (long (/ 1e9 pub-rate-msgs-sec))
+                                  stop-at-ns  (+ (System/nanoTime) (* 1e9 duration-sec))]
+                              (loop []
+                                (when (and (not @stop?) (< (System/nanoTime) stop-at-ns))
+                                  (let [next-pub-ns (+ (System/nanoTime) interval-ns)]
+                                    (mq/with-queue queue-name [q]
+                                      (mq/put q {:publish-ns (System/nanoTime)}))
+                                    (swap! publisher-published inc)
+                                    (let [sleep-ns (- next-pub-ns (System/nanoTime))]
+                                      (when (pos? sleep-ns)
+                                        (Thread/sleep (long (/ sleep-ns 1e6))
+                                                      (int (mod sleep-ns 1e6)))))
+                                    (recur))))))
+             pub-thread (Thread. ^Runnable pub-runnable)
+             start (System/nanoTime)]
+         (.start pub-thread)
+         (.join pub-thread (long (* 1000 (+ duration-sec 1))))
+         (reset! stop? true)
+         ;; Drain — wait for the consumer to catch up after publishing stops.
+         (let [drain-deadline (+ (System/currentTimeMillis) 30000)]
+           (while (and (< @received @publisher-published)
+                       (< (System/currentTimeMillis) drain-deadline))
+             (polling/notify-all!)
+             (Thread/sleep 50)))
+         (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)
+               stats      (collect-stats @latencies)]
+           (print-table!
+            (str "Concurrent Pub+Consume (rate=" pub-rate-msgs-sec "/sec, duration=" duration-sec "s"
+                 (when @rtt-ms (str ", rtt=" (first @rtt-ms) "-" (last @rtt-ms) "ms"))
+                 ")")
+            ["Metric" "Value"]
+            [["Published"        (str @publisher-published)]
+             ["Received"         (str @received)]
+             ["Drain ms"         (fmt (- elapsed-ms (* 1000 duration-sec)))]
+             ["P50 latency ms"   (fmt (:p50 stats))]
+             ["P95 latency ms"   (fmt (:p95 stats))]
+             ["P99 latency ms"   (fmt (:p99 stats))]
+             ["Max latency ms"   (fmt (:max stats))]
+             ["Pub/sec (actual)" (fmt (/ @publisher-published (double duration-sec)))]
+             ["Recv/sec total"   (fmt (/ @received (/ elapsed-ms 1000.0)))]])))
+       (finally
+         (reset! stop? true)
+         (mq/unlisten! queue-name)
+         (t2/delete! :queue_message_batch :queue_name (name queue-name)))))))
+
+;;; ---------------------------------------- Run All ----------------------------------------
+
+(defn run-all-benchmarks!
+  "Runs all benchmark scenarios with default parameters."
+  []
+  (println "=== MQ Performance Benchmarks ===")
+  (println)
+
+  (println "--- 1/5: Publishing Throughput ---")
+  (bench-publish-throughput! {:n 1000})
+
+  (println "--- 2/5: End-to-End Latency ---")
+  (doseq [rate [:low :medium :high :burst]]
+    (bench-e2e-latency! {:n 50 :rate rate}))
+
+  (println "--- 3/5: Sustained Throughput ---")
+  (bench-sustained-throughput! {:duration-sec 15 :rate :medium})
+
+  (println "--- 4/5: Batch Listener Comparison ---")
+  (bench-batch-listener! {:n 200})
+
+  (println "--- 5/7: Multiple Queues ---")
+  (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})
+
+  (println "--- 6/7: Concurrent Consumers ---")
+  (bench-concurrent-consumers! {:consumer-counts [1 2 4 8] :n 2000})
+
+  (println "--- 7/7: Stale Recovery Race ---")
+  (bench-stale-recovery-race! {:n 500 :workers 8})
+
+  (println "=== All benchmarks complete ==="))
+
+;;; ---------------------------------------- Multi-Node Coordination ----------------------------------------
+

dev/src/dev/mq_perf_multi.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# Runs multiple MQ benchmark nodes against the same database.
+#
+# Usage:
+#   ./dev/src/dev/mq_perf_multi.sh start       # Start 2 nodes (default)
+#   ./dev/src/dev/mq_perf_multi.sh start 4     # Start 4 nodes
+#   ./dev/src/dev/mq_perf_multi.sh go          # Send go signal to all waiting nodes
+#   ./dev/src/dev/mq_perf_multi.sh stop        # Stop all running nodes
+#   ./dev/src/dev/mq_perf_multi.sh logs        # Tail all node logs
+#
+# Each node starts a lightweight JVM with just DB + MQ (no server),
+# waits for a coordinated go signal, then runs all benchmarks.
+
+set -euo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+LOG_DIR="target/mq-perf-logs"
+PID_DIR="target/mq-perf-pids"
+
+cmd_start() {
+  local num_nodes="${1:-2}"
+  mkdir -p "$LOG_DIR" "$PID_DIR"
+
+  # Clean up stale pids
+  rm -f "$PID_DIR"/*.pid
+
+  echo "Starting $num_nodes MQ benchmark nodes..."
+  echo "Logs in $LOG_DIR/"
+  echo
+
+  for i in $(seq 1 "$num_nodes"); do
+    local log_file="$LOG_DIR/node-$i.log"
+    clojure -M:dev:ee:ee-dev -e "
+      (require 'dev.mq-perf)
+      (dev.mq-perf/start-mq!)
+      (dev.mq-perf/run-all-benchmarks-coordinated!)
+    " > "$log_file" 2>&1 &
+    local pid=$!
+    echo "$pid" > "$PID_DIR/node-$i.pid"
+    echo "  Node $i started (PID $pid, log: $log_file)"
+  done
+
+  echo
+  echo "Nodes are initializing. Once ready, run:"
+  echo "  $0 go"
+}
+
+cmd_go() {
+  echo "Sending go signal..."
+  clojure -M:dev:ee:ee-dev -e "
+    (require 'dev.mq-perf)
+    (dev.mq-perf/start-mq!)
+    (dev.mq-perf/signal-go!)
+  "
+  echo
+  echo "Benchmarks started. Watch progress with:"
+  echo "  $0 logs"
+}
+
+cmd_stop() {
+  if [ ! -d "$PID_DIR" ]; then
+    echo "No nodes running."
+    return
+  fi
+
+  for pid_file in "$PID_DIR"/*.pid; do
+    [ -f "$pid_file" ] || continue
+    local pid
+    pid=$(cat "$pid_file")
+    local name
+    name=$(basename "$pid_file" .pid)
+    if kill -0 "$pid" 2>/dev/null; then
+      kill "$pid"
+      echo "Stopped $name (PID $pid)"
+    else
+      echo "$name (PID $pid) already exited"
+    fi
+    rm -f "$pid_file"
+  done
+}
+
+cmd_logs() {
+  if [ ! -d "$LOG_DIR" ]; then
+    echo "No logs found."
+    return
+  fi
+  tail -f "$LOG_DIR"/node-*.log
+}
+
+case "${1:-}" in
+  start) cmd_start "${2:-2}" ;;
+  go)    cmd_go ;;
+  stop)  cmd_stop ;;
+  logs)  cmd_logs ;;
+  *)
+    echo "Usage: $0 {start [N]|go|stop|logs}"
+    echo
+    echo "  start [N]  Start N benchmark nodes (default 2)"
+    echo "  go         Send go signal to start benchmarks"
+    echo "  stop       Kill all running nodes"
+    echo "  logs       Tail all node logs"
+    exit 1
+    ;;
+esac

mage/src/mage/modules.clj

@@ -123,6 +123,7 @@
      login-history
      mcp
      metabot
+     mq
      notification
      oauth-server
      permissions

resources/migrations/062/20260512_persistent_queue.yaml

@@ -0,0 +1,79 @@
+## Adds support for appdb persistent queue backend
+
+databaseChangeLog:
+  - changeSet:
+      id: v62.asdgew
+      author: nvoxland
+      comment: Create the table backing the message queue
+      changes:
+        - createTable:
+            tableName: queue_message_batch
+            remarks: Lightweight message queue for cross-cluster communication
+            columns:
+              - column:
+                  name: id
+                  type: BIGINT
+                  autoIncrement: true
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: queue_name
+                  type: varchar(254)
+                  remarks: The name of queue this message is in
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: The timestamp of when the messages were created
+                  name: created_at
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: The status of the messages
+                  name: status
+                  type: varchar(10)
+                  defaultValue: 'pending'
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: Heartbeat to ensure status is not stuck
+                  name: status_heartbeat
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: Number of failures processing these messages
+                  name: failures
+                  type: int
+                  defaultValueNumeric: 0
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: Current owner of the messages
+                  name: owner
+                  type: varchar(36)
+              - column:
+                  remarks: JSON-encoded payload
+                  name: payload
+                  type: ${text.type}
+                  constraints:
+                    nullable: false
+
+  - changeSet:
+      id: v62.gdasfg2e
+      author: nvoxland
+      comment: Index queue_message_batch for efficient fetch
+      changes:
+        - createIndex:
+            tableName: queue_message_batch
+            indexName: idx_queue_message_fetch
+            columns:
+              - column:
+                  name: queue_name
+              - column:
+                  name: status
+              - column:
+                  name: id

src/metabase/analytics/prometheus.clj

@@ -648,6 +648,48 @@
                        {:description "A metabot turn was started (user row + assistant placeholder inserted)"
                         :labels [:profile-id]})
 
+   ;; messaging metrics
+   (prometheus/gauge :metabase-mq/queue-depth
+                     {:description "Batch count per queue by status, across all queue backends."
+                      :labels [:backend :channel :status]})
+   (prometheus/counter :metabase-mq/queue-poll-results
+                       {:description "Queue poll results by outcome, across all queue backends."
+                        :labels [:backend :result]})
+   (prometheus/counter :metabase-mq/batch-stale-recoveries
+                       {:description "Batches recovered from stale processing state."
+                        :labels [:backend :transport :channel]})
+   (prometheus/counter :metabase-mq/batches-handled
+                       {:description "Batches handled by status."
+                        :labels [:transport :channel :status]})
+   (prometheus/counter :metabase-mq/dedup-messages-dropped
+                       {:description "Messages dropped by dedup before publishing."
+                        :labels [:channel]})
+   (prometheus/histogram :metabase-mq/handle-duration-ms
+                         {:description "Duration in milliseconds to process a batch."
+                          :labels [:transport :channel]
+                          :buckets [1 5 10 50 100 500 1000 5000 10000 30000]})
+   (prometheus/counter :metabase-mq/messages-published
+                       {:description "Total messages published."
+                        :labels [:transport :channel]})
+   (prometheus/gauge :metabase-mq/publish-buffer-depth
+                     {:description "Messages sitting in the publish buffer awaiting flush."
+                      :labels [:channel]})
+   (prometheus/counter :metabase-mq/publish-buffer-flush-errors
+                       {:description "Publish buffer flush failures where messages were re-buffered for retry."
+                        :labels [:channel]})
+   (prometheus/counter :metabase-mq/queue-batch-permanent-failures
+                       {:description "Queue batches that exhausted retries."
+                        :labels [:backend :channel]})
+   (prometheus/counter :metabase-mq/queue-batch-retries
+                       {:description "Queue batches retried after transient failure."
+                        :labels [:backend :channel]})
+   (prometheus/counter :metabase-mq/messages-received
+                       {:description "Individual messages delivered to handlers."
+                        :labels [:transport :channel]})
+   (prometheus/counter :metabase-mq/handler-errors
+                       {:description "Errors thrown by queue message handlers."
+                        :labels [:transport :channel]})
+
    ;; release dashboard metrics
    (prometheus/counter :metabase-sync/failures
                        {:description "Number of sync operation failures."

src/metabase/app_db/connection.clj

@@ -134,23 +134,66 @@
 
 (def ^:private ^:dynamic *transaction-depth* 0)
 
+(def ^:dynamic *after-commit*
+  "When non-nil, an atom containing a vector of 0-arity fns to call after the outermost transaction commits.
+  Bound to a fresh atom at the outermost transaction boundary."
+  nil)
+
+(def ^:dynamic *transaction-state*
+  "When non-nil, an atom containing a map for arbitrary per-transaction data.
+  Any subsystem can store namespaced keys here. Bound to a fresh atom at the outermost transaction boundary."
+  nil)
+
 (defn in-transaction?
   "Whether we are currently in a transaction."
   []
   (pos? *transaction-depth*))
 
+(defn after-commit!
+  "Register `f` (a 0-arity fn) to run after the current outermost transaction commits.
+  Returns true if registered, false if not currently in a transaction."
+  [f]
+  (if *after-commit*
+    (do (swap! *after-commit* conj f) true)
+    false))
+
+(defn transaction-state
+  "Returns the current transaction state atom, or nil if not in a transaction."
+  []
+  *transaction-state*)
+
 (defn- do-transaction [^java.sql.Connection connection f]
   (letfn [(thunk []
-            (let [savepoint (.setSavepoint connection)]
+            (let [savepoint          (.setSavepoint connection)
+                  commit-snapshot    (when (and *after-commit* (> *transaction-depth* 1))
+                                       @*after-commit*)
+                  state-snapshot     (when (and *transaction-state* (> *transaction-depth* 1))
+                                       @*transaction-state*)]
               (try
                 (let [result (f connection)]
                   (when (= *transaction-depth* 1)
                     ;; top-level transaction, commit
-                    (.commit connection))
+                    (.commit connection)
+                    ;; drain and execute after-commit callbacks
+                    (loop []
+                      (when-let [callbacks (seq (first (reset-vals! *after-commit* [])))]
+                        (doseq [cb callbacks]
+                          (try
+                            (cb)
+                            (catch Throwable t
+                              (log/error t "Error in after-commit callback"))))
+                        ;; If callbacks registered new callbacks, drain those too
+                        (recur))))
                   result)
                 (catch Throwable txn-e
                   (try
                     (.rollback connection savepoint)
+                    ;; Restore after-commit and transaction-state to pre-savepoint values
+                    ;; so callbacks/state from the rolled-back savepoint are discarded
+                    (when commit-snapshot
+                      (reset! *after-commit* commit-snapshot))
+                    (when state-snapshot
+                      (reset! *transaction-state* state-snapshot))
                     (catch Exception rollback-e
                       (throw (ex-info
                               (str "Error rolling back after previous error: " (ex-message txn-e))
@@ -202,8 +245,11 @@
                     {:options options}))
 
     :else
-    (binding [*transaction-depth* (inc *transaction-depth*)]
-      (do-transaction connection f))))
+    (let [outermost? (zero? *transaction-depth*)]
+      (binding [*transaction-depth* (inc *transaction-depth*)
+                *after-commit*      (if outermost? (atom []) *after-commit*)
+                *transaction-state* (if outermost? (atom {}) *transaction-state*)]
+        (do-transaction connection f)))))
 
 (methodical/defmethod t2.pipeline/transduce-query :before :default
   "Make sure application database calls are not done inside core.async dispatch pool threads. This is done relatively

src/metabase/app_db/core.clj

@@ -37,7 +37,9 @@
   db-type
   in-transaction?
   quoting-style
-  unique-identifier]
+  unique-identifier
+  after-commit!
+  transaction-state]
 
  [mdb.connection-pool-setup
   recent-activity?]

src/metabase/core/core.clj

@@ -105,6 +105,7 @@
   []
   (log/info "Metabase Shutting Down ...")
   (queue/stop-listeners!)
+  (startup/run-shutdown-logic!)
   (task/stop-scheduler!)
   (server/stop-web-server!)
   (tracing/shutdown!)

src/metabase/core/init.clj

@@ -43,6 +43,7 @@
    [metabase.metabot.init]
    [metabase.model-persistence.init]
    [metabase.models.init]
+   [metabase.mq.init]
    [metabase.notification.init]
    [metabase.oauth-server.init]
    [metabase.parameters.init]

src/metabase/mq/README.md

@@ -0,0 +1,98 @@
+# metabase.mq
+
+A persistent message queue for Metabase. Supports both durable (database-backed) and in-memory transports with a
+unified API.
+
+## Quick Start
+
+```clojure
+;; Declare the queue (its broker-side identity and properties)
+(mq/def-queue! :queue/my-task)
+
+;; Register a listener (the consumer-side handler)
+(mq/def-listener! :queue/my-task
+  [msg]
+  (do-work! msg))
+
+;; Publish a message (inside a transaction — delivers after commit)
+(t2/with-transaction [_]
+  (mq/with-queue :queue/my-task [q]
+    (mq/put q {:key "value"})))
+```
+
+All queue config — `:exclusive`, `:max-batch-messages`, `:dedup-fn` — belongs on `def-queue!`,
+because it takes effect at publish time on every node, regardless of where listeners are
+registered. `def-listener!` only wires up the consumer-side handler. A listener registration
+throws if its queue hasn't been declared yet — typos are caught at startup, not at first
+publish.
+
+## Publishing Semantics
+
+`with-queue` is designed to read like a direct publish — you write a message and move on. Under the hood, delivery is intentionally deferred through several layers:
+
+1. **Macro body must succeed.** Messages are only enqueued if the body of `with-queue` returns normally. An exception discards them.
+
+2. **Surrounding transaction must commit.** If the call is inside a `t2/with-transaction` block (directly or transitively), messages are held until the transaction commits. A rollback discards them. This means a message will never be delivered for a database change that didn't happen.
+
+3. **Batching window.** After the transaction commits, messages enter a time-windowed buffer. Rapid-fire publishes to the same channel within a sliding window are coalesced into a single batch before hitting the transport layer. This amortizes per-batch overhead during bursts.
+
+4. **Deduplication.** Before dispatch, duplicate messages in the same batch are removed.
+
+## Message serialization
+
+Messages must be JSON-serializable. A batch is encoded to a JSON string once, at the publish
+boundary (`mq.transport/publish!` → `mq.payload/encode`), and decoded once, at the delivery
+boundary (`mq.impl/deliver!` → `mq.payload/decode`). In between, backends move the opaque string
+around and never look inside it — so encoding is not a backend concern, and every backend
+delivers an identical shape. Decoding keywordizes map keys, but values pass through JSON:
+keyword values become strings, sets become vectors, dates become strings.
+
+## Queue semantics
+
+|               | Queue                                       |
+|---------------|---------------------------------------------|
+| Delivery      | At-least-once per listener                  |
+| Retry         | Yes — up to `queue-max-retries` (default 5) |
+| Backing table | `queue_message_batch`                       |
+| Backlog       | Persists until processed or exhausted       |
+| Use when      | Work must complete reliably                 |
+
+## Architecture
+
+```
+User Code
+  with-queue
+        │
+        ▼
+Publish Pipeline (mq.publish)
+  collect inside tx → batch 100ms window → deduplicate
+        │
+        ▼
+Transport Dispatch (mq.transport)   ← multimethod on channel namespace
+  :queue → queue backends
+        │
+        ▼
+Backends
+  appdb  — FOR UPDATE SKIP LOCKED
+  memory — LinkedBlockingQueue per channel
+        │
+        ▼
+Delivery Engine (mq.impl)
+  worker pool → handle! → listener fn
+```
+### Idempotency
+
+Queue listeners must be idempotent. A batch can return to `pending` after a partial failure, and the same messages will
+be redelivered.
+
+## Testing
+
+Use `with-test-mq` from `metabase.mq.test` to run tests with an isolated in-memory backend. It supports double-delivery
+chaos testing to verify listener idempotency.
+
+```clojure
+(deftest my-test
+  (mq.test/with-test-mq
+    (mq/with-queue :queue/my-task {:id 1})
+    (is (= 1 @processed-count))))
+```

src/metabase/mq/core.clj

@@ -0,0 +1,73 @@
+(ns metabase.mq.core
+  "Unified public API for the Metabase messaging subsystem.
+
+  **Queue** — single-consumer, at-least-once delivery.  Each message is processed by exactly
+  one listener and removed after successful processing.  Failed messages are retried up to a
+  configurable limit.
+
+  A queue is declared once with `def-queue!` — this is the source of truth for what queues
+  exist and where all queue config lives (exclusivity, batch size, dedup). The consumer-side
+  handler is added with `def-listener!`. Splitting the two means publishers can route to
+  queues declared anywhere and listeners can live on any node, with queue config taking
+  effect at publish time on every node regardless of where listeners are registered.
+
+      (mq/def-queue! :queue/simple-task)
+      (mq/def-listener! :queue/simple-task [messages]
+        (doseq [msg messages] (process msg)))
+
+  Queue-level config:
+       (mq/def-queue! :queue/search-reindex {:exclusive true :max-batch-messages 50})
+       (mq/def-listener! :queue/search-reindex [messages]
+         (process-batch messages))
+
+  See [[metabase.mq.queue.registry/def-queue!]] for the full set of queue options.
+
+  Publishing is done using `with-queue` which binds a queue you can put messages on.
+  When the body finishes successfully, the message(s) in the bound queue actually publishes the messages.
+
+      (with-queue :queue/simple-task [q]
+        (put q message))"
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.init :as mq.init]
+   [metabase.mq.listener :as mq.listener]
+   [metabase.mq.publish :as mq.publish]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.impl :as q.impl]
+   [metabase.mq.queue.memory :as q.memory]
+   [metabase.mq.queue.registry :as q.registry]
+   [potemkin :as p]))
+
+(set! *warn-on-reflection* true)
+
+(comment
+  mq.impl/keep-me
+  mq.init/keep-me
+  mq.listener/keep-me
+  mq.publish/keep-me
+  q.appdb/keep-me
+  q.impl/keep-me
+  q.memory/keep-me
+  q.registry/keep-me)
+
+(p/import-vars
+ [mq.listener
+  def-listener!
+  register-listeners!
+  unlisten!]
+
+ [mq.publish
+  put]
+
+ [mq.impl
+  last-activity]
+
+ [mq.init
+  start!
+  stop!]
+
+ [q.impl
+  with-queue]
+
+ [q.registry
+  def-queue!])

src/metabase/mq/impl.clj

@@ -0,0 +1,194 @@
+(ns metabase.mq.impl
+  "Backend coordination: message delivery, analytics helpers, and lifecycle management."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.payload :as payload]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Callable ExecutorService Executors Future)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private active-handlers
+  "channel keyword → {:future Future, :metadata map}
+   Tracks which channels have an active handler running across all backends."
+  (atom {}))
+
+(def ^:private worker-pool
+  "Shared thread pool for processing messages off backend poll threads."
+  (atom nil))
+
+(def ^:private last-activity*
+  "channel keyword → nanoTime of last publish or handler completion."
+  (atom {}))
+
+(defn last-activity
+  "Returns the nanoTime of the last publish or handler completion for the given channel, or nil."
+  [channel]
+  (get @last-activity* channel))
+
+(defn record-publish-activity!
+  "Records a publish event for the given channel. Called by the publishing pipeline."
+  [channel]
+  (swap! last-activity* assoc channel (System/nanoTime)))
+
+(defn channel-busy?
+  "Returns true if the given channel has an active handler running."
+  [channel]
+  (contains? @active-handlers channel))
+
+(defn busy-channels
+  "Returns the set of channels that currently have an active handler."
+  []
+  (set (keys @active-handlers)))
+
+(defn active-handler-metadata
+  "Returns the metadata map for the active handler on the given channel, or nil.
+   Backends store backend-specific data here (e.g. queue backends store :batch-id for heartbeats)."
+  [channel]
+  (get-in @active-handlers [channel :metadata]))
+
+(defn invoke-listener!
+  "Common listener invocation skeleton.
+  Looks up listener via `listener-fn`, times execution, logs errors,
+  and records metrics. Calls `on-success` / `on-error` for queue ACK/NACK."
+  [{:keys [channel listener-fn invoke-fn on-success on-error]}]
+  (let [transport (namespace channel)
+        listener  (listener-fn)
+        labels    {:transport transport :channel (name channel)}
+        start     (System/nanoTime)]
+    (try
+      (if-not listener
+        (log/debugf "No listener registered for %s %s, skipping message" transport (name channel))
+        (do
+          (invoke-fn listener)
+          (when on-success (on-success))
+          (analytics/inc! :metabase-mq/batches-handled (assoc labels :status "success"))))
+      (catch Exception e
+        (log/error e (str "Error handling " transport " message") labels)
+        (when on-error (on-error e))
+        (analytics/inc! :metabase-mq/batches-handled (assoc labels :status "error")))
+      (finally
+        (analytics/observe! :metabase-mq/handle-duration-ms labels
+                            (/ (double (- (System/nanoTime) start)) 1e6))))))
+
+(defn- handle-batch-failure!
+  "Decides retry vs. permanent failure for a just-failed batch based on its prior failure count
+  and `queue-max-retries`, emits the matching metric, and tells the backend to re-enqueue or drop
+  it. No-ops if the backend no longer owns/knows the batch (`failure-count` returns nil). The
+  policy lives here so it's identical across backends."
+  [backend channel batch-id]
+  (when-let [failures (q.backend/failure-count backend channel batch-id)]
+    (let [labels {:backend (name (q.backend/backend-id backend)) :channel (name channel)}]
+      (if (>= (inc failures) (mq.settings/queue-max-retries))
+        (do
+          (log/warnf "Batch %s on %s exhausted retries (%d), dropping"
+                     batch-id channel (mq.settings/queue-max-retries))
+          (analytics/inc! :metabase-mq/queue-batch-permanent-failures labels)
+          (q.backend/fail-batch! backend channel batch-id))
+        (do
+          (analytics/inc! :metabase-mq/queue-batch-retries labels)
+          (q.backend/retry-batch! backend channel batch-id))))))
+
+(defn handle!
+  "Handles accumulated messages for a channel by invoking the registered listener.
+   Processes each message/batch with error isolation — one failure doesn't block others.
+   For queues, batches use all-or-nothing ACK semantics: if ANY message in the batch
+   fails, the ENTIRE batch is nacked and will be retried. This means successfully
+   processed messages may be re-delivered. Queue listeners MUST be idempotent.
+   The queue's :dedup-fn helps mitigate duplicate processing on retry."
+  [channel message-batches messages]
+  (swap! last-activity* assoc channel (System/nanoTime))
+  (let [batch-size (q.registry/max-batch-messages channel)
+        transport  (namespace channel)
+        labels     {:transport transport :channel (name channel)}]
+    (analytics/inc! :metabase-mq/messages-received labels (count messages))
+    (invoke-listener!
+     {:channel      channel
+      :listener-fn  #(:listener (listener/get-listener channel))
+      :invoke-fn    (fn [h]
+                      (let [error (atom nil)]
+                        (doseq [batch (partition-all batch-size messages)]
+                          (try (h (vec batch))
+                               (catch Exception e
+                                 (analytics/inc! :metabase-mq/handler-errors labels)
+                                 (reset! error e)
+                                 (log/errorf e "Error handling %s message for %s, continuing"
+                                             transport (name channel)))))
+                        (when-let [e @error]
+                          (throw (ex-info "One or more messages failed" {} e)))))
+      :on-success   #(doseq [[bid backend] message-batches]
+                       (q.backend/batch-successful! backend channel bid))
+      :on-error     (fn [_e]
+                      (doseq [[bid backend] message-batches]
+                        (handle-batch-failure! backend channel bid)))})))
+
+(defn deliver!
+  "Called by backends when a payload is ready for delivery. Decodes the opaque payload
+   string (the single point where messages re-enter the typed world) and hands the messages
+   to handle! for processing."
+  [channel payload batch-id backend]
+  (handle! channel (if batch-id {batch-id backend} {}) (payload/decode payload)))
+
+(defn submit-delivery!
+  "Submits a delivery to the shared worker pool for non-blocking processing.
+   Returns true if submitted, false if the channel already has an active handler.
+   `metadata` is an opaque map stored alongside the future — backends can use it for
+   backend-specific tracking (e.g. queues store {:batch-id id} for heartbeat updates).
+   Uses bound-fn to convey dynamic bindings to the worker thread.
+   The busy-check and registration are atomic via a single swap!.
+   A generation counter prevents a completed future's cleanup from clobbering a
+   re-submission that won the slot between the finally and the dissoc."
+  [channel payload batch-id backend metadata]
+  (let [gen     (Object.)
+        [old _] (swap-vals! active-handlers
+                            (fn [handlers]
+                              (if (contains? handlers channel)
+                                handlers
+                                (assoc handlers channel {:future nil :metadata metadata :gen gen}))))]
+    (if (contains? old channel)
+      false
+      (let [^Callable task (bound-fn []
+                             (try
+                               (deliver! channel payload batch-id backend)
+                               (finally
+                                 ;; Only remove if this generation still owns the slot
+                                 (swap! active-handlers
+                                        (fn [handlers]
+                                          (if (identical? gen (:gen (get handlers channel)))
+                                            (dissoc handlers channel)
+                                            handlers)))
+                                 (mq.polling/notify-all!))))
+            f    (.submit ^ExecutorService @worker-pool task)]
+        ;; Only set the future if this generation still owns the slot
+        (swap! active-handlers
+               (fn [handlers]
+                 (if (identical? gen (:gen (get handlers channel)))
+                   (assoc-in handlers [channel :future] f)
+                   handlers)))
+        true))))
+
+(defn start-worker-pool!
+  "Starts the shared worker pool for non-blocking message delivery. Idempotent.
+  Returns `true` if THIS call created the pool, `false` if one was already running."
+  []
+  (let [pool (Executors/newCachedThreadPool)]
+    (if (compare-and-set! worker-pool nil pool)
+      true
+      (do (.shutdown ^ExecutorService pool) false))))
+
+(defn shutdown-worker-pool!
+  "Cancels all active handler futures and shuts down the worker pool."
+  []
+  (doseq [[_ {:keys [^Future future]}] @active-handlers]
+    (when future
+      (.cancel future true)))
+  (reset! active-handlers {})
+  (when-let [^ExecutorService pool @worker-pool]
+    (.shutdownNow pool)
+    (reset! worker-pool nil)))

src/metabase/mq/init.clj

@@ -0,0 +1,104 @@
+(ns metabase.mq.init
+  "Initializes the mq subsystem at startup."
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.memory :as q.memory]
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.startup.core :as startup]
+   [metabase.util.log :as log]))
+
+(def ^:private queue-backends
+  {q.appdb/backend-id  q.appdb/backend
+   q.memory/backend-id q.memory/backend})
+
+(def ^:private valid-queue-backends (set (keys queue-backends)))
+
+(defn- resolve-backend [label table kw-or-instance]
+  (if (keyword? kw-or-instance)
+    (or (get table kw-or-instance)
+        (throw (ex-info (str "Unknown " label " backend: " kw-or-instance)
+                        {:backend kw-or-instance :valid (set (keys table))})))
+    kw-or-instance))
+
+(defn start!
+  "Initializes the MQ subsystem with the given backend. Returns a handle that must be
+   passed to [[stop!]] to shut down and restore the previous backend state.
+
+   `queue-be` may be either a keyword identifier (e.g. `:queue.backend/appdb`)
+   or a concrete backend instance.
+
+   Called by [[startup/def-startup-logic!]] with production backends (handle discarded;
+   production shutdown uses [[startup/def-shutdown-logic!]] instead) and by
+   `metabase.mq.test-util/with-test-mq` with isolated memory backends.
+
+   Backends are set globally via `alter-var-root`. For test isolation, bind
+   `listener/*listeners*` and `publish-buffer/*publish-buffer*` with fresh atoms.
+
+   The returned handle records whether THIS call started the global worker pool and
+   publish-buffer flush executor (`:owns-worker-pool?`, `:owns-buffer-flush?`). The
+   matching [[stop!]] only shuts those down if this call started them — so a test
+   fixture running against an already-live server can no-op the global teardown."
+  [queue-be]
+  (let [queue-instance     (resolve-backend "queue" queue-backends queue-be)
+        prev-queue-be      q.backend/*backend*]
+    (alter-var-root #'q.backend/*backend* (constantly queue-instance))
+    (log/infof "Queue backend: %s" queue-be)
+    (q.registry/register-queues!)
+    (listener/register-listeners!)
+    (let [owns-buffer-flush? (publish-buffer/start-publish-buffer-flush!)
+          owns-worker-pool?  (mq.impl/start-worker-pool!)]
+      (q.backend/start! queue-instance)
+      {:prev-queue-be      prev-queue-be
+       :queue-be           queue-instance
+       :owns-buffer-flush? owns-buffer-flush?
+       :owns-worker-pool?  owns-worker-pool?})))
+
+(defn stop!
+  "Shuts down the MQ subsystem.
+
+   Called with no arguments (production): shuts down all services using the current backend vars.
+
+   Called with a handle (the map returned by [[start!]]): additionally restores the previous
+   backend state and only shuts down globals (worker pool, publish-buffer flush) if this call
+   owned them. Used by `metabase.mq.test-util/with-test-mq` for test isolation — a test
+   that borrows an already-running server's globals leaves them alone on teardown."
+  ([]
+   (publish-buffer/stop-publish-buffer-flush!)
+   (when-let [qb q.backend/*backend*] (q.backend/shutdown! qb))
+   (mq.impl/shutdown-worker-pool!)
+   (reset! listener/*listeners* {}))
+  ([{:keys [prev-queue-be queue-be
+            owns-buffer-flush? owns-worker-pool?]}]
+   ;; Always force-drain the (currently bound) publish buffer so test fixtures that bound a
+   ;; fresh `*publish-buffer*` don't lose buffered messages when their teardown runs without
+   ;; owning the global flush executor. The executor itself is only shut down if THIS call
+   ;; started it.
+   (publish-buffer/flush-publish-buffer! true)
+   (when owns-buffer-flush? (publish-buffer/stop-publish-buffer-flush!))
+   (when queue-be (q.backend/shutdown! queue-be))
+   (when owns-worker-pool? (mq.impl/shutdown-worker-pool!))
+   (reset! listener/*listeners* {})
+   (alter-var-root #'q.backend/*backend* (constantly prev-queue-be))))
+
+(defn- resolve-queue-be []
+  (let [queue-be (keyword "queue.backend" (mq.settings/queue-backend))]
+    (when-not (contains? valid-queue-backends queue-be)
+      (throw (ex-info (str "Invalid queue backend: " queue-be
+                           ". Valid backends: " valid-queue-backends)
+                      {:backend queue-be :valid valid-queue-backends})))
+    queue-be))
+
+(defmethod startup/def-startup-logic! ::MqStart [_]
+  (start! (resolve-queue-be)))
+
+;; Run after all other startup logic so listener registration, transports, and polling threads
+;; only come up once the rest of the system is ready.
+(defmethod startup/startup-priority ::MqStart [_] Long/MAX_VALUE)
+
+(defmethod startup/def-shutdown-logic! ::MqShutdown [_]
+  (stop!))

src/metabase/mq/listener.clj

@@ -0,0 +1,111 @@
+(ns metabase.mq.listener
+  "Listener registry: registration, lookup, and the `def-listener!` macro.
+
+  A listener is the consumer-side wiring for a queue — just the handler fn. Everything else
+  about a queue (its broker-side properties, batch size, dedup) is declared on the queue via
+  [[metabase.mq.queue.registry/def-queue!]], which takes effect on every node regardless of
+  whether a listener is registered locally. A listener for a queue requires that queue to be
+  declared first."
+  (:require
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.util.log :as log]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *listeners*
+  "channel → {:listener fn} for all channels."
+  (atom {}))
+
+(defn queue-names
+  "Returns the seq of queue channel names currently registered in `*listeners*`."
+  []
+  (filter #(= "queue" (namespace %)) (keys @*listeners*)))
+
+(defn get-listener
+  "Returns the listener config map for `channel`, or nil if not registered."
+  [channel]
+  (get @*listeners* channel))
+
+(defn- register-listener!
+  "Atomically registers a listener for the given channel.
+
+  Throws if the queue has not been declared via [[q.registry/def-queue!]] — catching missing-queue
+  typos at startup rather than at first publish. Also throws if a listener is already
+  registered for the channel."
+  [channel listener-map]
+  (when (and (= "queue" (namespace channel))
+             (nil? (q.registry/get-queue channel)))
+    (throw (ex-info (str "No queue declared for " channel
+                         " — declare it with `def-queue!` before registering a listener.")
+                    {:channel channel
+                     :known-queues (set (keys @q.registry/*queues*))})))
+  (let [[old _] (swap-vals! *listeners*
+                            (fn [m]
+                              (if (contains? m channel)
+                                m
+                                (assoc m channel listener-map))))]
+    (when (contains? old channel)
+      (throw (ex-info (str "Listener already registered for " (namespace channel) " " (name channel))
+                      {:channel channel})))))
+
+(defn batch-listen!
+  "Low-level listener registration. Prefer [[def-listener!]] for production code — the macro
+   is the supported way to wire a handler so it gets activated through `register-listeners!`
+   at the correct point in startup.
+
+   Use this directly only when you need a runtime-dynamic registration (e.g. plugins, ad-hoc
+   tests). The listener is invoked with a vec of messages, sized up to the queue's
+   `:max-batch-messages`."
+  [channel listener]
+  (register-listener! channel {:listener listener}))
+
+(defn unlisten!
+  "Removes the listener for a channel."
+  [channel]
+  (swap! *listeners* dissoc channel))
+
+(mr/def ::channel
+  [:and :keyword [:fn {:error/message "Channel must be namespaced to 'queue'"}
+                  #(= "queue" (namespace %))]])
+
+(defmulti def-listener*
+  "Multimethod backing [[def-listener!]]."
+  {:arglists '([channel])}
+  identity)
+
+(defmacro def-listener!
+  "Declares a listener for a queue.
+
+   The queue itself must already be declared via `def-queue!` — that's where batch size,
+   exclusivity, and dedup live. The listener body receives a vec of messages; for per-message
+   handling write `(doseq [m messages] ...)` inside the body. Queue channels are namespaced
+   `:queue/*`.
+
+   Examples:
+
+       (mq/def-queue! :queue/simple-task)
+       (mq/def-listener! :queue/simple-task [messages]
+         (doseq [msg messages] (process msg)))
+
+       (mq/def-queue! :queue/search-reindex {:exclusive true :max-batch-messages 50})
+       (mq/def-listener! :queue/search-reindex [messages]
+         (process-batch messages))"
+  {:arglists '([channel bindings & body])}
+  [channel bindings & body]
+  `(defmethod def-listener* ~channel [~'_]
+     (batch-listen! ~channel (fn [~@bindings] ~@body))))
+
+(defn register-listeners!
+  "Call all [[def-listener!]] implementations to register their listeners.
+   Called at startup and in test setup (from `with-test-mq`).
+   Throws on the first registration failure so broken listeners are caught early."
+  []
+  (doseq [[k f] (methods def-listener*)]
+    (try
+      (f k)
+      (catch Throwable e
+        (log/errorf e "Failed to register listener %s" k)
+        (throw (ex-info (str "Failed to register listener " k)
+                        {:channel k}
+                        e))))))

src/metabase/mq/memory.clj

@@ -0,0 +1,67 @@
+(ns metabase.mq.memory
+  "In-memory message store for the memory queue backend. Rather than a FIFO queue of opaque
+  payloads, each `layer` tracks messages individually by a monotonically-increasing id (insertion
+  order) — mirroring the appdb row / redis stream-entry model — so a per-message failure count
+  lives on the message itself and the oldest message is simply the lowest id. Purely in-memory and
+  non-durable.
+
+  Tests construct isolated layers with [[make-layer]]; the poll context / poll thread lives on the
+  backend record, driven by [[metabase.mq.queue.polling]]. This layer only owns the storage.")
+
+(set! *warn-on-reflection* true)
+
+(defn make-layer
+  "Creates a fresh memory layer: a `sorted-map` of message id → message (ordered by id, so the
+  oldest message is first) plus the id counter."
+  []
+  {:messages (atom (sorted-map))
+   :next-id  (atom 0)})
+
+(defonce ^{:doc "Process-wide singleton layer shared by the production memory backend."}
+  default-layer
+  (make-layer))
+
+(defn publish!
+  "Stores a new pending message for `channel-name` with the given opaque payload string."
+  [{:keys [messages next-id]} channel-name payload]
+  (let [id (swap! next-id inc)]
+    (swap! messages assoc id {:id id :queue channel-name :payload payload :failures 0})))
+
+(defn take-oldest
+  "Returns the oldest stored message for `channel-name` as a `{:queue :payload :batch-id}`
+  descriptor, or nil if there are none. The message stays in the store (tracked by id) until it's
+  acked or dropped — the poll loop's channel-busy gate keeps it from being re-fetched while it's
+  in-flight."
+  [{:keys [messages]} channel-name]
+  (when-let [{:keys [id payload]} (->> (vals @messages)
+                                       (filter #(= channel-name (:queue %)))
+                                       first)]
+    {:queue channel-name :payload payload :batch-id id}))
+
+(defn message-failures
+  "Returns how many times message `id` has already failed, or nil if it's no longer stored."
+  [{:keys [messages]} id]
+  (:failures (get @messages id)))
+
+(defn ack!
+  "Removes message `id` from the store (on success or permanent failure)."
+  [{:keys [messages]} id]
+  (swap! messages dissoc id))
+
+(defn retry!
+  "Increments the failure count of message `id`, leaving it stored so it gets re-delivered."
+  [{:keys [messages]} id]
+  (swap! messages (fn [m] (if (contains? m id) (update-in m [id :failures] inc) m))))
+
+(defn depths
+  "Returns a `{:channel :status :count}` row per channel describing how many messages are stored."
+  [{:keys [messages]}]
+  (->> (vals @messages)
+       (group-by :queue)
+       (map (fn [[queue msgs]] {:channel (name queue) :status "pending" :count (count msgs)}))))
+
+(defn clear!
+  "Clears all stored messages. Called by the backend's `shutdown!`."
+  [{:keys [messages next-id]}]
+  (reset! messages (sorted-map))
+  (reset! next-id 0))

src/metabase/mq/payload.clj

@@ -0,0 +1,27 @@
+(ns metabase.mq.payload
+  "Canonical wire encoding for queue messages.
+
+  A batch of messages is [[encode]]d to a JSON string once, at the publish boundary
+  ([[metabase.mq.transport/publish!]]), and [[decode]]d once, at the delivery boundary
+  ([[metabase.mq.impl/deliver!]]). In between, backends move the opaque string around and
+  never look inside it — so storage is purely a backend concern and message shape is
+  identical regardless of which backend delivered it.
+
+  Messages must be JSON-serializable. Decoding keywordizes map keys, but values pass through
+  JSON's type system: keyword/symbol values become strings, sets become vectors, dates become
+  strings, etc."
+  (:require
+   [metabase.util.json :as json]))
+
+(set! *warn-on-reflection* true)
+
+(defn encode
+  "Serializes a vector of messages to a JSON string for storage/transport."
+  ^String [messages]
+  (json/encode messages))
+
+(defn decode
+  "Deserializes a JSON string produced by [[encode]] back into a vector of messages,
+  keywordizing map keys."
+  [payload]
+  (json/decode+kw payload))

src/metabase/mq/polling.clj

@@ -0,0 +1,80 @@
+(ns metabase.mq.polling
+  "Shared polling infrastructure for appdb and memory mq backends:
+  time-gated execution and polling thread lifecycle."
+  (:require
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Future)))
+
+(set! *warn-on-reflection* true)
+
+(defn periodically!
+  "Runs `f` at most once per `interval-ms`. Uses `state-atom` (an atom holding a long timestamp)
+   to track the last run time. Catches and logs exceptions using `label` for context."
+  [state-atom interval-ms label f]
+  (let [now (System/currentTimeMillis)]
+    (when (> (- now @state-atom) interval-ms)
+      (reset! state-atom now)
+      (try
+        (f)
+        (catch Exception e
+          (log/error e (str "Error in " label)))))))
+
+(def ^:private active-poll-states
+  "Set of poll states that currently have a running polling thread."
+  (atom #{}))
+
+(defn make-poll-state
+  "Creates a polling state map with an atom for the background process and a notify object."
+  []
+  {:process  (atom nil)
+   :running? (atom false)
+   :notify   (Object.)})
+
+(defn start-polling!
+  "Starts a polling thread that calls `poll-fn` in a loop, waiting on the notify object
+   for `wait-ms` between iterations. If `poll-fn` returns a truthy value (indicating work
+   was found), the next iteration runs immediately without waiting. Idempotent — second
+   call is a no-op."
+  [{:keys [process running? notify] :as poll-state} label wait-ms poll-fn]
+  (when (compare-and-set! running? false true)
+    (log/infof "Starting %s polling thread" label)
+    (swap! active-poll-states conj poll-state)
+    (reset! process
+            (future
+              (try
+                (loop []
+                  (when @running?
+                    (let [found-work? (try
+                                        (poll-fn)
+                                        (catch InterruptedException e (throw e))
+                                        (catch Exception e
+                                          (log/errorf e "Error in %s polling loop" label)
+                                          false))]
+                      (when-not found-work?
+                        (locking notify (.wait ^Object notify (long wait-ms)))))
+                    (recur)))
+                (catch InterruptedException _
+                  (log/infof "%s polling thread interrupted" label)))))))
+
+(defn stop-polling!
+  "Stops a polling thread."
+  [{:keys [process running?] :as poll-state} label]
+  (swap! active-poll-states disj poll-state)
+  (reset! running? false)
+  (when-let [f @process]
+    (reset! process nil)
+    (when (instance? Future f)
+      (.cancel ^Future f true))
+    (log/infof "%s polling thread stopped" label)))
+
+(defn notify!
+  "Wakes the polling thread so it runs immediately."
+  [{:keys [notify]}]
+  (locking notify (.notifyAll ^Object notify)))
+
+(defn notify-all!
+  "Wakes all active polling threads so they run immediately."
+  []
+  (doseq [ps @active-poll-states]
+    (notify! ps)))

src/metabase/mq/publish.clj

@@ -0,0 +1,89 @@
+(ns metabase.mq.publish
+  "Publishing pipeline: buffering, transaction support, and the `with-buffer` macro."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.app-db.core :as mdb]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(defprotocol MessageBuffer
+  "Protocol for buffering messages before publishing."
+  (put [this msg]
+    "Put a message on the buffer."))
+
+(defn publish!
+  "Publishes messages to a channel. Applies dedup-fn if registered, buffers,
+   publishes to the appropriate backend, and records analytics."
+  [channel messages]
+  (let [dedup-fn     (q.registry/dedup-fn channel)
+        before-count (count messages)
+        messages     (if dedup-fn (dedup-fn messages) messages)]
+    (when (and dedup-fn (< (count messages) before-count))
+      (analytics/inc! :metabase-mq/dedup-messages-dropped
+                      {:channel (name channel)}
+                      (- before-count (count messages))))
+    (when (seq messages)
+      (mq.impl/record-publish-activity! channel)
+      (publish-buffer/buffered-publish! channel messages)
+      (analytics/inc! :metabase-mq/messages-published
+                      {:transport (namespace channel) :channel (name channel)}
+                      (count messages)))))
+
+(defn flush-deferred-messages!
+  "Flushes all deferred messages accumulated during a transaction.
+   Called as an after-commit callback. Per-channel try/catch so one failure doesn't block others."
+  []
+  (when-let [state (mdb/transaction-state)]
+    (let [deferred (::deferred-messages @state)]
+      (doseq [[channel msgs] deferred]
+        (try
+          (publish! channel msgs)
+          (catch Exception e
+            (log/error e "Error flushing deferred messages" {:channel channel})))))))
+
+(defn defer-in-transaction!
+  "Accumulates msgs in *transaction-state* under [::deferred-messages channel].
+   Registers flush-deferred-messages! as an after-commit callback once per transaction."
+  [channel msgs]
+  (let [state     (mdb/transaction-state)
+        old-state (first (swap-vals! state
+                                     (fn [s]
+                                       (-> s
+                                           (update-in [::deferred-messages channel] (fnil into []) msgs)
+                                           (assoc ::flush-registered? true)))))]
+    (when-not (::flush-registered? old-state)
+      (mdb/after-commit! flush-deferred-messages!))))
+
+(defn run-with-buffer
+  "Runs `body-fn` with a MessageBuffer. On success, publishes collected messages —
+   deferred if inside a transaction, immediate via publish! otherwise.
+   On exception, discards buffered messages and rethrows."
+  [channel error-label body-fn]
+  (let [buffer     (atom [])
+        msg-buffer (reify MessageBuffer
+                     (put [_ msg] (swap! buffer conj msg)))]
+    (try
+      (let [result (body-fn msg-buffer)
+            msgs   @buffer]
+        (when (seq msgs)
+          (if (mdb/transaction-state)
+            (defer-in-transaction! channel msgs)
+            (publish! channel msgs)))
+        result)
+      (catch Exception e
+        (log/error e error-label)
+        (throw e)))))
+
+(defmacro with-buffer
+  "Runs the body with a MessageBuffer for the given channel.
+   Messages are buffered and only published if the body completes successfully.
+   Inside a transaction, messages are deferred until after commit."
+  [channel [binding] & body]
+  `(run-with-buffer
+    ~channel
+    (str "Error in " (namespace ~channel) " processing")
+    (fn [~binding] ~@body)))

src/metabase/mq/publish_buffer.clj

@@ -0,0 +1,137 @@
+(ns metabase.mq.publish-buffer
+  "Publish buffering: batches rapid-fire publishes into time-windowed groups
+   and flushes them on a background scheduled thread."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.mq.transport :as transport]
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Executors ScheduledExecutorService ThreadFactory TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *publish-buffer-ms*
+  "Time window in ms to buffer rapid-fire publishes. 0 = immediate publish.
+   Resets on every new message (sliding window)."
+  100)
+
+(def ^:dynamic *publish-buffer-max-ms*
+  "Maximum time in ms since the first message before a flush is forced. 0 = no max."
+  5000)
+
+(def ^:dynamic *publish-buffer-max-retries*
+  "Maximum number of flush attempts before messages are dropped. 0 = unlimited retries."
+  10)
+
+(def ^:dynamic *publish-buffer*
+  "Global buffer for buffering publishes.
+   channel → {:messages [...] :deadline-ms long :created-ms long :retries int}"
+  (atom {}))
+
+(defn- max-batch-size [channel]
+  (q.registry/max-batch-messages channel))
+
+(defn buffered-publish!
+  "Routes messages through *publish-buffer* or publishes immediately based on `*publish-buffer-ms*`.
+
+   The sync path (when `*publish-buffer-ms*` is 0) splits `messages` into chunks of
+   `:max-batch-messages` before handing them to the transport.
+
+   The buffered path appends to the buffer and, if buffering pushes the channel's queued
+   count to or past `:max-batch-messages`, trips the channel's deadline so the next tick
+   of the background flush executor drains it. The actual flush stays on the executor
+   thread — publisher threads never block on backend writes."
+  [channel messages]
+  (let [max-size (max-batch-size channel)]
+    (if (zero? *publish-buffer-ms*)
+      (doseq [chunk (partition-all max-size messages)]
+        (transport/publish! channel (vec chunk)))
+      (let [[_ new] (swap-vals! *publish-buffer*
+                                (fn [buf]
+                                  (let [now (System/currentTimeMillis)]
+                                    (update buf channel
+                                            (fn [entry]
+                                              (-> (or entry {:messages [] :deadline-ms 0 :created-ms now})
+                                                  (update :messages into messages)
+                                                  (assoc :deadline-ms (+ now *publish-buffer-ms*))))))))]
+        (when (>= (count (get-in new [channel :messages])) max-size)
+          ;; Buffer reached the per-channel max-batch limit. Trip its deadline so the next
+          ;; background flush tick drains it. Other channels' deadlines are left untouched.
+          ;; The publisher does not flush synchronously — that would block on a backend write.
+          (swap! *publish-buffer*
+                 (fn [buf] (cond-> buf (contains? buf channel) (assoc-in [channel :deadline-ms] 1)))))))))
+
+(defn flush-publish-buffer!
+  "Drains publish buffer entries past their deadline.
+   When `force?` is true (used by the graceful-shutdown path), every entry is drained
+   regardless of its deadline so no buffered messages are dropped on shutdown."
+  ([] (flush-publish-buffer! false))
+  ([force?]
+   (doseq [[channel entry] @*publish-buffer*]
+     (analytics/set-gauge! :metabase-mq/publish-buffer-depth
+                           {:channel (name channel)}
+                           (count (:messages entry))))
+   (doseq [channel (keys @*publish-buffer*)]
+     (let [drained (atom nil)]
+       (swap! *publish-buffer*
+              (fn [buf]
+                (if-let [{:keys [deadline-ms created-ms] :as entry} (get buf channel)]
+                  (let [now (System/currentTimeMillis)]
+                    (if (or force?
+                            (and (pos? deadline-ms) (>= now deadline-ms))
+                            (and (pos? *publish-buffer-max-ms*) (>= now (+ created-ms *publish-buffer-max-ms*))))
+                      (do (reset! drained entry) (dissoc buf channel))
+                      buf))
+                  buf)))
+       (when-let [{:keys [messages] :as entry} @drained]
+         (try (transport/publish! channel messages)
+              (catch Exception e
+                (let [retries (inc (:retries entry 0))]
+                  (analytics/inc! :metabase-mq/publish-buffer-flush-errors {:channel (name channel)})
+                  (if (and (pos? *publish-buffer-max-retries*) (>= retries *publish-buffer-max-retries*))
+                    (log/warnf "Dropping %d messages for %s after %d flush failures: %s"
+                               (count messages) channel retries (ex-message e))
+                    (do
+                      (log/error e "Error flushing publish buffer, re-buffering"
+                                 {:channel channel :retries retries})
+                     ;; Put messages back into the buffer for retry on next flush cycle
+                      (swap! *publish-buffer*
+                             (fn [buf]
+                               (update buf channel
+                                       (fn [existing]
+                                         (if existing
+                                           (-> existing
+                                              ;; Prepend the failed batch so retried (older) messages
+                                              ;; are published before messages that arrived during the
+                                              ;; flush — preserves publish order across retries.
+                                               (update :messages #(into (vec messages) %))
+                                               (update :retries (fnil max 0) retries))
+                                           (assoc entry
+                                                  :deadline-ms (+ (System/currentTimeMillis) *publish-buffer-ms*)
+                                                  :retries retries))))))))))))))))
+
+(defonce ^:private publish-buffer-executor (atom nil))
+
+(defn start-publish-buffer-flush!
+  "Starts a daemon thread that flushes the publish buffer every 100ms. Idempotent.
+  Returns `true` if THIS call created the executor, `false` if one was already running."
+  []
+  (let [exec (Executors/newSingleThreadScheduledExecutor
+              (reify ThreadFactory
+                (newThread [_ r]
+                  (doto (Thread. r "mq-publish-buffer-flush")
+                    (.setDaemon true)))))]
+    (if (compare-and-set! publish-buffer-executor nil exec)
+      (do (.scheduleAtFixedRate exec ^Runnable flush-publish-buffer! 100 100 TimeUnit/MILLISECONDS)
+          true)
+      (do (.shutdown exec) false))))
+
+(defn stop-publish-buffer-flush!
+  "Stops the publish buffer flush thread and force-drains all remaining entries
+   (graceful shutdown — buffered messages are flushed regardless of deadline)."
+  []
+  (when-let [^ScheduledExecutorService exec
+             (first (reset-vals! publish-buffer-executor nil))]
+    (.shutdownNow exec))
+  (flush-publish-buffer! true))

src/metabase/mq/queue/appdb.clj

@@ -0,0 +1,182 @@
+(ns metabase.mq.queue.appdb
+  "Database-backed implementation of the message queue using the application database."
+  (:require
+   [metabase.app-db.core :as mdb]
+   [metabase.models.interface :as mi]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.polling :as q.polling]
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(def backend-id
+  "This backend's identifier, registered with `metabase.mq.init`; its `name` labels metrics."
+  :queue.backend/appdb)
+
+(defn- for-update-clause
+  "Returns the FOR UPDATE clause for the current app DB.
+  Uses SKIP LOCKED on PostgreSQL and MySQL to avoid blocking on rows locked by other nodes.
+  Falls back to plain FOR UPDATE on H2."
+  []
+  (if (#{:postgres :mysql} (mdb/db-type))
+    [:update :skip-locked]
+    [:update]))
+
+;;; ------------------------------------------- Fetch -------------------------------------------
+
+(defn- fetch-one-pending!
+  "Selects and locks the oldest pending row for a single queue, marks it 'processing' with
+  the given owner, and returns the original row map. Returns nil if no pending row exists
+  or the row at the head of the queue is locked by another transaction (FOR UPDATE SKIP
+  LOCKED). Must be called inside an open transaction so the row lock survives the UPDATE."
+  [conn queue-name owner-id]
+  (when-let [row (first (t2/query conn {:select   [:*]
+                                        :from     [:queue_message_batch]
+                                        :where    [:and
+                                                   [:= :queue_name (name queue-name)]
+                                                   [:= :status "pending"]]
+                                        :order-by [[:id :asc]]
+                                        :limit    1
+                                        :for      (for-update-clause)}))]
+    (t2/query conn {:update [:queue_message_batch]
+                    :set    {:status_heartbeat (mi/now)
+                             :status           "processing"
+                             :owner            owner-id}
+                    :where  [:= :id (:id row)]})
+    row))
+
+;;; ------------------------------------------- Polling -------------------------------------------
+
+(defrecord AppDbQueueBackend [poll-context]
+  q.backend/QueueBackend
+  (backend-id [_this] backend-id)
+
+  (publish! [_this queue payload]
+    (t2/insert! :queue_message_batch
+                {:queue_name (name queue)
+                 :payload    payload}))
+
+  ;; Fetches the oldest pending row for each available queue, marking each 'processing'. Per-queue
+  ;; loop in one transaction for cross-DB portability; each SELECT uses FOR
+  ;; UPDATE SKIP LOCKED so concurrent consumers get different rows.
+  (fetch! [_this available-queues]
+    (when (seq available-queues)
+      (let [exclusive-names (q.registry/exclusive-queue-names)]
+        (t2/with-transaction [conn]
+          (let [blocked-queues  (when (seq exclusive-names)
+                                  (into #{}
+                                        (map :queue_name)
+                                        (t2/query conn {:select   [:queue_name]
+                                                        :from     [:queue_message_batch]
+                                                        :where    [:and
+                                                                   [:in :queue_name (vec exclusive-names)]
+                                                                   [:= :status "processing"]]
+                                                        :group-by [:queue_name]})))
+                fetchable-names (if (seq blocked-queues)
+                                  (remove #(contains? blocked-queues (name %)) available-queues)
+                                  available-queues)]
+            (when (seq fetchable-names)
+              (let [batches (into []
+                                  (keep (fn [qn]
+                                          (when-let [row (fetch-one-pending! conn qn (:id poll-context))]
+                                            {:batch-id (:id row)
+                                             :queue    (keyword "queue" (:queue_name row))
+                                             :payload  (:payload row)})))
+                                  fetchable-names)]
+                (when (seq batches) batches))))))))
+
+  (batch-successful! [_this _queue-name batch-id]
+    (let [deleted (t2/delete! :queue_message_batch :id batch-id :owner (:id poll-context))]
+      (when (= 0 deleted)
+        (log/warnf "Message %d was already deleted from the queue." batch-id))))
+
+  (queue-depths [_this]
+    (into []
+          (map (fn [{:keys [queue_name status cnt]}]
+                 {:channel queue_name :status status :count cnt}))
+          (t2/query {:select   [:queue_name :status [[:count :*] :cnt]]
+                     :from     [:queue_message_batch]
+                     :group-by [:queue_name :status]})))
+
+  (failure-count [_this _queue-name batch-id]
+    (:failures (t2/select-one :queue_message_batch :id batch-id :owner (:id poll-context))))
+
+  (retry-batch! [_this _queue-name batch-id]
+    (t2/update! :queue_message_batch
+                {:id batch-id :owner (:id poll-context)}
+                {:status "pending" :failures [:+ :failures 1] :status_heartbeat (mi/now) :owner nil}))
+
+  ;; Drop the row outright (like the redis/memory backends) — failed batches aren't retained.
+  (fail-batch! [_this _queue-name batch-id]
+    (t2/delete! :queue_message_batch :id batch-id :owner (:id poll-context)))
+
+  (start! [this]
+    (q.polling/start! this poll-context "Queue" 5000))
+
+  (shutdown! [_this]
+    (q.polling/stop! poll-context "Queue"))
+
+  ;; Recovers processing batches whose heartbeat is older than the stale timeout: retryable ones
+  ;; go back to 'pending' in a single bulk UPDATE; exhausted ones are deleted in a single bulk
+  ;; DELETE (failed batches aren't retained). Both ops are guarded on status/heartbeat so we don't
+  ;; clobber rows another node already recovered. Returns per-channel {:channel :recovered :failed}.
+  (recover-stale! [_this stale-timeout-ms max-retries]
+    (let [threshold   (Timestamp/from (.minusMillis (Instant/now) stale-timeout-ms))
+          candidates  (t2/select :queue_message_batch
+                                 :status "processing"
+                                 :status_heartbeat [:< threshold])
+          {retryable false permanent true}
+          (group-by #(>= (inc (:failures %)) max-retries) candidates)
+          n-retryable (if (seq retryable)
+                        (let [result (t2/query {:update :queue_message_batch
+                                                :set    {:status           "pending"
+                                                         :failures         [:+ :failures 1]
+                                                         :status_heartbeat (mi/now)
+                                                         :owner            nil}
+                                                :where  [:and
+                                                         [:in :id (mapv :id retryable)]
+                                                         [:= :status "processing"]
+                                                         [:< :status_heartbeat threshold]]})]
+                          ;; t2/query :update returns the affected-row count; some drivers wrap it in `[n]`.
+                          (if (sequential? result) (first result) result))
+                        0)
+          n-permanent (if (seq permanent)
+                        (t2/delete! :queue_message_batch
+                                    :id [:in (mapv :id permanent)]
+                                    :status "processing"
+                                    :status_heartbeat [:< threshold])
+                        0)]
+      (concat
+       (when (pos? n-retryable)
+         (for [[queue-name rows] (group-by :queue_name retryable)]
+           {:channel queue-name :recovered (count rows) :failed 0}))
+       (when (pos? n-permanent)
+         (for [[queue-name rows] (group-by :queue_name permanent)]
+           {:channel queue-name :recovered 0 :failed (count rows)})))))
+
+  (run-heartbeats! [_this]
+    (doseq [channel (filter #(= "queue" (namespace %)) (mq.impl/busy-channels))]
+      (when-let [{:keys [batch-id]} (mq.impl/active-handler-metadata channel)]
+        (try
+          (t2/update! :queue_message_batch
+                      {:id batch-id :owner (:id poll-context) :status "processing"}
+                      {:status_heartbeat (mi/now)})
+          (catch Exception e
+            (log/warnf e "Failed to update heartbeat for batch %d" batch-id)))))))
+
+(defn make-backend
+  "Constructs a fresh `AppDbQueueBackend` with its own poll context (which carries the instance
+   `:id` used as the row owner, the poll thread, and periodic-task rate-limit atoms). The
+   production [[backend]] singleton is one such instance; tests can build isolated ones so their
+   `start!`/`shutdown!` don't touch production state."
+  []
+  (->AppDbQueueBackend (q.polling/make-poll-context)))
+
+(def backend
+  "Singleton instance of the appdb queue backend."
+  (make-backend))

src/metabase/mq/queue/backend.clj

@@ -0,0 +1,47 @@
+(ns metabase.mq.queue.backend
+  "Backend abstraction for the message queue system.
+  Each concrete backend (appdb, memory, sync) is a record implementing `QueueBackend`.")
+
+(set! *warn-on-reflection* true)
+
+(defprotocol QueueBackend
+  "A queue backend handles durable storage, delivery, and retry semantics for queue messages.
+  Every backend is poll-loop-driven: the shared driver (`metabase.mq.queue.polling`) runs the
+  four maintenance tasks on a fixed cadence and the `fetch!`/submit cycle each iteration, so every
+  backend gets identical loop and respin behavior. Maintenance tasks that don't apply to a backend
+  (e.g. heartbeats for the single-process memory backend) are implemented as no-ops."
+  (backend-id        [this]
+    "Returns this backend's identifying keyword (e.g. `:queue.backend/redis`)")
+  (publish!          [this queue-name payload]
+    "Publishes a payload to the given queue. `payload` is an opaque, already-encoded string.
+    The backend stores/transports it without looking inside.")
+  (fetch!            [this available-queue-names]
+    "Fetches up to one batch per given queue. Returns a seq of
+    `{:queue :payload :batch-id}` maps (or nil/empty when there's nothing to do).")
+  (queue-depths      [this]
+    "Returns a seq of `{:channel :status :count}` maps describing current queue depth.")
+  (batch-successful! [this queue-name batch-id]
+    "Marks a batch as successfully processed.")
+  (failure-count     [this queue-name batch-id]
+    "Returns how many times the batch has already failed (0 on its first failure), or nil if the
+    batch isn't owned/known by this backend — in which case the caller no-ops.")
+  (retry-batch!      [this queue-name batch-id]
+    "Re-enqueues the batch for another delivery attempt, incrementing its stored failure count.")
+  (fail-batch!       [this queue-name batch-id]
+    "Permanently drops / marks-failed the batch after it has exhausted its retries.")
+  (recover-stale!    [this stale-timeout-ms max-retries]
+    "Recovers batches in-flight longer than `stale-timeout-ms` (abandoned by a crashed/stale
+    consumer) — re-queuing for retry, or dropping once `max-retries` is reached. Both thresholds
+    are passed by the shared driver so the policy is uniform. Returns a seq of
+    `{:channel :recovered :failed}` summarizing what happened per channel. Returns nil/empty when
+    nothing was recovered.")
+  (run-heartbeats!   [this]
+    "Renews the in-flight lease on batches this node is currently processing.")
+  (start!            [this]
+    "Starts the backend polling loop. Called once at init time.")
+  (shutdown!         [this]
+    "Shuts down all queue resources for this backend."))
+
+(def ^:dynamic *backend*
+  "The active `QueueBackend` instance. Set by `metabase.mq.init/start!`."
+  nil)

src/metabase/mq/queue/impl.clj

@@ -0,0 +1,26 @@
+(ns metabase.mq.queue.impl
+  "Publish-surface helpers for the queue messaging system.
+
+  Queue declarations (the [[metabase.mq.queue.registry/def-queue!]] macro and `*queues*`
+  registry) live in their own namespace so they can be referenced from `metabase.mq.listener`
+  without forming a cycle through `metabase.mq.publish`."
+  (:require
+   [metabase.mq.publish :as publish]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def :metabase.mq.queue/queue-name
+  [:and :keyword [:fn {:error/message "Queue name must be namespaced to 'queue'"}
+                  #(= "queue" (namespace %))]])
+
+(defmacro with-queue
+  "Runs the body with the ability to add messages to the given queue.
+  Messages are buffered and only published if the body completes successfully.
+  If an exception occurs, no messages are published and the exception is rethrown.
+
+  When called inside a database transaction, messages are accumulated and published
+  as a single batch after the transaction commits successfully. This prevents consumers
+  from reading uncommitted data."
+  [queue-name [queue-binding] & body]
+  `(publish/with-buffer ~queue-name [~queue-binding] ~@body))

src/metabase/mq/queue/memory.clj

@@ -0,0 +1,66 @@
+(ns metabase.mq.queue.memory
+  "In-memory queue backend. Delegates message storage to a shared memory layer (an id-keyed map,
+  like appdb's table) and the poll loop to the shared [[metabase.mq.queue.polling]] driver. Each
+  `MemoryQueueBackend` carries a reference to its layer so tests can construct isolated instances.
+
+  Being single-process and non-durable, the in-memory backend has no crashed-consumer recovery,
+  no cross-node lease to heartbeat, and nothing to clean up, so those maintenance hooks are
+  no-ops; only the depth gauge does real work."
+  (:require
+   [metabase.mq.memory :as memory]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.polling :as q.polling]))
+
+(set! *warn-on-reflection* true)
+
+(def backend-id
+  "This backend's identifier, registered with `metabase.mq.init`; its `name` labels metrics."
+  :queue.backend/memory)
+
+(defrecord MemoryQueueBackend [layer poll-context]
+  q.backend/QueueBackend
+
+  (backend-id [_this] backend-id)
+
+  (publish! [_this queue-name payload]
+    (memory/publish! layer queue-name payload))
+
+  (fetch! [_this available-queues]
+    (into [] (keep #(memory/take-oldest layer %)) available-queues))
+
+  (queue-depths [_this]
+    (memory/depths layer))
+
+  ;; The batch-id is the layer-unique message id, so the queue isn't needed to resolve it.
+  (batch-successful! [_this _queue-name batch-id]
+    (memory/ack! layer batch-id))
+
+  (failure-count [_this _queue-name batch-id]
+    (memory/message-failures layer batch-id))
+
+  (retry-batch! [_this _queue-name batch-id]
+    (memory/retry! layer batch-id))
+
+  (fail-batch! [_this _queue-name batch-id]
+    (memory/ack! layer batch-id))
+
+  (recover-stale! [_this _stale-timeout-ms _max-retries] nil)
+
+  (run-heartbeats! [_this] nil)
+
+  (start! [this]
+    (q.polling/start! this poll-context "Memory" 50))
+
+  (shutdown! [_this]
+    (q.polling/stop! poll-context "Memory")
+    (memory/clear! layer)))
+
+(defn make-backend
+  "Constructs a `MemoryQueueBackend` with its own poll context. With no args, wraps the
+  process-wide `memory/default-layer`. Tests can pass their own layer for isolation."
+  ([] (make-backend memory/default-layer))
+  ([layer] (->MemoryQueueBackend layer (q.polling/make-poll-context))))
+
+(def backend
+  "Singleton `MemoryQueueBackend` backed by `memory/default-layer`."
+  (make-backend))

src/metabase/mq/queue/polling.clj

@@ -0,0 +1,105 @@
+(ns metabase.mq.queue.polling
+  "Shared poll driver for queue backends. Every poll-based backend (appdb, memory, redis) runs
+  the same loop: four maintenance tasks on a fixed cadence, then a `fetch!`/submit pass that
+  respins immediately when work was found. The per-backend differences (how a batch is fetched,
+  what each maintenance task does) live behind the `QueueBackend` protocol, so the loop
+  and its rate-limiting / respin behavior exist in exactly one place."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private stale-check-interval-ms (* 60 1000))
+(def ^:private heartbeat-interval-ms   (* 2 60 1000))
+(def ^:private depth-gauge-interval-ms (* 30 1000))
+
+(def ^:private stale-processing-timeout-ms
+  "How long a batch can sit in-flight (delivered but unacked / 'processing') before it's assumed
+  to belong to a crashed consumer and is recovered."
+  (* 10 60 1000))
+
+(defn- report-depths!
+  "Emits a `:metabase-mq/queue-depth` gauge for each `{:channel :status :count}` the backend
+  reports. Centralized here so backends only produce depth data, not metrics."
+  [backend backend-name]
+  (doseq [{:keys [channel status count]} (q.backend/queue-depths backend)]
+    (analytics/set-gauge! :metabase-mq/queue-depth {:backend backend-name :channel channel :status status} count)))
+
+(defn- run-recover-stale!
+  "Runs the backend's stale-batch recovery and emits the resulting metrics + logs. The backend
+  returns a seq of `{:channel :recovered :failed}` (batches re-queued for retry vs. dropped after
+  exhausting retries); doing the analytics/logging here keeps them consistent across backends."
+  [backend backend-name]
+  (when-let [results (seq (q.backend/recover-stale! backend stale-processing-timeout-ms
+                                                    (mq.settings/queue-max-retries)))]
+    (let [total-recovered (reduce + 0 (keep :recovered results))
+          total-failed    (reduce + 0 (keep :failed results))]
+      (when (pos? total-recovered)
+        (log/warnf "Recovered %d stale batch(es)" total-recovered))
+      (when (pos? total-failed)
+        (log/warnf "Dropped %d stale batch(es) that exhausted retries" total-failed))
+      (doseq [{:keys [channel recovered failed]} results]
+        (when (and recovered (pos? recovered))
+          (analytics/inc! :metabase-mq/batch-stale-recoveries
+                          {:backend backend-name :transport "queue" :channel channel} recovered))
+        (when (and failed (pos? failed))
+          (analytics/inc! :metabase-mq/queue-batch-permanent-failures
+                          {:backend backend-name :channel channel} failed))))))
+
+(defn notify-on-publish!
+  "Wakes the poll loop for `poll-context` so it picks up a freshly published message — unless
+  `channel` is already being processed, in which case the loop re-checks when that finishes.
+  Called by the transport right after `publish!` (the caller reads the backend's `:poll-context`)."
+  [poll-context channel]
+  (when-not (mq.impl/channel-busy? channel)
+    (mq.polling/notify! (:poll-state poll-context))))
+
+(defn make-poll-context
+  "Creates the per-backend poll state: a unique instance `:id`, the polling thread handle, and the
+  rate-limit atoms for each maintenance task. The `:id` identifies this instance when claiming
+  work (the appdb row owner / the redis consumer name). Backends hold one of these and pass it to
+  [[start!]]; `publish!` wakes the loop via `(mq.polling/notify! (:poll-state ctx))`."
+  []
+  {:id                  (str (random-uuid))
+   :poll-state          (mq.polling/make-poll-state)
+   :last-stale-check-ms (atom 0)
+   :last-heartbeat-ms   (atom 0)
+   :last-depth-gauge-ms (atom 0)})
+
+(defn- poll-iteration!
+  "One iteration: run each maintenance task if its interval has elapsed, then fetch and submit
+  one batch per available queue. Returns true if any work was found so the polling
+  thread respins immediately."
+  [backend {:keys [last-stale-check-ms last-heartbeat-ms last-depth-gauge-ms]}]
+  (let [backend-name (name (q.backend/backend-id backend))]
+    (mq.polling/periodically! last-stale-check-ms stale-check-interval-ms "stale recovery"
+                              #(run-recover-stale! backend backend-name))
+    (mq.polling/periodically! last-heartbeat-ms heartbeat-interval-ms "heartbeat"
+                              #(q.backend/run-heartbeats! backend))
+    (mq.polling/periodically! last-depth-gauge-ms depth-gauge-interval-ms "depth gauge"
+                              #(report-depths! backend backend-name))
+    (let [found? (boolean
+                  (when-let [available (seq (remove mq.impl/channel-busy? (listener/queue-names)))]
+                    (when-let [batches (seq (q.backend/fetch! backend available))]
+                      (doseq [{:keys [queue payload batch-id]} batches]
+                        (mq.impl/submit-delivery! queue payload batch-id backend {:batch-id batch-id}))
+                      true)))]
+      (analytics/inc! :metabase-mq/queue-poll-results {:backend backend-name :result (if found? "work" "empty")})
+      found?)))
+
+(defn start!
+  "Starts the polling thread for `backend`, driving [[poll-iteration!]] every `wait-ms` (and
+  immediately whenever it finds work or `publish!` calls `notify!`). Idempotent."
+  [backend poll-context label wait-ms]
+  (mq.polling/start-polling! (:poll-state poll-context) label wait-ms #(poll-iteration! backend poll-context)))
+
+(defn stop!
+  "Stops the polling thread for `poll-context`."
+  [poll-context label]
+  (mq.polling/stop-polling! (:poll-state poll-context) label))

src/metabase/mq/queue/registry.clj

@@ -0,0 +1,124 @@
+(ns metabase.mq.queue.registry
+  "Queue registry. Holds [[*queues*]] (declared queues and their broker-side properties) and
+  the [[def-queue!]] macro for populating it.
+
+  Queues exist independently of any listener so publishers can route to them from anywhere
+  in the cluster. Backends use this registry at startup to pre-arrange broker-side resources
+  (e.g. RabbitMQ queue declarations) for the full set of known queues."
+  (:require
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(def default-max-batch-messages
+  "Default `:max-batch-messages` for a queue that doesn't declare one. Bounds both the
+  publish-time batch size (how many messages are coalesced into one stored batch) and the
+  consumer-side slice size (how many messages the handler receives per invocation)."
+  100)
+
+(mr/def :metabase.mq.queue/queue-config
+  [:map
+   [:exclusive          {:optional true} :boolean]
+   [:max-batch-messages {:optional true} pos-int?]
+   [:dedup-fn           {:optional true} fn?]])
+
+(def ^:dynamic *queues*
+  "queue-name (keyword) → config map for every declared queue.
+
+  Test fixtures rebind this to a fresh atom so test-defined queues don't leak across
+  scenarios. Production code uses the root binding."
+  (atom {}))
+
+(defn get-queue
+  "Returns the config map for `queue-name`, or nil if no such queue has been declared."
+  [queue-name]
+  (get @*queues* queue-name))
+
+(defn queue-names
+  "Returns the seq of currently-declared queue names."
+  []
+  (keys @*queues*))
+
+(defn register-queue!
+  "Atomically registers `config` for `queue-name`. Re-registering with identical config is
+  a no-op (handy for repeated `register-queues!` calls in tests); mismatched config throws.
+
+  `config` keys (all optional):
+    `:exclusive`  — when true, at most one batch is in-flight cluster-wide for this queue."
+  [queue-name config]
+  (let [config (or config {})
+        [old _] (swap-vals! *queues*
+                            (fn [m] (if (contains? m queue-name)
+                                      m
+                                      (assoc m queue-name config))))]
+    (when-let [existing (get old queue-name)]
+      (when (not= existing config)
+        (throw (ex-info (str "Queue " queue-name " is already registered with different config.")
+                        {:queue    queue-name
+                         :existing existing
+                         :new      config}))))))
+
+(defn exclusive?
+  "Returns true if `queue-name` is declared with `:exclusive true`."
+  [queue-name]
+  (boolean (:exclusive (get @*queues* queue-name))))
+
+(defn max-batch-messages
+  "Returns the `:max-batch-messages` declared for `queue-name`, or [[default-max-batch-messages]]."
+  [queue-name]
+  (or (:max-batch-messages (get @*queues* queue-name)) default-max-batch-messages))
+
+(defn dedup-fn
+  "Returns the `:dedup-fn` declared for `queue-name`, or nil if none."
+  [queue-name]
+  (:dedup-fn (get @*queues* queue-name)))
+
+(defn exclusive-queue-names
+  "Returns the set of queue names (as strings) declared with `:exclusive true`."
+  []
+  (into #{}
+        (comp (filter (fn [[_k v]] (:exclusive v)))
+              (map (fn [[k _v]] (name k))))
+        @*queues*))
+
+(defmulti def-queue*
+  "Multimethod backing [[def-queue!]]. Each `(def-queue! :queue/foo …)` adds a method here;
+  [[register-queues!]] iterates them at startup to populate `*queues*`."
+  {:arglists '([queue-name])}
+  identity)
+
+(defmacro def-queue!
+  "Declares a queue with its broker-side configuration. Queues exist independently of any
+  listener — a publisher can route to a queue declared anywhere in the codebase, and a
+  listener for that queue can live on a different node.
+
+  Optional config keys:
+    `:exclusive`          — when true, at most one batch for this queue is in-flight cluster-wide.
+    `:max-batch-messages` — batch size for this queue (defaults to [[default-max-batch-messages]]).
+                            Used at publish time (coalescing) and consume time (slicing).
+    `:dedup-fn`           — `messages -> messages` applied at publish time to drop duplicates
+                            from a batch before it is buffered.
+
+  These are all properties of the queue itself — they take effect at publish time, on every
+  node, regardless of whether a listener is registered locally.
+
+  Examples:
+
+      (mq/def-queue! :queue/simple-task)
+
+      (mq/def-queue! :queue/search-reindex {:exclusive true :max-batch-messages 50})"
+  {:arglists '([queue-name] [queue-name config])}
+  [queue-name & [config]]
+  `(defmethod def-queue* ~queue-name [~'_]
+     (register-queue! ~queue-name ~config)))
+
+(defn register-queues!
+  "Realizes every [[def-queue!]] declaration into [[*queues*]]. Called at startup before
+  the backend is started and before listeners are registered, so backends can pre-arrange
+  broker-side resources for the full set of known queues."
+  []
+  (doseq [[k f] (methods def-queue*)]
+    (try
+      (f k)
+      (catch Throwable e
+        (throw (ex-info (str "Failed to register queue " k) {:queue k} e))))))

src/metabase/mq/settings.clj

@@ -0,0 +1,20 @@
+(ns metabase.mq.settings
+  (:require
+   [metabase.settings.core :refer [defsetting]]
+   [metabase.util.i18n :refer [deferred-tru]]))
+
+(defsetting queue-max-retries
+  (deferred-tru "Maximum number of times a failed queue message will be retried before being dropped.")
+  :type       :integer
+  :default    5
+  :visibility :internal
+  :encryption :no
+  :export?    false)
+
+(defsetting queue-backend
+  (deferred-tru "Which queue backend to use. Valid values: `appdb`, `memory`.")
+  :type       :string
+  :default    "appdb"
+  :visibility :internal
+  :encryption :no
+  :export?    false)

src/metabase/mq/transport.clj

@@ -0,0 +1,27 @@
+(ns metabase.mq.transport
+  "Thin dispatcher that routes channel operations to the queue backend.
+  Selection is based on the channel keyword's namespace (`:queue/*`)."
+  (:require
+   [metabase.mq.payload :as payload]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.polling :as q.polling]))
+
+(set! *warn-on-reflection* true)
+
+(defn transport-type
+  "Returns :queue for a namespaced channel keyword."
+  [channel]
+  (keyword (namespace channel)))
+
+(defn publish!
+  "Publishes messages to the appropriate backend for the channel's transport type, then wakes the
+  backend's poll loop so it picks the work up promptly. Messages are encoded to an opaque string
+  payload here — once, upstream of every backend — so backends only move bytes around and every
+  backend delivers an identical message shape."
+  [channel messages]
+  (let [encoded (payload/encode messages)
+        backend q.backend/*backend*]
+    (case (transport-type channel)
+      :queue (do
+               (q.backend/publish! backend channel encoded)
+               (q.polling/notify-on-publish! (:poll-context backend) channel)))))

src/metabase/startup/core.clj

@@ -1,5 +1,5 @@
 (ns metabase.startup.core
-  "Defines the `def-startup-logic!` multimethod, which is used to run initialization logic when the server starts up."
+  "Defines the `def-startup-logic!` and `def-shutdown-logic!` multimethods for server lifecycle hooks."
   (:require
    [metabase.util :as u]
    [metabase.util.log :as log]))
@@ -15,17 +15,49 @@
 
   For logic that should run on only one node at a time, use `cluster-lock/with-cluster-lock` in your function.
 
+  To control ordering, add a [[startup-priority]] method for the same dispatch value (lower
+  priorities run first; default 0).
+
     (defmethod startup/init-logic! ::ExampleLogic [_]
       (future (println \"Running example logic...\")))"
   {:arglists '([job-name-string])}
   keyword)
 
+(defmulti startup-priority
+  "Ordering priority for [[def-startup-logic!]] implementations. Lower runs first; higher runs
+  later. Default is 0. Use `Long/MAX_VALUE` for logic that must run after everything else."
+  {:arglists '([job-name-string])}
+  keyword)
+
+(defmethod startup-priority :default [_] 500)
+
 (defn run-startup-logic!
-  "Call all implementations of `def-startup-logic!`. Called by metabase.core/init!"
+  "Call all implementations of `def-startup-logic!`. Called by metabase.core/init!
+  Methods run in ascending [[startup-priority]] order, with ties broken by dispatch-value name
+  for determinism."
   []
-  (doseq [[k f] (methods def-startup-logic!)]
+  (doseq [[k f] (sort-by (fn [[k _]] [(startup-priority k) (name k)])
+                         (methods def-startup-logic!))]
     (try
       (log/infof "Running setup logic %s %s" (u/format-color 'green (name k)) (u/emoji "☑\uFE0F"))
       (f k)
       (catch Throwable e
         (log/errorf e "Error initializing startup logic %s" k)))))
+
+(defmulti def-shutdown-logic!
+  "Runs shutdown logic with a given name. All implementations are called during graceful server
+  shutdown. Counterpart to [[def-startup-logic!]].
+
+  The dispatch value can be any unique keyword and is used for logging."
+  {:arglists '([job-name-string])}
+  keyword)
+
+(defn run-shutdown-logic!
+  "Call all implementations of `def-shutdown-logic!`. Called during graceful server shutdown."
+  []
+  (doseq [[k f] (methods def-shutdown-logic!)]
+    (try
+      (log/infof "Running shutdown logic %s" (name k))
+      (f k)
+      (catch Throwable e
+        (log/errorf e "Error running shutdown logic %s" k)))))

test/metabase/app_db/connection_test.clj

@@ -134,6 +134,95 @@
             (when-not (is-transaction-exception? e)
               (throw e))))))))
 
+(deftest after-commit-returns-false-outside-transaction-test
+  (testing "after-commit! returns false when not in a transaction"
+    (is (false? (mdb.connection/after-commit! (fn []))))))
+
+(deftest after-commit-returns-true-inside-transaction-test
+  (testing "after-commit! returns true when inside a transaction"
+    (t2/with-transaction []
+      (is (true? (mdb.connection/after-commit! (fn [])))))))
+
+(deftest after-commit-callbacks-execute-after-commit-test
+  (testing "Callbacks registered via after-commit! execute after successful commit"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit! (fn [] (swap! result conj :callback-1)))
+        (mdb.connection/after-commit! (fn [] (swap! result conj :callback-2)))
+        (is (= [] @result) "Callbacks should not have executed yet"))
+      (is (= [:callback-1 :callback-2] @result) "Callbacks should execute after commit"))))
+
+(deftest after-commit-callbacks-not-executed-on-rollback-test
+  (testing "Callbacks are not executed when a transaction is rolled back"
+    (let [result (atom [])]
+      (try
+        (t2/with-transaction []
+          (mdb.connection/after-commit! (fn [] (swap! result conj :should-not-run)))
+          (throw (ex-info "force rollback" {})))
+        (catch Exception _))
+      (is (= [] @result) "Callbacks should not execute on rollback"))))
+
+(deftest after-commit-callback-registers-new-callback-test
+  (testing "A callback that registers another callback — both execute"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit!
+         (fn []
+           (swap! result conj :first)
+           ;; Register a new callback from within a callback
+           (mdb.connection/after-commit! (fn [] (swap! result conj :second))))))
+      (is (= [:first :second] @result) "Both callbacks should execute")))
+
+  (testing "Deeply chained callbacks all execute"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit!
+         (fn []
+           (swap! result conj :a)
+           (mdb.connection/after-commit!
+            (fn []
+              (swap! result conj :b)
+              (mdb.connection/after-commit!
+               (fn [] (swap! result conj :c))))))))
+      (is (= [:a :b :c] @result)))))
+
+(deftest nested-transactions-share-after-commit-and-state-test
+  (testing "Nested transactions share the same *after-commit* and *transaction-state*"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit! (fn [] (swap! result conj :outer)))
+        (let [outer-state mdb.connection/*transaction-state*]
+          (t2/with-transaction []
+            (is (identical? outer-state mdb.connection/*transaction-state*)
+                "*transaction-state* should be the same atom in nested transactions")
+            (mdb.connection/after-commit! (fn [] (swap! result conj :inner)))))
+        (is (= [] @result) "Callbacks should not have executed yet"))
+      (is (= [:outer :inner] @result) "Both callbacks should execute after outermost commit"))))
+
+(deftest after-commit-discarded-on-nested-rollback-test
+  (testing "Callbacks registered in a nested transaction that rolls back are NOT executed"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit! (fn [] (swap! result conj :outer)))
+        (try
+          (t2/with-transaction []
+            (mdb.connection/after-commit! (fn [] (swap! result conj :inner-should-not-run)))
+            (throw (Exception. "force savepoint rollback")))
+          (catch Exception _)))
+      (is (= [:outer] @result)
+          "Only the outer callback should run; the inner one from the rolled-back savepoint should be discarded")))
+
+  (testing "Transaction state from rolled-back nested transaction is also discarded"
+    (t2/with-transaction []
+      (swap! mdb.connection/*transaction-state* assoc :outer-key "outer-val")
+      (try
+        (t2/with-transaction []
+          (swap! mdb.connection/*transaction-state* assoc :inner-key "inner-val")
+          (throw (Exception. "force savepoint rollback")))
+        (catch Exception _))
+      (is (= {:outer-key "outer-val"} @mdb.connection/*transaction-state*)
+          "State from rolled-back nested transaction should be discarded"))))
+
 (deftest ^:parallel transaction-isolation-level-test
   (testing "We should always use READ_COMMITTED for the app DB (#44505)"
     (with-open [conn (.getConnection mdb.connection/*application-db*)]

test/metabase/metabot/api_test.clj

@@ -975,20 +975,21 @@
                                                                           (reset! captured-args args)
                                                                           ;; Return a minimal streaming response
                                                                           nil)]
-        (api/streaming-request {:metabot_id      test-metabot-id
-                                :profile_id      nil
-                                :message         "test message"
-                                :context         {}
-                                :history         []
-                                :conversation_id (str (random-uuid))
-                                :state           {}
-                                :debug           false}
-                               {:origin nil :referer nil :user-agent nil :ip-address nil})
-        (testing "metabot-id is included in the arguments"
-          (is (some? (:metabot-id @captured-args))
-              "metabot-id should not be nil")
-          (is (= test-metabot-id (:metabot-id @captured-args))
-              "metabot-id should match the input metabot_id"))))))
+        (mt/with-test-user :rasta
+          (api/streaming-request {:metabot_id      test-metabot-id
+                                  :profile_id      nil
+                                  :message         "test message"
+                                  :context         {}
+                                  :history         []
+                                  :conversation_id (str (random-uuid))
+                                  :state           {}
+                                  :debug           false}
+                                 {:origin nil :referer nil :user-agent nil :ip-address nil})
+          (testing "metabot-id is included in the arguments"
+            (is (some? (:metabot-id @captured-args))
+                "metabot-id should not be nil")
+            (is (= test-metabot-id (:metabot-id @captured-args))
+                "metabot-id should match the input metabot_id")))))))
 
 (deftest streaming-request-ip-address-test
   (mt/with-model-cleanup [:model/MetabotMessage

test/metabase/mq/backend_parity_test.clj

@@ -0,0 +1,128 @@
+(ns metabase.mq.backend-parity-test
+  "Contract-level parity between the queue backends.
+
+  Each scenario is written once and run against every backend kind so that
+  semantic drift between the memory and appdb implementations is caught at
+  the source rather than discovered later in production.
+
+  Every scenario is ALSO run with `:duplicate-delivery? true` so that
+  listeners in the parity suite are forced to tolerate the MQ's at-least-
+  once contract. Assertions use multiset / containment checks rather than
+  exact-sequence equality so that a single logical message can legitimately
+  be delivered more than once.
+
+  Scenarios use `mq.tu/eventually!` rather than `mq.tu/flush!` for their
+  assertions, because `wait-for-idle!` cannot see pending rows inside the
+  appdb tables — it only checks the publish buffer, the worker-pool busy
+  set, and memory channels."
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.test-util :as mq.tu]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private backend-kinds
+  "Backend kinds exercised by the parity scenarios. `:memory` covers the
+  fast async in-process backend; `:appdb` covers the durable DB-backed one."
+  [:memory :appdb])
+
+(def ^:private delivery-modes
+  "Delivery modes exercised by the parity scenarios. `:normal` is the
+  production path; `:duplicate` wraps both backends with a decorator that
+  publishes every message twice, forcing listeners to tolerate at-least-
+  once delivery."
+  [:normal :duplicate])
+
+(def ^:private scenario-timeout-ms
+  "Budget for waiting on eventual delivery in parity scenarios. appdb runs
+  through the publish buffer → DB insert → 5s-default poll loop, but the
+  fixture's `wait-for-idle!` calls `notify-all!` each tick so the effective
+  poll latency is much lower."
+  10000)
+
+(defn- cleanup-appdb-channel!
+  "Removes any rows left in the message tables for the given channel so a
+  parity scenario does not pollute later tests."
+  [channel]
+  (case (namespace channel)
+    "queue" (t2/delete! :queue_message_batch :queue_name (name channel))))
+
+(defn- unique-channel [transport-ns suffix]
+  (keyword transport-ns (str suffix "-" (random-uuid))))
+
+(defn- run-parity!
+  "Runs `scenario-fn` once per (kind × delivery-mode) combination. Each
+  invocation gets a freshly-minted unique channel keyword so scenarios do
+  not interfere with each other. On completion the scenario's channel is
+  unlistened (if still present) and any appdb rows for it are cleaned up."
+  ([channel-prefix scenario-fn]
+   (run-parity! backend-kinds delivery-modes channel-prefix scenario-fn))
+  ([kinds modes channel-prefix scenario-fn]
+   (doseq [kind  kinds
+           mode  modes]
+     (testing (str kind " backend / " mode " delivery")
+       (let [channel (unique-channel (namespace channel-prefix) (name channel-prefix))]
+         (try
+           (mq.tu/with-test-mq [ctx {:backend                kind
+                                     :duplicate-delivery? (= mode :duplicate)}]
+             (scenario-fn ctx channel))
+           (finally
+             (when (= :appdb kind)
+               (cleanup-appdb-channel! channel)))))))))
+
+(deftest queue-delivers-published-messages-test
+  (run-parity!
+   :queue/parity-delivery
+   (fn [ctx queue-name]
+     (let [received (atom [])]
+       (mq.tu/listen! queue-name #(swap! received conj %))
+       (mq/with-queue queue-name [q]
+         (mq/put q "a")
+         (mq/put q "b")
+         (mq/put q "c"))
+       (mq.tu/eventually! ctx
+                          #(= #{"a" "b" "c"} (set @received))
+                          scenario-timeout-ms)
+       (is (= #{"a" "b" "c"} (set @received))
+           "Every unique message was delivered at least once")
+       (is (every? pos? (vals (frequencies @received)))
+           "No spurious messages appeared in @received")
+       (mq/unlisten! queue-name)))))
+
+(deftest queue-retries-failed-batch-test
+  (run-parity!
+   :queue/parity-retry
+   (fn [ctx queue-name]
+     (let [calls-by-msg (atom {})]
+       (mq.tu/listen! queue-name
+                      (fn [msg]
+                        (let [n (get (swap! calls-by-msg update msg (fnil inc 0)) msg)]
+                          (when (= 1 n)
+                            (throw (ex-info "first attempt" {:msg msg}))))))
+       (mq/with-queue queue-name [q]
+         (mq/put q "retry-me"))
+       (mq.tu/eventually! ctx
+                          #(>= (get @calls-by-msg "retry-me" 0) 2)
+                          scenario-timeout-ms)
+       (is (>= (get @calls-by-msg "retry-me" 0) 2)
+           "Message delivered at least twice (retry-after-failure or chaos duplication)")
+       (mq/unlisten! queue-name)))))
+
+(deftest queue-normalizes-message-shape-test
+  (testing "Every backend delivers messages in the same canonical JSON-normalized shape:
+            keyword keys preserved, keyword/symbol values and other non-JSON-native
+            values collapsed to their JSON form."
+    (run-parity!
+     :queue/parity-shape
+     (fn [ctx queue-name]
+       (let [received (atom [])]
+         (mq.tu/listen! queue-name #(swap! received conj %))
+         (mq/with-queue queue-name [q]
+           (mq/put q {:id 1 :status :active :nested {:flag true}}))
+         (mq.tu/eventually! ctx #(seq @received) scenario-timeout-ms)
+         (is (= #{{:id 1 :status "active" :nested {:flag true}}}
+                (set @received))
+             "Message delivered in canonical shape regardless of backend")
+         (mq/unlisten! queue-name))))))

test/metabase/mq/listener_test.clj

@@ -0,0 +1,92 @@
+(ns metabase.mq.listener-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.mq.test-util :as mq.tu])
+  (:import
+   (clojure.lang ExceptionInfo)))
+
+(set! *warn-on-reflection* true)
+
+;;; Queues + listeners declared the production way — macros at namespace-load time. This lets
+;;; us exercise the full def-queue! → register-queues! and def-listener! → register-listeners!
+;;; activation path that mq.init/start! (and therefore with-test-mq) drives. Because the queues
+;;; are declared, they coexist harmlessly with every other with-test-mq run in the suite (the
+;;; handlers are registered but never invoked unless something publishes to these queues).
+
+(def ^:private macro-received (atom []))
+
+(mq/def-queue! :queue/listener-macro-test)
+(mq/def-listener! :queue/listener-macro-test [messages]
+  (swap! macro-received into messages))
+
+(def ^:private multi-count (atom 0))
+(def ^:private multi-received (atom []))
+
+(mq/def-queue! :queue/listener-multi-test)
+(mq/def-listener! :queue/listener-multi-test [messages]
+  (swap! multi-count inc)
+  (swap! multi-received into messages))
+
+(deftest def-listener!-activation-test
+  (reset! macro-received [])
+  (mq.tu/with-test-mq [ctx]
+    (testing "register-listeners! wires the macro-declared handler into *listeners*"
+      (is (fn? (:listener (listener/get-listener :queue/listener-macro-test)))))
+    (testing "the handler receives published messages as a vec"
+      (mq/with-queue :queue/listener-macro-test [q]
+        (mq/put q "a")
+        (mq/put q "b"))
+      (mq.tu/eventually! ctx #(= ["a" "b"] @macro-received) 5000)
+      (is (= ["a" "b"] @macro-received)))))
+
+(deftest def-listener!-multi-statement-body-test
+  (reset! multi-count 0)
+  (reset! multi-received [])
+  (mq.tu/with-test-mq [ctx]
+    (testing "every expression in a multi-statement body runs, and `messages` is bound"
+      (mq/with-queue :queue/listener-multi-test [q]
+        (mq/put q "m1"))
+      (mq.tu/eventually! ctx #(seq @multi-received) 5000)
+      (is (= ["m1"] @multi-received))
+      (is (pos? @multi-count) "the side-effecting first statement also ran"))))
+
+(deftest listener-requires-declared-queue-test
+  (binding [listener/*listeners* (atom {})
+            q.registry/*queues*  (atom {})]
+    (testing "registering a listener for an undeclared queue throws"
+      (is (thrown-with-msg? ExceptionInfo #"No queue declared"
+                            (listener/batch-listen! :queue/never-declared (fn [_])))))
+    (testing "succeeds once the queue is declared"
+      (q.registry/register-queue! :queue/now-declared {})
+      (listener/batch-listen! :queue/now-declared (fn [_]))
+      (is (fn? (:listener (listener/get-listener :queue/now-declared)))))))
+
+(deftest duplicate-listener-throws-test
+  (binding [listener/*listeners* (atom {})
+            q.registry/*queues*  (atom {})]
+    (q.registry/register-queue! :queue/dup {})
+    (listener/batch-listen! :queue/dup (fn [_]))
+    (testing "a second listener on the same queue throws"
+      (is (thrown-with-msg? ExceptionInfo #"Listener already registered"
+                            (listener/batch-listen! :queue/dup (fn [_])))))))
+
+(deftest register-listeners!-wraps-failures-test
+  (let [channel :queue/register-failure-test]
+    (try
+      ;; Simulate a def-listener! whose queue was never declared.
+      (defmethod listener/def-listener* channel [_]
+        (listener/batch-listen! channel (fn [_])))
+      (binding [listener/*listeners* (atom {})
+                q.registry/*queues*  (atom {})]
+        ;; Declare every OTHER macro-registered queue so the only failure is ours.
+        (doseq [qn (keys (methods listener/def-listener*))
+                :when (not= qn channel)]
+          (q.registry/register-queue! qn {}))
+        (testing "register-listeners! rethrows a registration failure with context"
+          (is (thrown-with-msg? ExceptionInfo #"Failed to register listener"
+                                (listener/register-listeners!)))))
+      (finally
+        (remove-method listener/def-listener* channel)))))

test/metabase/mq/publish_buffer_test.clj

@@ -0,0 +1,89 @@
+(ns metabase.mq.publish-buffer-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.transport :as transport]))
+
+(set! *warn-on-reflection* true)
+
+(deftest flush-re-buffers-on-failure-test
+  (testing "When transport/publish! throws, messages are re-buffered (not lost)"
+    (let [published (atom [])
+          call-count (atom 0)]
+      (binding [publish-buffer/*publish-buffer* (atom {})
+                publish-buffer/*publish-buffer-ms* 0
+                publish-buffer/*publish-buffer-max-ms* 0]
+        ;; Directly populate the buffer with an entry past its deadline
+        (reset! publish-buffer/*publish-buffer*
+                {:queue/test {:messages ["msg1" "msg2"]
+                              :deadline-ms 1
+                              :created-ms 1}})
+        (with-redefs [transport/publish! (fn [channel messages]
+                                           (swap! call-count inc)
+                                           (if (= 1 @call-count)
+                                             (throw (ex-info "publish failed" {}))
+                                             (swap! published conj {:channel channel :messages messages})))]
+          ;; First flush — publish throws, messages should be re-buffered
+          (publish-buffer/flush-publish-buffer!)
+          (testing "Messages are back in the buffer after failure"
+            (is (contains? @publish-buffer/*publish-buffer* :queue/test))
+            (is (= ["msg1" "msg2"]
+                   (get-in @publish-buffer/*publish-buffer* [:queue/test :messages]))))
+
+          ;; Second flush — publish succeeds
+          (publish-buffer/flush-publish-buffer!)
+          (testing "Messages are delivered on retry"
+            (is (= [{:channel :queue/test :messages ["msg1" "msg2"]}] @published))
+            (is (empty? @publish-buffer/*publish-buffer*))))))))
+
+(deftest flush-drops-messages-after-max-retries-test
+  (testing "Messages are dropped after exceeding max retries"
+    (binding [publish-buffer/*publish-buffer*             (atom {})
+              publish-buffer/*publish-buffer-ms*          0
+              publish-buffer/*publish-buffer-max-ms*      0
+              publish-buffer/*publish-buffer-max-retries* 3]
+      (with-redefs [transport/publish! (fn [_channel _messages]
+                                         (throw (ex-info "always fails" {})))]
+        ;; Populate buffer with messages past their deadline
+        (reset! publish-buffer/*publish-buffer*
+                {:queue/test {:messages   ["msg1" "msg2"]
+                              :deadline-ms 1
+                              :created-ms  1
+                              :retries     0}})
+        ;; Flush 1: retries=1, re-buffered
+        (publish-buffer/flush-publish-buffer!)
+        (is (contains? @publish-buffer/*publish-buffer* :queue/test)
+            "Messages should be re-buffered after first failure")
+
+        ;; Flush 2: retries=2, re-buffered
+        (publish-buffer/flush-publish-buffer!)
+        (is (contains? @publish-buffer/*publish-buffer* :queue/test)
+            "Messages should be re-buffered after second failure")
+
+        ;; Flush 3: retries=3 >= max-retries=3, dropped
+        (publish-buffer/flush-publish-buffer!)
+        (is (empty? @publish-buffer/*publish-buffer*)
+            "Messages should be dropped after reaching max retries")))))
+
+(deftest flush-re-buffer-merges-with-new-messages-test
+  (testing "Re-buffered messages merge with new messages that arrived during flush"
+    (binding [publish-buffer/*publish-buffer* (atom {})
+              publish-buffer/*publish-buffer-ms* 100
+              publish-buffer/*publish-buffer-max-ms* 0]
+      ;; Populate buffer with messages past deadline
+      (reset! publish-buffer/*publish-buffer*
+              {:queue/test {:messages ["old1" "old2"]
+                            :deadline-ms 1
+                            :created-ms 1}})
+      (with-redefs [transport/publish! (fn [_channel _messages]
+                                         ;; Simulate new messages arriving during flush
+                                         (swap! publish-buffer/*publish-buffer*
+                                                assoc :queue/test
+                                                {:messages ["new1"]
+                                                 :deadline-ms (+ (System/currentTimeMillis) 100)
+                                                 :created-ms (System/currentTimeMillis)})
+                                         (throw (ex-info "publish failed" {})))]
+        (publish-buffer/flush-publish-buffer!)
+        (testing "Re-buffered (older) messages are prepended so publish order is preserved across retries"
+          (let [messages (get-in @publish-buffer/*publish-buffer* [:queue/test :messages])]
+            (is (= ["old1" "old2" "new1"] messages))))))))

test/metabase/mq/queue/appdb_test.clj

@@ -0,0 +1,301 @@
+(ns metabase.mq.queue.appdb-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.test :as mt]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(deftest publish-test
+  (let [queue-name (keyword "queue" (str (gensym "publish-test-")))]
+    (try
+      (q.backend/publish! q.appdb/backend queue-name (json/encode ["test message"]))
+      (testing "Messages are persisted in the queue"
+        (let [row (t2/select-one :queue_message_batch :queue_name (name queue-name))]
+          (is (= ["test message"] (json/decode (:payload row))))
+          (is (= (name queue-name) (:queue_name row)))
+          (is (= "pending" (:status row)))
+          (is (not (nil? (:status_heartbeat row))))
+          (is (= (:created_at row) (:status_heartbeat row)))
+          (is (= 0 (:failures row)))
+          (is (nil? (:owner row)))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest fetch-test
+  (let [queue1 (keyword "queue" (str "queue1-" (gensym)))
+        queue2 (keyword "queue" (str "queue2-" (gensym)))
+        invalid-queue (keyword "queue" (str "queue-invalid-" (gensym)))]
+    (binding [listener/*listeners* (atom {queue1 {:listener identity} queue2 {:listener identity}})]
+      (try
+        (t2/with-connection [_conn]
+          (testing "Returns nil if no rows are found"
+            (is (nil? (q.backend/fetch! q.appdb/backend (listener/queue-names)))))
+
+          (t2/insert! :queue_message_batch
+                      {:queue_name (name invalid-queue)
+                       :payload   (json/encode ["invalid"])})
+          (t2/insert! :queue_message_batch
+                      {:queue_name (name queue1)
+                       :payload   (json/encode ["data1"])})
+          (t2/insert! :queue_message_batch
+                      {:queue_name (name queue2)
+                       :payload   (json/encode ["data2"])})
+
+          (testing "Returns nil if no queues are defined"
+            (binding [listener/*listeners* (atom {})]
+              (is (nil? (q.backend/fetch! q.appdb/backend (listener/queue-names))))))
+          (testing "Fetches one row per queue in a single call"
+            (let [results (q.backend/fetch! q.appdb/backend (listener/queue-names))
+                  by-queue (into {} (map (juxt :queue identity)) results)]
+              (is (= 2 (count results)))
+              (testing "queue1 row"
+                (let [{:keys [batch-id payload]} (get by-queue queue1)]
+                  (is (pos-int? batch-id))
+                  (is (= ["data1"] (json/decode payload)))
+                  (testing "Fetched row gets marked as processing"
+                    (let [updated-row (t2/select-one :queue_message_batch :id batch-id)]
+                      (is (= "processing" (:status updated-row)))
+                      (is (not= (:created_at updated-row) (:status_heartbeat updated-row)))
+                      (is (= 0 (:failures updated-row)))
+                      (is (= (:id (:poll-context q.appdb/backend)) (:owner updated-row)))))))
+              (testing "queue2 row"
+                (let [{:keys [batch-id payload]} (get by-queue queue2)]
+                  (is (pos-int? batch-id))
+                  (is (= ["data2"] (json/decode payload)))))))
+          (testing "When everything valid is processing, return nil"
+            (is (nil? (q.backend/fetch! q.appdb/backend (listener/queue-names))))))
+        (finally
+          (t2/delete! :queue_message_batch :queue_name [:in [(name queue1) (name queue2) (name invalid-queue)]]))))))
+
+(deftest exclusive-fetch-test
+  (let [exclusive-q (keyword "queue" (str "exclusive-" (random-uuid)))
+        normal-q    (keyword "queue" (str "normal-" (random-uuid)))]
+    (binding [listener/*listeners* (atom {exclusive-q {:listener identity}
+                                          normal-q    {:listener identity}})
+              q.registry/*queues*  (atom {exclusive-q {:exclusive true}
+                                          normal-q    {}})]
+      (try
+        (t2/with-connection [_conn]
+          (testing "With no processing rows, exclusive queue messages are fetched normally"
+            (t2/insert! :queue_message_batch
+                        {:queue_name (name exclusive-q)
+                         :payload   (json/encode ["exclusive-msg1"])})
+            (let [results (q.backend/fetch! q.appdb/backend (listener/queue-names))
+                  {:keys [queue payload]} (first results)]
+              (is (= 1 (count results)))
+              (is (= exclusive-q queue))
+              (is (= ["exclusive-msg1"] (json/decode payload)))))
+
+          (testing "With a processing row on exclusive queue, fetch skips it"
+            (t2/insert! :queue_message_batch
+                        {:queue_name (name exclusive-q)
+                         :payload   (json/encode ["exclusive-msg2"])})
+            ;; The first message is now 'processing', so the second should be skipped
+            (is (nil? (q.backend/fetch! q.appdb/backend (listener/queue-names)))
+                "Should return nil because exclusive queue has a processing row"))
+
+          (testing "Non-exclusive queue is still fetchable even when exclusive queue is blocked"
+            (t2/insert! :queue_message_batch
+                        {:queue_name (name normal-q)
+                         :payload   (json/encode ["normal-msg1"])})
+            (let [results (q.backend/fetch! q.appdb/backend (listener/queue-names))
+                  {:keys [queue payload]} (first results)]
+              (is (= 1 (count results)))
+              (is (= normal-q queue))
+              (is (= ["normal-msg1"] (json/decode payload)))))
+
+          (testing "After exclusive processing completes, next message can be fetched"
+            ;; Mark the processing row as done by deleting it (simulating batch-successful!)
+            (t2/delete! :queue_message_batch :queue_name (name exclusive-q) :status "processing")
+            (let [results (q.backend/fetch! q.appdb/backend (listener/queue-names))
+                  {:keys [queue payload]} (first results)]
+              (is (= exclusive-q queue))
+              (is (= ["exclusive-msg2"] (json/decode payload))))))
+        (finally
+          (t2/delete! :queue_message_batch :queue_name [:in [(name exclusive-q) (name normal-q)]]))))))
+
+(deftest message-successful-test
+  (let [queue-name (keyword "queue" (str (gensym "successful-test-")))]
+    (testing "Message successful deletes the message"
+      (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                {:queue_name (name queue-name)
+                                                 :payload (json/encode ["test-message"])
+                                                 :status "processing"
+                                                 :owner (:id (:poll-context q.appdb/backend))})]
+        (q.backend/batch-successful! q.appdb/backend queue-name message-id)
+        (is (nil? (t2/select-one :queue_message_batch :id message-id)))))
+
+    (testing "Message successful on non-existent message does not fail"
+      (is (nil? (q.backend/batch-successful! q.appdb/backend queue-name 99999))))))
+
+;; Note: no cleanup needed — batch-successful! deletes the row
+
+(deftest failure-handling-test
+  (let [queue-name (keyword "queue" (str (gensym "failed-test-")))
+        owner      (:id (:poll-context q.appdb/backend))
+        mk-row     (fn [failures]
+                     (t2/insert-returning-pk! :queue_message_batch
+                                              {:queue_name (name queue-name)
+                                               :payload   (json/encode ["m"])
+                                               :status     "processing"
+                                               :failures   failures
+                                               :owner      owner}))]
+    (try
+      (testing "failure-count returns the stored failure count"
+        (is (= 2 (q.backend/failure-count q.appdb/backend queue-name (mk-row 2)))))
+
+      (testing "failure-count is nil for an unknown message (caller no-ops)"
+        (is (nil? (q.backend/failure-count q.appdb/backend queue-name 99999))))
+
+      (testing "failure-count is nil when another node owns the message"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name (name queue-name)
+                                                   :payload   (json/encode ["m"])
+                                                   :status     "processing"
+                                                   :owner      "another-node"
+                                                   :failures   0})]
+          (is (nil? (q.backend/failure-count q.appdb/backend queue-name message-id)))
+          (testing "and retry-batch! leaves the other node's row untouched"
+            (q.backend/retry-batch! q.appdb/backend queue-name message-id)
+            (let [row (t2/select-one :queue_message_batch :id message-id)]
+              (is (= "processing" (:status row)))
+              (is (= 0 (:failures row)))
+              (is (= "another-node" (:owner row)))))))
+
+      (testing "retry-batch! resets to pending and increments failures"
+        (let [message-id (mk-row 2)]
+          (q.backend/retry-batch! q.appdb/backend queue-name message-id)
+          (let [row (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "pending" (:status row)))
+            (is (= 3 (:failures row)))
+            (is (nil? (:owner row)))
+            (is (some? (:status_heartbeat row))))))
+
+      (testing "fail-batch! deletes the message (failed batches aren't retained)"
+        (let [message-id (mk-row 2)]
+          (q.backend/fail-batch! q.appdb/backend queue-name message-id)
+          (is (nil? (t2/select-one :queue_message_batch :id message-id)))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest max-retries-decision-test
+  (testing "the shared delivery layer retries below max-retries and fails permanently at max"
+    (let [queue-name (keyword "queue" (str (gensym "decision-test-")))
+          owner      (:id (:poll-context q.appdb/backend))
+          mk-row     (fn [failures]
+                       (t2/insert-returning-pk! :queue_message_batch
+                                                {:queue_name (name queue-name)
+                                                 :payload   (json/encode ["m"])
+                                                 :status     "processing"
+                                                 :failures   failures
+                                                 :owner      owner}))]
+      (try
+        (testing "below max retries -> re-queued as pending"
+          (let [id (mk-row 0)]
+            (#'mq.impl/handle-batch-failure! q.appdb/backend queue-name id)
+            (is (= "pending" (:status (t2/select-one :queue_message_batch :id id))))))
+        (testing "at max retries -> dropped (deleted)"
+          (let [id (mk-row (dec (mq.settings/queue-max-retries)))]
+            (#'mq.impl/handle-batch-failure! q.appdb/backend queue-name id)
+            (is (nil? (t2/select-one :queue_message_batch :id id)))))
+        (finally
+          (t2/delete! :queue_message_batch :queue_name (name queue-name)))))))
+
+(deftest recover-stale-processing-batches-test
+  (let [queue-name      (keyword "queue" (str "stale-test-" (random-uuid)))
+        stale-ms        (* 10 60 1000)
+        max-retries     (mq.settings/queue-max-retries)
+        stale-heartbeat (Timestamp/from (.minusMillis (Instant/now) (+ stale-ms 60000)))]
+    (try
+      (testing "Stale processing batch is reset to pending with incremented failures"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :payload         (json/encode ["stale-msg"])
+                                                   :status           "processing"
+                                                   :status_heartbeat stale-heartbeat
+                                                   :failures         1
+                                                   :owner            "dead-owner"})]
+          (let [recovered (->> (q.backend/recover-stale! q.appdb/backend stale-ms max-retries)
+                               (filter #(= (name queue-name) (:channel %)))
+                               (keep :recovered)
+                               (reduce + 0))]
+            (is (pos? recovered) "recover-stale! reports the recovered batch for this channel"))
+          (let [row (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "pending" (:status row)))
+            (is (= 2 (:failures row)))
+            (is (nil? (:owner row))))))
+
+      (testing "Stale batch at max retries is dropped (deleted)"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :payload         (json/encode ["stale-msg-max"])
+                                                   :status           "processing"
+                                                   :status_heartbeat stale-heartbeat
+                                                   :failures         (dec (mq.settings/queue-max-retries))
+                                                   :owner            "dead-owner"})]
+          (let [failed (->> (q.backend/recover-stale! q.appdb/backend stale-ms max-retries)
+                            (filter #(= (name queue-name) (:channel %)))
+                            (keep :failed)
+                            (reduce + 0))]
+            (is (pos? failed) "recover-stale! reports the permanently-failed batch for this channel"))
+          (is (nil? (t2/select-one :queue_message_batch :id message-id))
+              "exhausted stale batch is deleted")))
+
+      (testing "Recent processing batch is not recovered by stale check"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :payload         (json/encode ["recent-msg"])
+                                                   :status           "processing"
+                                                   :failures         0
+                                                   :owner            "active-owner"})]
+          (q.backend/recover-stale! q.appdb/backend stale-ms max-retries)
+          (let [row (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "processing" (:status row)))
+            (is (= 0 (:failures row))))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest heartbeat-test
+  (let [queue-name (keyword "queue" (str "heartbeat-test-" (random-uuid)))]
+    (try
+      (let [message-id         (t2/insert-returning-pk! :queue_message_batch
+                                                        {:queue_name (name queue-name)
+                                                         :payload   (json/encode ["msg"])
+                                                         :status     "processing"
+                                                         :owner      (:id (:poll-context q.appdb/backend))})
+            original-heartbeat (:status_heartbeat (t2/select-one :queue_message_batch :id message-id))]
+        (Thread/sleep 100) ; Ensure enough time passes for timestamp to differ
+        (mt/with-dynamic-fn-redefs [mq.impl/busy-channels          (constantly #{queue-name})
+                                    mq.impl/active-handler-metadata (fn [_ch] {:batch-id message-id})]
+          (q.backend/run-heartbeats! q.appdb/backend))
+        (testing "Heartbeat is updated after run-heartbeats!"
+          (let [updated-heartbeat (:status_heartbeat (t2/select-one :queue_message_batch :id message-id))]
+            (is (not= original-heartbeat updated-heartbeat)
+                "Heartbeat timestamp should be updated"))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest queue-depths-test
+  (let [queue-name (keyword "queue" (str "depth-gauge-test-" (random-uuid)))]
+    (try
+      (t2/insert! :queue_message_batch {:queue_name (name queue-name) :payload (json/encode ["m1"]) :status "pending"})
+      (t2/insert! :queue_message_batch {:queue_name (name queue-name) :payload (json/encode ["m2"]) :status "processing"})
+      (testing "queue-depths returns a {:channel :status :count} row per queue/status combination"
+        (let [by-status (into {}
+                              (comp (filter #(= (name queue-name) (:channel %)))
+                                    (map (juxt :status :count)))
+                              (q.backend/queue-depths q.appdb/backend))]
+          (is (= 1 (get by-status "pending")) "Should count 1 pending message")
+          (is (= 1 (get by-status "processing")) "Should count 1 processing message")))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))

test/metabase/mq/queue/core_test.clj

@@ -0,0 +1,285 @@
+(ns metabase.mq.queue.core-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.app-db.connection :as app-db.conn]
+   [metabase.mq.core :as mq]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.payload :as payload]
+   [metabase.mq.publish :as mq.publish]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.mq.test-util :as mq.tu])
+  (:import (clojure.lang ExceptionInfo)
+           (java.util.concurrent CountDownLatch CyclicBarrier)))
+
+(set! *warn-on-reflection* true)
+
+(deftest e2e-test
+  (let [heard-messages (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [message] (swap! heard-messages conj message))}
+      (testing "The messages are heard and processed"
+        (mq/with-queue :queue/test [q]
+          (mq/put q "test message 1")
+          (mq/put q "test message 2"))
+        (mq.tu/flush! test-mq)
+        (is (= ["test message 1" "test message 2"] @heard-messages))))))
+
+(deftest publish-to-unlistened-queue-test
+  (mq.tu/with-test-mq [_test-mq]
+    (testing "with-queue on a queue with no listener still succeeds"
+      (mq/with-queue :queue/nonexistent [q]
+        (mq/put q "msg")))))
+
+(deftest with-queue-success-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (testing "with-queue publishes buffered messages on success"
+        (let [result (mq/with-queue :queue/test [q]
+                       (mq/put q "a")
+                       (mq/put q "b")
+                       :done)]
+          (is (= :done result))
+          (mq.tu/flush! test-mq)
+          (is (= ["a" "b"] @heard)))))))
+
+(deftest with-queue-exception-discards-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (testing "with-queue discards buffered messages on exception"
+        (is (thrown? Exception
+                     (mq/with-queue :queue/test [q]
+                       (mq/put q "should-be-discarded")
+                       (throw (ex-info "boom" {})))))
+        (mq.tu/flush! test-mq)
+        (is (empty? @heard))))))
+
+(deftest with-queue-no-listener-test
+  (mq.tu/with-test-mq [_test-mq]
+    (testing "with-queue on queue with no listener still succeeds"
+      (mq/with-queue :queue/nonexistent [q]
+        (mq/put q "msg")))))
+
+(deftest double-listen-throws-test
+  (mq.tu/with-test-mq [_test-mq]
+    {:queue/test (fn [_] nil)}
+    (testing "Registering a second listener on the same queue throws"
+      (is (thrown-with-msg? ExceptionInfo #"Listener already registered"
+                            (mq.tu/listen! :queue/test (fn [_] nil)))))))
+
+(deftest concurrent-listen-throws-test
+  (mq.tu/with-test-mq [_test-mq]
+    (let [queue-name :queue/test
+          n          10
+          barrier    (CyclicBarrier. n)
+          results    (atom [])]
+      (testing "Concurrent listen! calls: exactly one succeeds, rest throw"
+        (let [threads (mapv (fn [_]
+                              (let [f (bound-fn []
+                                        (.await barrier)
+                                        (try
+                                          (mq.tu/listen! queue-name (fn [_] nil))
+                                          (swap! results conj :ok)
+                                          (catch ExceptionInfo _
+                                            (swap! results conj :error))))]
+                                (Thread. ^Runnable f)))
+                            (range n))]
+          (run! (fn [^Thread t] (.start t)) threads)
+          (run! (fn [^Thread t] (.join t 5000)) threads)
+          (is (= 1 (count (filter #{:ok} @results))))
+          (is (= (dec n) (count (filter #{:error} @results))))))
+      (mq/unlisten! queue-name))))
+
+(deftest exclusive-listen-test
+  (let [heard-messages (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      (q.registry/register-queue! :queue/exclusive-test {:exclusive true :max-batch-messages 1})
+      (mq.tu/listen! :queue/exclusive-test #(swap! heard-messages conj %))
+      (testing "Exclusive queue processes messages via the async memory backend"
+        (mq/with-queue :queue/exclusive-test [q]
+          (mq/put q "msg1")
+          (mq/put q "msg2"))
+        (mq.tu/flush! test-mq)
+        (is (= ["msg1" "msg2"] @heard-messages))))))
+
+(deftest batch-listen-exclusive-test
+  (let [heard-batches (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      (q.registry/register-queue! :queue/batch-exclusive {:exclusive true :max-batch-messages 10})
+      (listener/batch-listen! :queue/batch-exclusive
+                              (fn [batch] (swap! heard-batches conj batch)))
+      (testing "Exclusive batch-listen! registers and processes"
+        (mq/with-queue :queue/batch-exclusive [q]
+          (mq/put q "a")
+          (mq/put q "b"))
+        (mq.tu/flush! test-mq)
+        (is (= [["a" "b"]] @heard-batches))))))
+
+(deftest transaction-defers-publish-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (binding [app-db.conn/*after-commit*  (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (testing "Inside a transaction, messages are accumulated, not published immediately"
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg1")
+            (mq/put q "msg2"))
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg3"))
+          (testing "Messages are deferred in transaction state"
+            (is (= ["msg1" "msg2" "msg3"]
+                   (get-in @app-db.conn/*transaction-state*
+                           [::mq.publish/deferred-messages :queue/test]))))
+          (testing "Listener has not been called yet"
+            (is (= [] @heard))))
+        (testing "After flush (simulating commit), all messages are delivered"
+          (mq.publish/flush-deferred-messages!)
+          (mq.tu/flush! test-mq)
+          (is (= ["msg1" "msg2" "msg3"] @heard)))))))
+
+(deftest transaction-rollback-discards-messages-test
+  (mq.tu/with-test-mq [test-mq]
+    {:queue/test (fn [_] nil)}
+    (binding [app-db.conn/*after-commit*  (atom [])
+              app-db.conn/*transaction-state* (atom {})]
+      (testing "Messages accumulated during a failed transaction are discarded"
+        (is (thrown? Exception
+                     (mq/with-queue :queue/test [q]
+                       (mq/put q "should-be-discarded")
+                       (throw (ex-info "boom" {})))))
+        (is (nil? (get-in @app-db.conn/*transaction-state*
+                          [::mq.publish/deferred-messages :queue/test]))
+            "No messages should be in transaction state after exception")
+        (mq.tu/flush! test-mq)))))
+
+(deftest outside-transaction-publishes-immediately-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (testing "Outside a transaction, messages are published"
+        (is (nil? app-db.conn/*transaction-state*))
+        (mq/with-queue :queue/test [q]
+          (mq/put q "msg1"))
+        (mq.tu/flush! test-mq)
+        (is (= ["msg1"] @heard))))))
+
+(deftest multiple-queues-in-transaction-test
+  (let [heard-a (atom [])
+        heard-b (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test-a (fn [msg] (swap! heard-a conj msg))
+       :queue/test-b (fn [msg] (swap! heard-b conj msg))}
+      (binding [app-db.conn/*after-commit*  (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (mq/with-queue :queue/test-a [q]
+          (mq/put q "a1")
+          (mq/put q "a2"))
+        (mq/with-queue :queue/test-b [q]
+          (mq/put q "b1"))
+        (testing "Both queues have deferred messages"
+          (is (= ["a1" "a2"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :queue/test-a])))
+          (is (= ["b1"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :queue/test-b]))))
+        (testing "After flush, both queues receive their messages"
+          (mq.publish/flush-deferred-messages!)
+          (mq.tu/flush! test-mq)
+          (is (= ["a1" "a2"] @heard-a))
+          (is (= ["b1"] @heard-b)))))))
+
+(deftest buffering-combines-messages-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      (q.registry/register-queue! :queue/test {:max-batch-messages 50})
+      (listener/batch-listen! :queue/test
+                              (fn [batch] (swap! heard into batch)))
+      (binding [publish-buffer/*publish-buffer-ms* 100
+                publish-buffer/*publish-buffer*    (atom {})]
+        (testing "buffered-publish! buffers when *publish-buffer-ms* > 0"
+          (mq.publish/publish! :queue/test ["msg1"])
+          (mq.publish/publish! :queue/test ["msg2"])
+          (is (empty? @heard)
+              "Nothing delivered yet because buffer window hasn't elapsed")
+          (is (= ["msg1" "msg2"]
+                 (:messages (get @publish-buffer/*publish-buffer* :queue/test)))
+              "Messages are buffered"))
+        (testing "After flush, all messages are delivered"
+          (swap! publish-buffer/*publish-buffer*
+                 update :queue/test assoc :deadline-ms 1)
+          (publish-buffer/flush-publish-buffer!)
+          (mq.tu/flush! test-mq)
+          (is (= ["msg1" "msg2"] @heard)
+              "Both messages delivered after flush"))))))
+
+(deftest buffering-immediate-when-zero-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (binding [publish-buffer/*publish-buffer-ms* 0
+                publish-buffer/*publish-buffer*    (atom {})]
+        (testing "When *publish-buffer-ms* is 0, messages publish without buffering"
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg1"))
+          (mq.tu/flush! test-mq)
+          (is (= ["msg1"] @heard))
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg2"))
+          (mq.tu/flush! test-mq)
+          (is (= ["msg1" "msg2"] @heard)))))))
+
+(deftest buffering-flush-delivers-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      (q.registry/register-queue! :queue/test {:max-batch-messages 3})
+      (listener/batch-listen! :queue/test
+                              (fn [batch] (swap! heard into batch)))
+      (binding [publish-buffer/*publish-buffer-ms*  1
+                publish-buffer/*publish-buffer-max-ms* 0
+                publish-buffer/*publish-buffer*     (atom {})]
+        (testing "Messages are buffered until flushed"
+          (mq.publish/publish! :queue/test ["a" "b"])
+          (is (empty? @heard) "not yet flushed")
+          (mq.publish/publish! :queue/test ["c"])
+          (is (empty? @heard) "still buffered, no proactive flush")
+          (Thread/sleep 5)
+          (publish-buffer/flush-publish-buffer!)
+          (mq.tu/flush! test-mq)
+          (is (= ["a" "b" "c"] @heard) "all delivered after flush"))))))
+
+(deftest fifo-ordering-test
+  (let [received (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [message] (swap! received conj message))}
+      (doseq [i (range 10)]
+        (mq/with-queue :queue/test [q]
+          (mq/put q i)))
+      (mq.tu/flush! test-mq)
+      (testing "All messages are delivered in order"
+        (is (= (range 10) @received))))))
+
+(deftest exclusive-queue-single-active-handler-test
+  "Verifies that submit-delivery! returns false for a channel that already has an
+   active handler, enforcing at-most-one concurrent delivery per channel."
+  (mq.impl/start-worker-pool!)
+  (mq.tu/with-test-mq [_test-mq]
+    (let [queue-name :queue/exclusive-concurrency-test
+          latch      (CountDownLatch. 1)]
+      (mq.tu/listen! queue-name
+                     (fn [_] (.await latch))) ; Block listener until released
+      (testing "First submission succeeds and marks the channel as busy"
+        (is (true? (mq.impl/submit-delivery! queue-name (payload/encode ["msg1"]) nil nil nil)))
+        (Thread/sleep 50) ; Give worker thread time to start and block on latch
+        (is (true? (mq.impl/channel-busy? queue-name))))
+      (testing "Second submission returns false while channel is busy"
+        (is (false? (mq.impl/submit-delivery! queue-name (payload/encode ["msg2"]) nil nil nil))))
+      (.countDown latch) ; Release the listener
+      (Thread/sleep 100) ; Wait for delivery to complete and active-handlers to clear
+      (testing "Channel is no longer busy after delivery completes"
+        (is (false? (mq.impl/channel-busy? queue-name))))
+      (mq/unlisten! queue-name))))

test/metabase/mq/queue/memory_test.clj

@@ -0,0 +1,36 @@
+(ns metabase.mq.queue.memory-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.queue.registry :as q.registry]
+   [metabase.mq.test-util :as mq.tu]))
+
+(set! *warn-on-reflection* true)
+
+(deftest publish-test
+  (mq.tu/with-test-mq [ctx]
+    (let [queue-name (keyword "queue" (str "publish-test-" (gensym)))
+          received   (atom [])]
+      (mq.tu/listen! queue-name (fn [msg] (swap! received conj msg)))
+      (testing "Publish lazily creates channel and delivers messages"
+        (mq/with-queue queue-name [q]
+          (mq/put q "test-message")
+          (mq/put q "test-message2"))
+        (mq.tu/flush! ctx)
+        (is (= ["test-message" "test-message2"] @received))))))
+
+(deftest exclusive-memory-test
+  (mq.tu/with-test-mq [ctx]
+    (let [queue-name :queue/exclusive-test
+          received   (atom [])]
+      (q.registry/register-queue! queue-name {:exclusive true})
+      (mq.tu/listen! queue-name (fn [msg] (swap! received conj msg)))
+      (testing "Exclusive queue processes all messages"
+        (mq/with-queue queue-name [q]
+          (mq/put q "msg1")
+          (mq/put q "msg2")
+          (mq/put q "msg3")
+          (mq/put q "msg4")
+          (mq/put q "msg5"))
+        (mq.tu/flush! ctx)
+        (is (= ["msg1" "msg2" "msg3" "msg4" "msg5"] @received))))))

test/metabase/mq/shutdown_drain_test.clj

@@ -0,0 +1,39 @@
+(ns metabase.mq.shutdown-drain-test
+  "Tests that `mq.init/stop!` performs a graceful drain of the publish buffer
+  before shutting down backends, so buffered messages still land in the appdb
+  tables rather than being lost."
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.init :as mq.init]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest queue-shutdown-drains-publish-buffer-test
+  (testing "stop! drains the publish buffer for queues so buffered messages land in the appdb table"
+    (let [queue-name (keyword "queue" (str "shutdown-drain-" (gensym)))]
+      (try
+        (binding [listener/*listeners*                   (atom {})
+                  publish-buffer/*publish-buffer*        (atom {})
+                  publish-buffer/*publish-buffer-ms*     60000
+                  publish-buffer/*publish-buffer-max-ms* 0]
+          (let [handle (mq.init/start! :queue.backend/appdb)]
+            (mq/with-queue queue-name [q]
+              (mq/put q "drain-me"))
+            (is (= ["drain-me"]
+                   (get-in @publish-buffer/*publish-buffer* [queue-name :messages]))
+                "Message should be sitting in the publish buffer before stop!")
+            (is (empty? (t2/select :queue_message_batch :queue_name (name queue-name)))
+                "Buffer drain hasn't happened yet — table should be empty")
+            (mq.init/stop! handle))
+          (let [rows (t2/select :queue_message_batch :queue_name (name queue-name))]
+            (is (= 1 (count rows))
+                "Exactly one row should exist after the graceful shutdown drain")
+            (is (= ["drain-me"] (json/decode (:payload (first rows))))
+                "The buffered message should have been persisted on shutdown")))
+        (finally
+          (t2/delete! :queue_message_batch :queue_name (name queue-name)))))))

test/metabase/mq/test_util.clj

@@ -0,0 +1,297 @@
+(ns metabase.mq.test-util
+  "Test fixture for exercising the MQ subsystem in tests. `with-test-mq` starts a
+  fresh async memory backend and exposes a fixture context that carries the layer,
+  the backend instances, and convenience fns (`flush!`, `wait-for-idle!`,
+  `eventually!`) for driving delivery and waiting for quiescence.
+
+  `do-with-test-mq!` also accepts `:backend :appdb` for running the real database-
+  backed backend in parity tests, and `:duplicate-delivery? true` to wrap the
+  backends with a decorator that publishes every message twice. The duplicate-
+  delivery mode exists to force listeners under test to prove they handle the
+  MQ's at-least-once contract — any scenario that silently relies on exactly-
+  once delivery will fail loudly under this mode."
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.init :as mq.init]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.memory :as memory]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.memory :as q.memory]
+   [metabase.mq.queue.registry :as q.registry]))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------- Quiescence -------------------------------------------
+
+(defn- layer-drained?
+  "Returns true if no stored memory message belongs to a channel that currently has a listener.
+  Messages on channels with no listener are orphaned by design (published after `unlisten!`) and
+  would otherwise block teardown. Returns true for non-memory fixtures (appdb/redis) — no layer."
+  [{:keys [layer]}]
+  (if (nil? layer)
+    true
+    (let [listeners @listener/*listeners*]
+      (not-any? #(contains? listeners (:queue %)) (vals @(:messages layer))))))
+
+(defn- unmet-idle-conditions
+  "Returns a set of the idle conditions that currently do NOT hold. Empty set == idle."
+  [ctx]
+  (cond-> #{}
+    (seq @publish-buffer/*publish-buffer*) (conj :publish-buffer-nonempty)
+    (seq (mq.impl/busy-channels))          (conj :busy-channels)
+    (not (layer-drained? ctx))             (conj :memory-messages-pending)))
+
+(defn wait-for-idle!
+  "Blocks until the MQ fixture reaches quiescence or `timeout-ms` elapses. Throws
+  an ex-info whose `:unmet` key names the conditions still holding on timeout.
+
+  Because `poll-once!` drains memory channels *before* setting `active-handlers`
+  on the next submit-delivery, a single idle observation can occur in the gap
+  between the two. We require two consecutive idle observations 20 ms apart."
+  ([ctx] (wait-for-idle! ctx 5000))
+  ([ctx timeout-ms]
+   (let [deadline (+ (System/currentTimeMillis) timeout-ms)]
+     (loop [consecutive-idle 0]
+       ;; Wake every active polling thread on each tick so appdb backends drain
+       ;; immediately instead of waiting on their 5s default poll interval.
+       (mq.polling/notify-all!)
+       (let [unmet (unmet-idle-conditions ctx)]
+         (cond
+           (and (empty? unmet) (>= consecutive-idle 1)) :idle
+           (empty? unmet) (do (Thread/sleep 20) (recur (inc consecutive-idle)))
+           (< (System/currentTimeMillis) deadline) (do (Thread/sleep 10) (recur 0))
+           :else (throw (ex-info "MQ did not reach idle state before timeout"
+                                 {:timeout-ms timeout-ms :unmet unmet}))))))))
+
+(defn- force-expire-publish-buffer!
+  "Marks every entry in the publish buffer as past its deadline so the next
+  `flush-publish-buffer!` call drains it immediately."
+  []
+  (swap! publish-buffer/*publish-buffer*
+         (fn [buf]
+           (into {} (map (fn [[k v]] [k (assoc v :deadline-ms 1)])) buf))))
+
+(defn flush!
+  "Force-drains the publish buffer and waits until the MQ is idle. This is the
+  standard 'move time forward' primitive for tests: publish things, call `flush!`,
+  then assert on the resulting state."
+  ([ctx] (flush! ctx 5000))
+  ([ctx timeout-ms]
+   (force-expire-publish-buffer!)
+   (publish-buffer/flush-publish-buffer!)
+   (wait-for-idle! ctx timeout-ms)))
+
+(defn eventually!
+  "Polls `pred` until truthy or `timeout-ms` elapses. Returns the final value of
+  `pred`. Use when the thing you're waiting on can't be expressed as 'MQ is idle'
+  — for example, when waiting for a specific count to be reached or for
+  appdb-backed delivery to round-trip through the database."
+  ([_ctx pred] (eventually! _ctx pred 2000))
+  ([_ctx pred timeout-ms]
+   (let [deadline (+ (System/currentTimeMillis) timeout-ms)]
+     (loop []
+       ;; Wake any polling threads so appdb backends drain promptly instead of
+       ;; blocking on their production poll interval.
+       (mq.polling/notify-all!)
+       (let [v (pred)]
+         (if v
+           v
+           (if (< (System/currentTimeMillis) deadline)
+             (do (Thread/sleep 10) (recur))
+             v)))))))
+
+;;; ------------------------------------------- Fixture -------------------------------------------
+
+(defn listen!
+  "Test helper — registers a per-message listener by wrapping `listener-fn` so it's called
+  once per message in each batch, with error isolation. Production code should use
+  [[metabase.mq.core/def-listener!]] instead; this is for ad-hoc test registration where
+  declaring at namespace-load time isn't convenient.
+
+  As a convenience, auto-declares the queue (with default config) if it hasn't been declared
+  yet. Tests that need queue-level config (e.g. `:exclusive`) should call
+  [[q.registry/register-queue!]] explicitly before calling this."
+  [channel listener-fn]
+  (when (and (= "queue" (namespace channel))
+             (nil? (q.registry/get-queue channel)))
+    (q.registry/register-queue! channel {}))
+  (listener/batch-listen! channel
+                          (fn [messages]
+                            (let [error (volatile! nil)]
+                              (doseq [m messages]
+                                (try (listener-fn m)
+                                     (catch Throwable e (vreset! error e))))
+                              (when-let [e @error] (throw e))))))
+
+(defn- merge-listeners!
+  "Merges a user-supplied listener map on top of whatever listeners
+  `register-listeners!` already registered. Values can be plain single-message
+  fns (wrapped to match the vec-of-messages contract that `handle!` expects)
+  or full config maps with `:listener` etc. Map-form values are passed through
+  as-is — their `:listener` must already accept a vec of messages.
+
+  Auto-declares the queue for any `:queue/*` channel that isn't already declared, matching
+  the convenience semantics of [[listen!]]."
+  [listeners]
+  (when (seq listeners)
+    (doseq [k (keys listeners)
+            :when (and (= "queue" (namespace k))
+                       (nil? (q.registry/get-queue k)))]
+      (q.registry/register-queue! k {}))
+    (swap! listener/*listeners*
+           (fn [current]
+             (reduce-kv (fn [m k v]
+                          (assoc m k (if (fn? v)
+                                       (if-let [existing (get current k)]
+                                         (assoc existing :listener (partial run! v))
+                                         {:listener (partial run! v)})
+                                       v)))
+                        current
+                        listeners)))))
+
+(defn- make-fixture-backends
+  "Constructs backend instances for the given fixture kind. Memory gets a fresh
+  isolated layer; appdb shares the DB table but gets its own per-instance state
+  (poll-state, owner-id, offsets, periodic-task rate-limit atoms) so that
+  test teardown doesn't tear down the production singleton's polling thread."
+  [kind]
+  (case kind
+    :memory (let [layer (memory/make-layer)]
+              {:backend  :memory
+               :layer    layer
+               :queue-be (q.memory/make-backend layer)})
+    :appdb  {:backend  :appdb
+             :queue-be (q.appdb/make-backend)}))
+
+;; A defrecord (not reify) so it exposes a `:poll-context` field — the transport reads
+;; `(:poll-context backend)` to wake the poll loop, and that must resolve to the inner backend's
+;; poll context (the one the inner poll loop actually waits on).
+(defrecord DoubleDeliveryBackend [inner poll-context]
+  q.backend/QueueBackend
+  (publish! [_ queue-name messages]
+    (q.backend/publish! inner queue-name messages)
+    (q.backend/publish! inner queue-name messages))
+  (batch-successful! [_ queue-name batch-id]
+    (q.backend/batch-successful! inner queue-name batch-id))
+  (failure-count [_ queue-name batch-id]
+    (q.backend/failure-count inner queue-name batch-id))
+  (retry-batch! [_ queue-name batch-id]
+    (q.backend/retry-batch! inner queue-name batch-id))
+  (fail-batch! [_ queue-name batch-id]
+    (q.backend/fail-batch! inner queue-name batch-id))
+  (start! [_] (q.backend/start! inner))
+  (shutdown! [_] (q.backend/shutdown! inner))
+  (backend-id [_] (q.backend/backend-id inner))
+  (fetch! [_ available-queues] (q.backend/fetch! inner available-queues))
+  (recover-stale! [_ stale-timeout-ms max-retries]
+    (q.backend/recover-stale! inner stale-timeout-ms max-retries))
+  (run-heartbeats! [_] (q.backend/run-heartbeats! inner))
+  (queue-depths [_] (q.backend/queue-depths inner)))
+
+(defn- double-delivery-queue-backend!
+  "Wraps an inner `QueueBackend` so that `publish!` calls the inner backend's
+  `publish!` twice with the same payload. Used by `do-with-test-mq!` when
+  `:duplicate-delivery?` is set, to force listeners under test to handle the
+  MQ's at-least-once contract."
+  [inner]
+  (->DoubleDeliveryBackend inner (:poll-context inner)))
+
+(defn do-with-test-mq!
+  "Function form of [[with-test-mq]]. `opts` may include:
+
+  - `:backend`                 `:memory` (default) or `:appdb`
+  - `:listeners`            listener map, same shape as `with-sync-mq` had
+  - `:duplicate-delivery?`  when true, wraps both backends with a decorator
+                            that publishes every message twice — used to
+                            verify listeners handle at-least-once delivery
+
+  The fixture forces `*publish-buffer-ms*` to 0 so publishes go straight to the
+  backend without waiting for the background flush scheduler. Tests that want to
+  exercise buffering behaviour must rebind it explicitly in the body."
+  ([f] (do-with-test-mq! {} f))
+  ([opts f]
+   (let [{:keys [backend listeners duplicate-delivery?] :or {backend :memory}} opts]
+     (binding [listener/*listeners*               (atom {})
+               q.registry/*queues*                    (atom {})
+               publish-buffer/*publish-buffer*    (atom {})
+               publish-buffer/*publish-buffer-ms* 0]
+       (let [backends (make-fixture-backends backend)
+             queue-be (cond-> (:queue-be backends)
+                        duplicate-delivery? double-delivery-queue-backend!)
+             handle   (mq.init/start! queue-be)
+             ctx      (assoc backends :queue-be queue-be :handle handle)]
+         (try
+           (merge-listeners! listeners)
+           (let [result (f ctx)]
+             (flush! ctx)
+             result)
+           (finally
+             (mq.init/stop! handle))))))))
+
+(defmacro with-test-mq
+  "Starts an MQ subsystem, runs body with `ctx-sym` bound to a fixture context
+  map, then drains and shuts down.
+
+  The binding vector is `[ctx-sym]` or `[ctx-sym opts-map]` where `opts-map`
+  may contain `:backend`, `:duplicate-delivery?`, etc. — anything
+  [[do-with-test-mq!]] accepts.
+
+  An optional listener map may follow the binding vector (same shape as the
+  old `with-sync-mq`). It is merged into the opts under `:listeners`.
+
+  Use `(mq.tu/flush! ctx)` between publishes and assertions, and
+  `(mq.tu/wait-for-idle! ctx)` / `(mq.tu/eventually! ctx pred)` where finer
+  control is needed.
+
+  Examples:
+
+      (with-test-mq [ctx]
+        body)
+
+      (with-test-mq [ctx]
+        {:queue/foo (fn [msg] ...)}
+        body)
+
+      (with-test-mq [ctx {:backend :appdb :duplicate-delivery? true}]
+        body)"
+  {:style/indent 1}
+  [[ctx-sym & [opts-expr]] & body]
+  (let [[listeners body] (if (map? (first body))
+                           [(first body) (rest body)]
+                           [nil body])]
+    `(do-with-test-mq! (merge ~opts-expr {:listeners ~listeners})
+                       (fn [~ctx-sym] ~@body))))
+
+;;; ------------------------------------------- Fixture self-tests -------------------------------------------
+
+(deftest duplicate-delivery-doubles-queue-messages-test
+  (testing "With :duplicate-delivery? true, the queue backend delivers each message twice"
+    (with-test-mq [ctx {:duplicate-delivery? true}]
+      (let [received (atom [])]
+        (listen! :queue/chaos-smoke #(swap! received conj %))
+        (mq/with-queue :queue/chaos-smoke [q]
+          (mq/put q "hello"))
+        (eventually! ctx #(= 2 (count @received)) 5000)
+        (is (= ["hello" "hello"] @received)
+            "Single published message is delivered to the listener twice")
+        (mq/unlisten! :queue/chaos-smoke)))))
+
+(deftest duplicate-delivery-off-by-default-test
+  (testing "The default fixture delivers each message exactly once"
+    (with-test-mq [ctx]
+      (let [received (atom [])]
+        (listen! :queue/chaos-default #(swap! received conj %))
+        (mq/with-queue :queue/chaos-default [q]
+          (mq/put q "once"))
+        (eventually! ctx #(>= (count @received) 1) 5000)
+        ;; Give the poll thread an extra moment in case a second (unexpected)
+        ;; delivery is on the way, then confirm exactly one happened.
+        (Thread/sleep 100)
+        (is (= ["once"] @received)
+            "Default fixture delivers exactly once")
+        (mq/unlisten! :queue/chaos-default)))))

test/metabase/test/initialize.clj

@@ -93,9 +93,17 @@
   ((resolve 'metabase.test.initialize.plugins/init-test-drivers!)
    [:driver-deprecation-test-legacy :driver-deprecation-test-new :secret-test-driver]))
 
+;; Initialize the MQ subsystem: sync backends, no buffering, register listeners.
+;; Must happen before :db so that listeners (e.g. connection-pool-invalidated) are available
+;; when database initialization triggers events.
+(define-initialization :mq
+  (classloader/require 'metabase.test.initialize.mq)
+  ((resolve 'metabase.test.initialize.mq/init!)))
+
 ;; initializing the DB also does setup needed so the scheduler will work correctly. (Remember that the scheduler uses
 ;; a JDBC backend!)
 (define-initialization :db
+  (initialize-if-needed! :mq)
   (classloader/require 'metabase.test.initialize.db)
   ((resolve 'metabase.test.initialize.db/init!)))
 

test/metabase/test/initialize/mq.clj

@@ -0,0 +1,17 @@
+(ns metabase.test.initialize.mq
+  "Initializes the MQ subsystem for test mode: memory backends, zero publish
+  buffering, and registering all `def-listener!` implementations."
+  (:require
+   [metabase.mq.init :as mq.init]
+   [metabase.mq.publish-buffer :as publish-buffer]))
+
+(defn init! []
+  (alter-var-root #'publish-buffer/*publish-buffer-ms* (constantly 0))
+  ;; Load driver.init so namespaces containing def-listener! calls are loaded
+  ;; before register-listeners! (called inside mq.init/start!) iterates them.
+  ;; In production, core/init.clj loads these.
+  (require 'metabase.driver.init)
+  ;; Drive through mq.init/start! so the worker pool, publish-buffer flush thread,
+  ;; and the memory backend's polling loop all come up — otherwise MQ-driven behaviour
+  ;; outside `with-test-mq` would silently drop messages onto a non-polling queue.
+  (mq.init/start! :queue.backend/memory))