Fix CI failures: portable fetch query, force-flush buffer on stop!

- queue/appdb fetch!: rewrite the per-queue head selection as N small SELECT FOR
  UPDATE SKIP LOCKED queries inside one transaction, instead of a single LATERAL
  + VALUES query. Postgres, MySQL 8+, and H2 all support LATERAL but MariaDB does
  not, so the LATERAL version broke the MariaDB / MySQL test jobs. The per-queue
  loop is N+1 round trips per poll iteration, but N is bounded by the registered
  listener count (typically <10) and SKIP LOCKED still applies per row, so
  concurrent consumers still get different rows. Empty result correctly returns
  nil (was leaking [] in the new version).

- mq.init/stop! handle: always force-flush the (currently bound) publish buffer
  before checking executor ownership. Previously, in CI test mode where
  `metabase.test.initialize/mq` claimed ownership of the global flush executor
  first, the shutdown-drain tests' own stop! call ran with owns-buffer-flush?
  = false, so it skipped the flush — and their bound *publish-buffer* never
  drained to the DB. The fix runs `(flush-publish-buffer! true)` unconditionally,
  which drains whatever atom is currently bound. The executor shutdown remains
  ownership-gated.

.clj-kondo/config.edn

@@ -312,8 +312,7 @@
    metabase-enterprise.transforms.interface/execute!  {:message "Use metabase-enterprise.transforms.execute/execute! instead of metabase-enterprise.transforms.interface/execute!"}
    next.jdbc/active-tx?                               {:message "Use metabase.app-db.core/in-transaction? to see if we are in a transaction"}
    toucan2.core/debug                                 {:message "Don't commit code using t2/debug"}
-   toucan2.tools.debug/debug                          {:message "Don't commit code using t2/debug"}
-   }
+   toucan2.tools.debug/debug                          {:message "Don't commit code using t2/debug"}}
 
   :discouraged-namespace
   {camel-snake-kebab.core {:message "CSK is not Turkish-safe, use the versions in metabase.util instead."}
@@ -593,6 +592,10 @@
    metabase.model-persistence.api-test/with-persistence-setup!                                                               hooks.common/with-one-top-level-binding
    metabase.model-persistence.api-test/with-setup!                                                                           hooks.common/with-one-top-level-binding
    metabase.model-persistence.test-util/with-persistence-enabled!                                                            hooks.metabase.model-persistence.test-util/with-persistence-enabled!
+   metabase.mq.core/with-queue                                                                                               hooks.common/let-second
+   metabase.mq.core/with-topic                                                                                               hooks.common/let-second
+   metabase.mq.test-util/with-test-mq                                                                                        hooks.common/with-one-binding
+   metabase.mq.core/def-listener!                                                                                            hooks.metabase.mq.impl/def-listener
    metabase.models.interface/define-batched-hydration-method                                                                 hooks.metabase.models.interface/define-hydration-method
    metabase.models.interface/define-simple-hydration-method                                                                  hooks.metabase.models.interface/define-hydration-method
    metabase.permissions.core/allow-changing-all-users-group-members                                                          hooks.common/do*

.clj-kondo/config/modules/config.edn

@@ -1354,6 +1354,16 @@
                     :model/Table
                     :model/User}}
 
+  mq
+  {:team "UX West"
+   :api  #{metabase.mq.core metabase.mq.init}
+   :uses #{analytics-interface
+           app-db
+           models
+           settings
+           startup
+           util}}
+
   native-query-snippets
   {:team "Graphy"
    :api  #{metabase.native-query-snippets.api

.clj-kondo/src/hooks/metabase/mq/impl.clj

@@ -0,0 +1,24 @@
+(ns hooks.metabase.mq.impl
+  (:require
+   [clj-kondo.hooks-api :as hooks]))
+
+(defn def-listener
+  "Kondo hook for `mq/def-listener`. Rewrites to a `fn` form so kondo can analyze bindings.
+
+   Forms:
+     (mq/def-listener! :topic/foo [msg] body...)
+     (mq/def-listener! :queue/bar {:config map} [msgs] body...)"
+  [{:keys [node]}]
+  (let [[_ _channel-name & args] (:children node)
+        ;; skip optional config map
+        args (if (hooks/map-node? (first args))
+               (rest args)
+               args)
+        [bindings & body] args
+        ;; rewrite as: (fn [bindings...] body...)
+        new-node (hooks/list-node
+                  (list*
+                   (hooks/token-node 'fn)
+                   bindings
+                   body))]
+    {:node new-node}))

dev/src/dev/mq_perf.clj

@@ -0,0 +1,870 @@
+(ns dev.mq-perf
+  "Performance benchmarks for the appdb-backed MQ system (queues and topics).
+
+  Measures publishing throughput, end-to-end latency, and sustained throughput
+  under different load levels. Benchmarks exercise the full realistic pipeline:
+  publish buffer → DB → poll thread → worker pool → listener.
+
+  Self-contained — starts the MQ subsystem without the full Metabase server.
+  Supports multi-node testing: run multiple JVM instances against the same DB.
+
+  Example usage:
+    ;; Single-node: start MQ and run all benchmarks
+    (start-mq!)
+    (run-all-benchmarks!)
+
+    ;; Multi-node coordination: start multiple JVMs, each calls:
+    (start-mq!)
+    (run-all-benchmarks-coordinated!)
+    ;; Then from any node (or a separate REPL), publish the go signal:
+    (signal-go!)
+
+    ;; Individual benchmarks
+    (bench-publish-throughput! {:n 1000 :batch-sizes [1 10 100]})
+    (bench-e2e-latency! {:n 50 :rate :medium :transport :queue})
+    (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :queue})
+    (bench-batch-listener! {:n 200 :batch-sizes [1 10 50]})
+    (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})"
+  (:require
+   [clojure.string :as str]
+   [metabase.app-db.connection :as mdb.conn]
+   [metabase.mq.core :as mq]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as polling]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.topic.appdb :as topic.appdb]
+   [metabase.mq.topic.backend :as topic.backend]
+   [toucan2.core :as t2])
+  (:import
+   (java.lang.reflect InvocationHandler Method Proxy)
+   (java.sql Connection PreparedStatement Statement)
+   (java.util.concurrent CountDownLatch TimeUnit)
+   (javax.sql DataSource)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------------ RTT injection ------------------------------------------------
+;;;
+;;; Wraps the application DB's `DataSource` so every JDBC `execute*` call sleeps a randomized
+;;; interval before hitting Postgres. This lets us see what the perf picture looks like with
+;;; cloud-realistic latency (~1-5 ms RTT) without leaving the JVM. The wrapper preserves the
+;;; pooled DataSource underneath, so connection acquisition is unaffected — only query
+;;; round-trips pay the simulated cost.
+
+(defonce ^:private rtt-ms
+  ;; Use an atom rather than a dynamic var so the injected RTT applies to *every* thread —
+  ;; including the backend poll threads and the worker pool — without needing per-thread
+  ;; bindings to be conveyed.
+  (atom nil))
+
+(defn- sleep-rtt! []
+  (when-let [[mn mx] @rtt-ms]
+    (let [delay (long (if (= mn mx) mn (+ mn (rand-int (inc (- mx mn))))))]
+      (when (pos? delay) (Thread/sleep delay)))))
+
+(defn- proxy-statement
+  "Wrap a Statement / PreparedStatement so every `execute*` call sleeps the injected RTT
+  before delegating. Returns a proxy that implements both `Statement` and
+  `PreparedStatement` (PreparedStatement extends Statement, so a single proxy covers both)."
+  [^Statement stmt]
+  (Proxy/newProxyInstance
+   (.getClassLoader Statement)
+   (into-array Class [Statement PreparedStatement])
+   (reify InvocationHandler
+     (invoke [_ _proxy method args]
+       (when (.startsWith (.getName ^Method method) "execute")
+         (sleep-rtt!))
+       (.invoke method stmt args)))))
+
+(defn- proxy-connection
+  "Wrap a Connection so prepareStatement / createStatement return latency-injecting Statements."
+  [^Connection conn]
+  (Proxy/newProxyInstance
+   (.getClassLoader Connection)
+   (into-array Class [Connection])
+   (reify InvocationHandler
+     (invoke [_ _proxy method args]
+       (let [n (.getName ^Method method)
+             result (.invoke method conn args)]
+         (if (and (or (= n "prepareStatement") (= n "createStatement"))
+                  (instance? Statement result))
+           (proxy-statement result)
+           result))))))
+
+(defn- latent-datasource
+  "Returns a wrapped DataSource whose connections inject RTT before every `execute*`.
+  Delegates to the underlying pooled DataSource for connection acquisition itself."
+  ^DataSource [^DataSource inner]
+  (reify DataSource
+    (getConnection [_]            (proxy-connection (.getConnection inner)))
+    (getConnection [_ user pw]    (proxy-connection (.getConnection inner user pw)))
+    (getLoginTimeout [_]          (.getLoginTimeout inner))
+    (setLoginTimeout [_ s]        (.setLoginTimeout inner s))
+    (getLogWriter [_]             (.getLogWriter inner))
+    (setLogWriter [_ pw]          (.setLogWriter inner pw))
+    (getParentLogger [_]          (.getParentLogger inner))
+    (^boolean isWrapperFor [_ ^Class iface] (.isWrapperFor inner iface))
+    (^Object unwrap [_ ^Class iface]        (.unwrap inner iface))))
+
+(defonce ^:private wrapped-datasource? (atom false))
+
+(defn install-rtt!
+  "Wraps the app DB's DataSource so every JDBC `execute*` call sleeps `[min-ms max-ms]`
+  before delegating. Idempotent — calling this twice keeps a single wrapper. Use
+  `(install-rtt! nil nil)` (or `(uninstall-rtt!)`) to disable injection without unwrapping.
+  Returns the new effective `[min max]`."
+  [min-ms max-ms]
+  (reset! rtt-ms (when (and min-ms max-ms) [min-ms max-ms]))
+  (when-not @wrapped-datasource?
+    (let [orig    mdb.conn/*application-db*
+          wrapped (mdb.conn/application-db (.db-type orig)
+                                           (latent-datasource (.data-source orig))
+                                           :create-pool? false)]
+      (alter-var-root #'mdb.conn/*application-db* (constantly wrapped))
+      (reset! wrapped-datasource? true)))
+  @rtt-ms)
+
+(defn uninstall-rtt!
+  "Disables RTT injection. Leaves the DataSource wrapper in place (no-op when rtt-ms is nil)
+  to avoid surprises if other code captured the wrapped instance."
+  []
+  (reset! rtt-ms nil))
+
+(defmacro with-rtt-ms
+  "Runs `body` with `[min-ms max-ms]` RTT injected on every JDBC execute, then restores the
+  prior setting."
+  [[min-ms max-ms] & body]
+  `(let [prev# @rtt-ms]
+     (install-rtt! ~min-ms ~max-ms)
+     (try ~@body
+          (finally
+            (reset! rtt-ms prev#)))))
+
+;;; ------------------------------------------------ Startup ------------------------------------------------
+
+(defonce ^:private started? (atom false))
+
+(defn start-mq!
+  "Minimal startup: connects to app DB and starts the MQ subsystem.
+  No Jetty, no scheduler, no sample data. Idempotent."
+  []
+  (let [setup-db! (requiring-resolve 'metabase.app-db.core/setup-db!)]
+    (setup-db! :create-sample-content? false))
+  (let [setup-prometheus! (requiring-resolve 'metabase.analytics.prometheus/setup!)]
+    (setup-prometheus!))
+  (require 'metabase.mq.init)
+  (let [startup! (requiring-resolve 'metabase.startup.core/def-startup-logic!)]
+    (startup! :metabase.mq.init/MqStart))
+  (reset! started? true)
+  (println "MQ system started."))
+
+(defn- ensure-started!
+  "Starts the MQ system if not yet started, and restarts any dead poll threads."
+  []
+  (when-not @started?
+    (start-mq!))
+  ;; Restart poll threads if they died (e.g., from InterruptedException during a previous benchmark).
+  ;; start! is idempotent and start-polling! detects dead futures.
+  (let [start-transports! (requiring-resolve 'metabase.mq.impl/start-transports)]
+    (start-transports!)))
+
+;;; ------------------------------------------------ Utilities ------------------------------------------------
+
+(def ^:private shared-queues
+  "Fixed set of 10 shared queue names used across all nodes for cross-node contention testing."
+  (mapv #(keyword "queue" (str "bench-shared-" %)) (range 10)))
+
+(def ^:private shared-topic :topic/bench-shared)
+
+(defn- run-id
+  "Generates an 8-character unique identifier for a benchmark run."
+  []
+  (subs (str (random-uuid)) 0 8))
+
+(defn- make-channel
+  "Creates a unique channel keyword for benchmarking."
+  [transport id suffix]
+  (keyword (name transport) (str "bench-" id "-" suffix)))
+
+(defn- random-shared-queue
+  "Returns a random shared queue name."
+  []
+  (nth shared-queues (rand-int (count shared-queues))))
+
+(defn- percentile
+  "Returns the p-th percentile from a sorted vector (p in [0,1])."
+  [sorted-vec p]
+  (if (empty? sorted-vec)
+    0.0
+    (let [idx (min (dec (count sorted-vec))
+                   (int (* p (count sorted-vec))))]
+      (double (nth sorted-vec idx)))))
+
+(defn- collect-stats
+  "Computes summary statistics from a collection of numeric values.
+  Returns {:count :min :max :avg :p50 :p95 :p99}."
+  [values]
+  (if (empty? values)
+    {:count 0 :min 0.0 :max 0.0 :avg 0.0 :p50 0.0 :p95 0.0 :p99 0.0}
+    (let [sorted (vec (sort values))
+          n      (count sorted)]
+      {:count n
+       :min   (double (first sorted))
+       :max   (double (last sorted))
+       :avg   (/ (reduce + 0.0 sorted) n)
+       :p50   (percentile sorted 0.50)
+       :p95   (percentile sorted 0.95)
+       :p99   (percentile sorted 0.99)})))
+
+(defn- fmt
+  "Formats a number for display."
+  [x]
+  (if (integer? x)
+    (str x)
+    (format "%.2f" (double x))))
+
+(defn- print-table!
+  "Prints a formatted ASCII table."
+  [title headers rows]
+  (let [col-widths (mapv (fn [i]
+                           (apply max
+                                  (count (nth headers i))
+                                  (map #(count (str (nth % i))) rows)))
+                         (range (count headers)))
+        separator  (str "+-" (str/join "-+-" (map #(apply str (repeat % \-)) col-widths)) "-+")
+        fmt-row    (fn [vals]
+                     (str "| "
+                          (str/join " | "
+                                    (map-indexed (fn [i v]
+                                                   (format (str "%-" (nth col-widths i) "s") (str v)))
+                                                 vals))
+                          " |"))]
+    (println)
+    (println title)
+    (println separator)
+    (println (fmt-row headers))
+    (println separator)
+    (doseq [row rows]
+      (println (fmt-row row)))
+    (println separator)
+    (println)))
+
+(defn- cleanup!
+  "Removes all benchmark data for the given run-id."
+  [id]
+  (t2/delete! :queue_message_batch :queue_name [:like (str "bench-" id "%")])
+  (t2/delete! :topic_message_batch :topic_name [:like (str "bench-" id "%")])
+  (doseq [[ch _] @listener/*listeners*]
+    (when (str/includes? (name ch) (str "bench-" id))
+      (mq/unlisten! ch))))
+
+(defn- publish-at-rate!
+  "Publishes `n` messages at the given rate (msgs/sec). `publish-fn` is called with each seq num.
+  If rate is nil, publishes as fast as possible (burst mode)."
+  [n rate publish-fn]
+  (if (nil? rate)
+    (dotimes [i n]
+      (publish-fn i))
+    (let [interval-ms (long (/ 1000 rate))]
+      (dotimes [i n]
+        (let [start (System/currentTimeMillis)]
+          (publish-fn i)
+          (let [elapsed (- (System/currentTimeMillis) start)
+                sleep-ms (- interval-ms elapsed)]
+            (when (pos? sleep-ms)
+              (Thread/sleep (long sleep-ms)))))))))
+
+(def ^:private rate-map
+  {:low    10
+   :medium 100
+   :high   1000
+   :burst  nil})
+
+(defn- rate->msgs-sec [rate]
+  (get rate-map rate rate))
+
+;;; ---------------------------------------- Benchmark 1: Publishing Throughput ----------------------------------------
+
+(defn bench-publish-throughput!
+  "Measures raw publishing speed for different batch sizes.
+  This benchmark bypasses the publish buffer to measure direct DB insert speed.
+  Options:
+    :n           - total messages to publish per batch-size test (default 1000)
+    :batch-sizes - vector of batch sizes to test (default [1 10 100])
+    :transport   - :queue or :topic (default :queue)"
+  ([] (bench-publish-throughput! {}))
+  ([{:keys [n batch-sizes transport]
+     :or   {n 1000 batch-sizes [1 10 100] transport :queue}}]
+   (ensure-started!)
+   (let [id      (run-id)
+         results (atom [])]
+     (try
+       (doseq [bs batch-sizes]
+         (let [channel    (make-channel transport id (str "pub-" bs))
+               iterations (/ n bs)
+               start      (System/nanoTime)]
+           (binding [publish-buffer/*publish-buffer-ms* 0]
+             (dotimes [i iterations]
+               (let [msgs (vec (for [j (range bs)]
+                                 {:seq (+ (* i bs) j) :ts (System/currentTimeMillis)}))]
+                 (if (= transport :queue)
+                   (q.backend/publish! q.appdb/backend channel msgs)
+                   (topic.backend/publish! topic.appdb/backend channel msgs)))))
+           (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+             (swap! results conj
+                    [(str bs) (fmt (/ n (/ elapsed-ms 1000.0))) (fmt elapsed-ms)]))))
+       (print-table!
+        (str "Publishing Throughput (" (name transport) ", n=" n ")")
+        ["Batch Size" "Msgs/sec" "Total ms"]
+        @results)
+       (finally
+         (cleanup! id))))))
+
+;;; ---------------------------------------- Benchmark 2: End-to-End Latency ----------------------------------------
+
+(defn bench-e2e-latency!
+  "Measures end-to-end latency from publish to listener receipt via the full async pipeline.
+  Uses the real publish API including the publish buffer.
+  Publishes randomly to the 10 shared queues (or the shared topic) so multiple nodes
+  compete for the same messages.
+  Options:
+    :n         - number of messages to publish (default 50)
+    :rate      - :low, :medium, :high, or :burst (default :medium)
+    :transport - :queue or :topic (default :queue)"
+  ([] (bench-e2e-latency! {}))
+  ([{:keys [n rate transport]
+     :or   {n 50 rate :medium transport :queue}}]
+   (ensure-started!)
+   (let [latencies (atom [])
+         received  (atom 0)
+         channels  (if (= transport :queue) shared-queues [shared-topic])]
+     (try
+       ;; Register listeners on all shared channels
+       (doseq [ch channels]
+         (when-not (listener/get-listener ch)
+           (listener/batch-listen! ch
+                                   (fn [msgs]
+                                     (doseq [msg msgs]
+                                       (let [receive-ns (System/nanoTime)
+                                             publish-ns (get msg "publish-ns")]
+                                         (when publish-ns
+                                           (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6))))
+                                       (swap! received inc)))
+                                   {})))
+       (let [msgs-sec (rate->msgs-sec rate)
+             start    (System/nanoTime)]
+         ;; Publish to random shared channels
+         (publish-at-rate!
+          n msgs-sec
+          (fn [i]
+            (let [ch (if (= transport :queue) (random-shared-queue) shared-topic)]
+              (if (= transport :queue)
+                (mq/with-queue ch [q]
+                  (mq/put q {:seq i :publish-ns (System/nanoTime)}))
+                (mq/with-topic ch [t]
+                  (mq/put t {:seq i :publish-ns (System/nanoTime)}))))))
+         ;; Force flush the publish buffer then wait for delivery
+         (publish-buffer/flush-publish-buffer!)
+         ;; Wait until we've received at least n messages or timeout
+         (let [deadline (+ (System/currentTimeMillis)
+                           (long (* 1000 (+ 30 (/ n (max 1 (or msgs-sec 1000)))))))]
+           (while (and (< @received n)
+                       (< (System/currentTimeMillis) deadline))
+             (Thread/sleep (long 100))))
+         (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)
+               stats      (collect-stats @latencies)]
+           (print-table!
+            (str "End-to-End Latency (" (name transport) ", rate=" (name rate) ", n=" n ")")
+            ["Metric" "Value"]
+            [["Published" (str n)]
+             ["Received" (str @received)]
+             ["Other nodes" (str (max 0 (- n @received)))]
+             ["From others" (str (max 0 (- @received n)))]
+             ["Total ms" (fmt elapsed-ms)]
+             ["P50 ms" (fmt (:p50 stats))]
+             ["P95 ms" (fmt (:p95 stats))]
+             ["P99 ms" (fmt (:p99 stats))]
+             ["Min ms" (fmt (:min stats))]
+             ["Max ms" (fmt (:max stats))]
+             ["Avg ms" (fmt (:avg stats))]])))
+       (finally
+         (doseq [ch channels]
+           (when (listener/get-listener ch)
+             (mq/unlisten! ch))))))))
+
+;;; ---------------------------------------- Benchmark 3: Sustained Throughput ----------------------------------------
+
+(defn bench-sustained-throughput!
+  "Publishes continuously while consuming for a given duration via the full async pipeline.
+  Publishes randomly to the 10 shared queues (or shared topic).
+  Options:
+    :duration-sec - how long to publish (default 15)
+    :rate         - :low, :medium, :high, or :burst (default :medium)
+    :transport    - :queue or :topic (default :queue)"
+  ([] (bench-sustained-throughput! {}))
+  ([{:keys [duration-sec rate transport]
+     :or   {duration-sec 15 rate :medium transport :queue}}]
+   (ensure-started!)
+   (let [channels  (if (= transport :queue) shared-queues [shared-topic])
+         received  (atom 0)
+         latencies (atom [])
+         published (atom 0)
+         stop?     (atom false)]
+     (try
+       ;; Register listeners on all shared channels
+       (doseq [ch channels]
+         (when-not (listener/get-listener ch)
+           (listener/batch-listen! ch
+                                   (fn [msgs]
+                                     (doseq [msg msgs]
+                                       (let [receive-ns (System/nanoTime)
+                                             publish-ns (get msg "publish-ns")]
+                                         (when publish-ns
+                                           (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6)))
+                                         (swap! received inc))))
+                                   {})))
+       (let [msgs-sec   (rate->msgs-sec rate)
+             start      (System/nanoTime)
+             pub-thread (Thread.
+                         (fn []
+                           (loop []
+                             (when-not @stop?
+                               (try
+                                 (let [ch (if (= transport :queue) (random-shared-queue) shared-topic)]
+                                   (if (= transport :queue)
+                                     (mq/with-queue ch [q]
+                                       (mq/put q {:seq @published :publish-ns (System/nanoTime)}))
+                                     (mq/with-topic ch [t]
+                                       (mq/put t {:seq @published :publish-ns (System/nanoTime)}))))
+                                 (swap! published inc)
+                                 (catch Exception _))
+                               (when msgs-sec
+                                 (Thread/sleep (long (/ 1000 msgs-sec))))
+                               (recur)))))]
+         (.start pub-thread)
+         (Thread/sleep (long (* duration-sec 1000)))
+         (reset! stop? true)
+         (.join pub-thread 5000)
+         ;; Flush publish buffer and wait for remaining messages to drain
+         (publish-buffer/flush-publish-buffer!)
+         (let [drain-start (System/currentTimeMillis)]
+           (while (and (< @received @published)
+                       (< (- (System/currentTimeMillis) drain-start) 30000))
+             (publish-buffer/flush-publish-buffer!)
+             (Thread/sleep (long 500))))
+         (let [elapsed-ms  (/ (- (System/nanoTime) start) 1e6)
+               stats       (collect-stats @latencies)]
+           (print-table!
+            (str "Sustained Throughput (" (name transport) ", rate=" (name rate)
+                 ", duration=" duration-sec "s)")
+            ["Metric" "Value"]
+            [["Published" (str @published)]
+             ["Received" (str @received)]
+             ["Other nodes" (str (max 0 (- @published @received)))]
+             ["From others" (str (max 0 (- @received @published)))]
+             ["Duration ms" (fmt elapsed-ms)]
+             ["Actual pub/sec" (fmt (/ @published (/ elapsed-ms 1000.0)))]
+             ["Actual recv/sec" (fmt (/ @received (/ elapsed-ms 1000.0)))]
+             ["P50 latency ms" (fmt (:p50 stats))]
+             ["P95 latency ms" (fmt (:p95 stats))]
+             ["P99 latency ms" (fmt (:p99 stats))]
+             ["Max latency ms" (fmt (:max stats))]])))
+       (finally
+         (doseq [ch channels]
+           (when (listener/get-listener ch)
+             (mq/unlisten! ch))))))))
+
+;;; ---------------------------------------- Benchmark 4: Batch Listener ----------------------------------------
+
+(defn bench-batch-listener!
+  "Compares single-message vs batch listener processing rates.
+  Uses the shared queues so multiple nodes compete.
+  Options:
+    :n           - total messages to publish (default 200)
+    :batch-sizes - vector of max-batch-messages values to test (default [1 10 50])"
+  ([] (bench-batch-listener! {}))
+  ([{:keys [n batch-sizes]
+     :or   {n 200 batch-sizes [1 10 50]}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [bs batch-sizes]
+       (let [received (atom 0)]
+         (try
+           ;; Register listeners on all shared queues
+           (doseq [ch shared-queues]
+             (when-not (listener/get-listener ch)
+               (listener/batch-listen! ch
+                                       (fn [msgs]
+                                         (swap! received + (count msgs)))
+                                       {:max-batch-messages bs})))
+           ;; Publish all messages randomly to shared queues
+           (dotimes [i n]
+             (mq/with-queue (random-shared-queue) [q]
+               (mq/put q {:seq i})))
+           (publish-buffer/flush-publish-buffer!)
+           ;; Wait for messages to be consumed
+           (let [start   (System/nanoTime)
+                 deadline (+ (System/currentTimeMillis) 30000)]
+             (while (and (< @received n)
+                         (< (System/currentTimeMillis) deadline))
+               (Thread/sleep (long 100)))
+             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+               (swap! results conj
+                      [(str bs) (str @received) (fmt elapsed-ms)
+                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
+           (finally
+             (doseq [ch shared-queues]
+               (when (listener/get-listener ch)
+                 (mq/unlisten! ch)))))))
+     (print-table!
+      (str "Batch Listener Comparison (n=" n ")")
+      ["Max Batch" "Received" "Process ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Benchmark 5: Multiple Queues ----------------------------------------
+
+(defn bench-multi-queue!
+  "Measures performance with varying numbers of the shared queues.
+  Tests with subsets of the 10 shared queues to show scaling.
+  Options:
+    :queue-counts   - vector of queue counts to test (default [1 5 10])
+    :msgs-per-queue - messages per queue (default 50)"
+  ([] (bench-multi-queue! {}))
+  ([{:keys [queue-counts msgs-per-queue]
+     :or   {queue-counts [1 5 10] msgs-per-queue 50}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [qc queue-counts]
+       (let [channels (subvec shared-queues 0 (min qc (count shared-queues)))
+             total    (* (count channels) msgs-per-queue)
+             received (atom 0)]
+         (try
+           ;; Register listeners
+           (doseq [ch channels]
+             (when-not (listener/get-listener ch)
+               (listener/batch-listen! ch
+                                       (fn [msgs]
+                                         (swap! received + (count msgs)))
+                                       {})))
+           ;; Publish to the queues
+           (let [start (System/nanoTime)]
+             (doseq [ch channels]
+               (dotimes [i msgs-per-queue]
+                 (mq/with-queue ch [q]
+                   (mq/put q {:seq i}))))
+             (publish-buffer/flush-publish-buffer!)
+             ;; Wait for messages
+             (let [deadline (+ (System/currentTimeMillis) 60000)]
+               (while (and (< @received total)
+                           (< (System/currentTimeMillis) deadline))
+                 (Thread/sleep (long 100))))
+             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+               (swap! results conj
+                      [(str (count channels)) (str total) (str @received) (fmt elapsed-ms)
+                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
+           (finally
+             (doseq [ch channels]
+               (when (listener/get-listener ch)
+                 (mq/unlisten! ch)))))))
+     (print-table!
+      (str "Multiple Queues (msgs-per-queue=" msgs-per-queue ")")
+      ["Queues" "Total Msgs" "Received" "Total ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Benchmark 6: Concurrent Consumers (LATERAL/SKIP LOCKED) ----------------------------------------
+
+(defn bench-concurrent-consumers!
+  "Validates the LATERAL/SKIP LOCKED fetch and exactly-once delivery under multi-consumer
+  contention by spinning up multiple `AppDbQueueBackend` instances in the same JVM, each
+  with its own listener atom and poll thread. All instances share the queue and DB.
+
+  What this actually measures:
+   - **Correctness**: every row is processed exactly once (`Dups` should always be 0; the
+     `Imbalance` shows how the work spread but the total received must equal `n`).
+   - **Per-row contention**: `mq.impl/active-handlers` is a JVM-global atom, so at most ONE
+     consumer can be actively handling the channel at a time within a single JVM. The other
+     consumers see `channel-busy?` and skip the queue. So this test does *not* validate
+     cross-node throughput scaling — for that you need separate JVMs.
+   - **Buffer-vs-row behavior**: `*publish-buffer-ms*` is forced to 0 so each `put` becomes
+     its own row; otherwise the buffer collapses all `n` messages into a single row that
+     only one consumer can grab.
+
+  Options:
+    :consumer-counts - vector of consumer counts to test (default [1 2 4 8])
+    :n               - total messages per run (default 2000)
+    :queue-name      - queue to use (default :queue/bench-concurrent)"
+  ([] (bench-concurrent-consumers! {}))
+  ([{:keys [consumer-counts n queue-name]
+     :or   {consumer-counts [1 2 4 8] n 2000 queue-name :queue/bench-concurrent}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [c consumer-counts]
+       (let [per-consumer-received (vec (repeatedly c #(atom 0)))
+             seen-batch-ids        (atom #{})
+             duplicates            (atom 0)
+             backends              (vec (repeatedly c q.appdb/make-backend))
+             listener-atoms        (vec (repeatedly c #(atom {})))]
+         (try
+           (doseq [i (range c)]
+             (binding [listener/*listeners* (nth listener-atoms i)]
+               (listener/batch-listen! queue-name
+                                       (fn [msgs]
+                                         (doseq [m msgs]
+                                           (let [bid (get m "batch-id" m)]
+                                             (when (contains? @seen-batch-ids bid)
+                                               (swap! duplicates inc))
+                                             (swap! seen-batch-ids conj bid)))
+                                         (swap! (nth per-consumer-received i) + (count msgs)))
+                                       {})
+               (q.backend/start! (nth backends i))))
+           (let [start (System/nanoTime)]
+             ;; Publish via root listener atom (so the buffer flush sees them). Bypass the
+             ;; time-windowed publish buffer (set *publish-buffer-ms* to 0) so each `put`
+             ;; becomes its own DB row — otherwise all `n` messages collapse into one row
+             ;; and there's nothing for multiple consumers to fetch in parallel.
+             (binding [listener/*listeners* (atom {queue-name {:listener identity
+                                                               :max-batch-messages 100}})
+                       publish-buffer/*publish-buffer-ms* 0]
+               (dotimes [i n]
+                 (mq/with-queue queue-name [q]
+                   (mq/put q {:id i})))
+               (publish-buffer/flush-publish-buffer!))
+             (let [total (atom 0)
+                   deadline (+ (System/currentTimeMillis) 60000)]
+               (while (and (< @total n)
+                           (< (System/currentTimeMillis) deadline))
+                 ;; Notify all poll threads so consumer backends drain promptly. In production
+                 ;; the publish-side notify reaches the same node; multi-instance benchmarks
+                 ;; sit between nodes where no cross-node wakeup exists.
+                 (polling/notify-all!)
+                 (reset! total (reduce + (map deref per-consumer-received)))
+                 (Thread/sleep 50))
+               (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)
+                     received-per-consumer (mapv deref per-consumer-received)
+                     min-c   (apply min received-per-consumer)
+                     max-c   (apply max received-per-consumer)
+                     spread  (when (pos? max-c)
+                               (* 100.0 (- 1.0 (/ (double min-c) max-c))))]
+                 (swap! results conj
+                        [(str c)
+                         (str @total)
+                         (str (apply max received-per-consumer))
+                         (str (apply min received-per-consumer))
+                         (if spread (format "%.1f%%" spread) "n/a")
+                         (str @duplicates)
+                         (fmt elapsed-ms)
+                         (fmt (/ @total (/ elapsed-ms 1000.0)))]))))
+           (finally
+             ;; Shut each backend's polling thread (per-instance, doesn't touch production)
+             (doseq [be backends] (q.backend/shutdown! be))
+             ;; Clean DB rows for this queue
+             (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+     (print-table!
+      (str "Concurrent Consumers (n=" n " per run, queue=" (name queue-name) ")")
+      ["Consumers" "Total" "Max/c" "Min/c" "Imbalance" "Dups" "Total ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Benchmark 7: Stale-Recovery Race ----------------------------------------
+
+(defn bench-stale-recovery-race!
+  "Pre-populates `:n` rows as `status=processing` with a stale heartbeat, then runs the
+  recovery function from `:workers` threads simultaneously. Validates that the guarded
+  bulk UPDATE delivers each row to exactly one winner regardless of concurrency.
+
+  Options:
+    :n       - number of stale rows to seed (default 500)
+    :workers - concurrent recovery threads (default 8)"
+  ([] (bench-stale-recovery-race! {}))
+  ([{:keys [n workers] :or {n 500 workers 8}}]
+   (ensure-started!)
+   (let [queue-name (keyword "queue" (str "bench-stale-" (run-id)))
+         stale-ts   (java.sql.Timestamp/from
+                     (.minusMillis (java.time.Instant/now) (* 20 60 1000)))]
+     (try
+       (let [rows (vec (for [_ (range n)]
+                         {:queue_name (name queue-name)
+                          :messages   "[]"
+                          :status     "processing"
+                          :status_heartbeat stale-ts
+                          :owner      "dead-node"}))]
+         (t2/insert! :queue_message_batch rows))
+       (let [recoveries (atom [])
+             latch      (CountDownLatch. 1)
+             threads    (mapv (fn [_]
+                                (let [r (bound-fn []
+                                          (.await latch)
+                                          (let [n-recovered (@#'q.appdb/recover-stale-processing-batches!)]
+                                            (swap! recoveries conj n-recovered)))]
+                                  (Thread. ^Runnable r)))
+                              (range workers))
+             start      (System/nanoTime)]
+         (run! #(.start ^Thread %) threads)
+         (.countDown latch)
+         (run! #(.join ^Thread % 30000) threads)
+         (let [elapsed-ms  (/ (- (System/nanoTime) start) 1e6)
+               total-rec   (reduce + @recoveries)
+               final-state (frequencies (map :status (t2/select :queue_message_batch
+                                                                :queue_name (name queue-name))))]
+           (print-table!
+            (str "Stale Recovery Race (n=" n " stale, workers=" workers ")")
+            ["Metric" "Value"]
+            [["Sum of returned counts" (str total-rec)]
+             ["Stale rows pre-populated" (str n)]
+             ["Sum == n? (correctness)" (if (= total-rec n) "YES" (str "NO  diff=" (- total-rec n)))]
+             ["Final 'pending'" (str (get final-state "pending" 0))]
+             ["Final 'failed'" (str (get final-state "failed" 0))]
+             ["Final 'processing'" (str (get final-state "processing" 0))]
+             ["Total wall-clock ms" (fmt elapsed-ms)]
+             ["Per-row ms (effective)" (fmt (/ elapsed-ms (max 1 n)))]])))
+       (finally
+         (t2/delete! :queue_message_batch :queue_name (name queue-name)))))))
+
+;;; ---------------------------------------- Benchmark 8: Concurrent Publish + Consume ----------------------------------------
+
+(defn bench-concurrent-pub-consume!
+  "Runs a publisher thread and a consumer simultaneously against one queue, measuring
+  end-to-end latency under write contention. Unlike `bench-e2e-latency!`, the publisher and
+  the wait-for-receive happen on separate threads, so we exercise the actual
+  publish-while-consume code path (buffer atom contention, fetch contending with INSERT,
+  the per-channel `active-handlers` gate).
+
+  Each published message carries a `publish-ns` timestamp; the listener measures
+  `receive-ns - publish-ns` and records it as the per-message latency.
+
+  Options:
+    :duration-sec - how long to publish for (default 10)
+    :pub-rate-msgs-sec - target publish rate (default 50)
+    :queue-name   - queue to use (default :queue/bench-pub-consume)"
+  ([] (bench-concurrent-pub-consume! {}))
+  ([{:keys [duration-sec pub-rate-msgs-sec queue-name]
+     :or   {duration-sec 10 pub-rate-msgs-sec 50 queue-name :queue/bench-pub-consume}}]
+   (ensure-started!)
+   (let [received    (atom 0)
+         latencies   (atom [])
+         stop?       (atom false)
+         publisher-published (atom 0)]
+     (try
+       (listener/batch-listen! queue-name
+                               (fn [msgs]
+                                 (let [now (System/nanoTime)]
+                                   (doseq [m msgs]
+                                     (when-let [pub-ns (get m "publish-ns")]
+                                       (swap! latencies conj (/ (- now (double pub-ns)) 1e6))))
+                                   (swap! received + (count msgs))))
+                               {})
+       (let [pub-runnable (bound-fn []
+                            (let [interval-ns (long (/ 1e9 pub-rate-msgs-sec))
+                                  stop-at-ns  (+ (System/nanoTime) (* 1e9 duration-sec))]
+                              (loop []
+                                (when (and (not @stop?) (< (System/nanoTime) stop-at-ns))
+                                  (let [next-pub-ns (+ (System/nanoTime) interval-ns)]
+                                    (mq/with-queue queue-name [q]
+                                      (mq/put q {:publish-ns (System/nanoTime)}))
+                                    (swap! publisher-published inc)
+                                    (let [sleep-ns (- next-pub-ns (System/nanoTime))]
+                                      (when (pos? sleep-ns)
+                                        (Thread/sleep (long (/ sleep-ns 1e6))
+                                                      (int (mod sleep-ns 1e6)))))
+                                    (recur))))))
+             pub-thread (Thread. ^Runnable pub-runnable)
+             start (System/nanoTime)]
+         (.start pub-thread)
+         (.join pub-thread (long (* 1000 (+ duration-sec 1))))
+         (reset! stop? true)
+         ;; Drain — wait for the consumer to catch up after publishing stops.
+         (let [drain-deadline (+ (System/currentTimeMillis) 30000)]
+           (while (and (< @received @publisher-published)
+                       (< (System/currentTimeMillis) drain-deadline))
+             (polling/notify-all!)
+             (Thread/sleep 50)))
+         (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)
+               stats      (collect-stats @latencies)]
+           (print-table!
+            (str "Concurrent Pub+Consume (rate=" pub-rate-msgs-sec "/sec, duration=" duration-sec "s"
+                 (when @rtt-ms (str ", rtt=" (first @rtt-ms) "-" (last @rtt-ms) "ms"))
+                 ")")
+            ["Metric" "Value"]
+            [["Published"        (str @publisher-published)]
+             ["Received"         (str @received)]
+             ["Drain ms"         (fmt (- elapsed-ms (* 1000 duration-sec)))]
+             ["P50 latency ms"   (fmt (:p50 stats))]
+             ["P95 latency ms"   (fmt (:p95 stats))]
+             ["P99 latency ms"   (fmt (:p99 stats))]
+             ["Max latency ms"   (fmt (:max stats))]
+             ["Pub/sec (actual)" (fmt (/ @publisher-published (double duration-sec)))]
+             ["Recv/sec total"   (fmt (/ @received (/ elapsed-ms 1000.0)))]])))
+       (finally
+         (reset! stop? true)
+         (mq/unlisten! queue-name)
+         (t2/delete! :queue_message_batch :queue_name (name queue-name)))))))
+
+;;; ---------------------------------------- Run All ----------------------------------------
+
+(defn run-all-benchmarks!
+  "Runs all benchmark scenarios with default parameters."
+  []
+  (println "=== MQ Performance Benchmarks ===")
+  (println)
+
+  (println "--- 1/5: Publishing Throughput (Queue) ---")
+  (bench-publish-throughput! {:n 1000 :transport :queue})
+
+  (println "--- 1b/5: Publishing Throughput (Topic) ---")
+  (bench-publish-throughput! {:n 1000 :transport :topic})
+
+  (println "--- 2/5: End-to-End Latency ---")
+  (doseq [rate [:low :medium :high :burst]]
+    (bench-e2e-latency! {:n 50 :rate rate :transport :queue}))
+  (doseq [rate [:low :medium :burst]]
+    (bench-e2e-latency! {:n 50 :rate rate :transport :topic}))
+
+  (println "--- 3/5: Sustained Throughput ---")
+  (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :queue})
+  (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :topic})
+
+  (println "--- 4/5: Batch Listener Comparison ---")
+  (bench-batch-listener! {:n 200})
+
+  (println "--- 5/7: Multiple Queues ---")
+  (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})
+
+  (println "--- 6/7: Concurrent Consumers ---")
+  (bench-concurrent-consumers! {:consumer-counts [1 2 4 8] :n 2000})
+
+  (println "--- 7/7: Stale Recovery Race ---")
+  (bench-stale-recovery-race! {:n 500 :workers 8})
+
+  (println "=== All benchmarks complete ==="))
+
+;;; ---------------------------------------- Multi-Node Coordination ----------------------------------------
+
+(defn run-all-benchmarks-coordinated!
+  "Waits for a 'go' signal on :topic/bench-coordination, then runs all benchmarks.
+  Use with multiple JVM instances to start benchmarks simultaneously.
+
+  Usage:
+    ;; On each node:
+    (start-mq!)
+    (run-all-benchmarks-coordinated!)
+
+    ;; Then from any node or separate REPL:
+    (signal-go!)"
+  []
+  (let [latch (CountDownLatch. 1)]
+    (listener/batch-listen! :topic/bench-coordination
+                            (fn [_msgs] (.countDown latch))
+                            {})
+    (println "Waiting for go signal... (call (signal-go!) from any node)")
+    (.await latch 300 TimeUnit/SECONDS)
+    (mq/unlisten! :topic/bench-coordination)
+    (println "Go signal received! Starting benchmarks...")
+    (run-all-benchmarks!)))
+
+(defn signal-go!
+  "Sends the go signal to all waiting nodes."
+  []
+  (mq/with-topic :topic/bench-coordination [t]
+    (mq/put t {:go true :time (System/currentTimeMillis)}))
+  (println "Go signal sent."))

dev/src/dev/mq_perf_multi.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# Runs multiple MQ benchmark nodes against the same database.
+#
+# Usage:
+#   ./dev/src/dev/mq_perf_multi.sh start       # Start 2 nodes (default)
+#   ./dev/src/dev/mq_perf_multi.sh start 4     # Start 4 nodes
+#   ./dev/src/dev/mq_perf_multi.sh go          # Send go signal to all waiting nodes
+#   ./dev/src/dev/mq_perf_multi.sh stop        # Stop all running nodes
+#   ./dev/src/dev/mq_perf_multi.sh logs        # Tail all node logs
+#
+# Each node starts a lightweight JVM with just DB + MQ (no server),
+# waits for a coordinated go signal, then runs all benchmarks.
+
+set -euo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+LOG_DIR="target/mq-perf-logs"
+PID_DIR="target/mq-perf-pids"
+
+cmd_start() {
+  local num_nodes="${1:-2}"
+  mkdir -p "$LOG_DIR" "$PID_DIR"
+
+  # Clean up stale pids
+  rm -f "$PID_DIR"/*.pid
+
+  echo "Starting $num_nodes MQ benchmark nodes..."
+  echo "Logs in $LOG_DIR/"
+  echo
+
+  for i in $(seq 1 "$num_nodes"); do
+    local log_file="$LOG_DIR/node-$i.log"
+    clojure -M:dev:ee:ee-dev -e "
+      (require 'dev.mq-perf)
+      (dev.mq-perf/start-mq!)
+      (dev.mq-perf/run-all-benchmarks-coordinated!)
+    " > "$log_file" 2>&1 &
+    local pid=$!
+    echo "$pid" > "$PID_DIR/node-$i.pid"
+    echo "  Node $i started (PID $pid, log: $log_file)"
+  done
+
+  echo
+  echo "Nodes are initializing. Once ready, run:"
+  echo "  $0 go"
+}
+
+cmd_go() {
+  echo "Sending go signal..."
+  clojure -M:dev:ee:ee-dev -e "
+    (require 'dev.mq-perf)
+    (dev.mq-perf/start-mq!)
+    (dev.mq-perf/signal-go!)
+  "
+  echo
+  echo "Benchmarks started. Watch progress with:"
+  echo "  $0 logs"
+}
+
+cmd_stop() {
+  if [ ! -d "$PID_DIR" ]; then
+    echo "No nodes running."
+    return
+  fi
+
+  for pid_file in "$PID_DIR"/*.pid; do
+    [ -f "$pid_file" ] || continue
+    local pid
+    pid=$(cat "$pid_file")
+    local name
+    name=$(basename "$pid_file" .pid)
+    if kill -0 "$pid" 2>/dev/null; then
+      kill "$pid"
+      echo "Stopped $name (PID $pid)"
+    else
+      echo "$name (PID $pid) already exited"
+    fi
+    rm -f "$pid_file"
+  done
+}
+
+cmd_logs() {
+  if [ ! -d "$LOG_DIR" ]; then
+    echo "No logs found."
+    return
+  fi
+  tail -f "$LOG_DIR"/node-*.log
+}
+
+case "${1:-}" in
+  start) cmd_start "${2:-2}" ;;
+  go)    cmd_go ;;
+  stop)  cmd_stop ;;
+  logs)  cmd_logs ;;
+  *)
+    echo "Usage: $0 {start [N]|go|stop|logs}"
+    echo
+    echo "  start [N]  Start N benchmark nodes (default 2)"
+    echo "  go         Send go signal to start benchmarks"
+    echo "  stop       Kill all running nodes"
+    echo "  logs       Tail all node logs"
+    exit 1
+    ;;
+esac

mage/src/mage/modules.clj

@@ -123,6 +123,7 @@
      login-history
      mcp
      metabot
+     mq
      notification
      oauth-server
      permissions

resources/migrations/062/20260512_persistent_queue.yaml

@@ -0,0 +1,142 @@
+## Adds support for appdb persistent queue backend
+
+databaseChangeLog:
+  - changeSet:
+      id: v62.asdgew
+      author: nvoxland
+      comment: Create the table backing the message queue
+      changes:
+        - createTable:
+            tableName: queue_message_batch
+            remarks: Lightweight message queue for cross-cluster communication
+            columns:
+              - column:
+                  name: id
+                  type: BIGINT
+                  autoIncrement: true
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: queue_name
+                  type: varchar(254)
+                  remarks: The name of queue this message is in
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: The timestamp of when the messages were created
+                  name: created_at
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: The status of the messages
+                  name: status
+                  type: varchar(10)
+                  defaultValue: 'pending'
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: Heartbeat to ensure status is not stuck
+                  name: status_heartbeat
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: Number of failures processing these messages
+                  name: failures
+                  type: int
+                  defaultValueNumeric: 0
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: Current owner of the messages
+                  name: owner
+                  type: varchar(36)
+              - column:
+                  remarks: JSON-encoded array of messages
+                  name: messages
+                  type: ${text.type}
+                  constraints:
+                    nullable: false
+
+  - changeSet:
+      id: v62.gdasfg2e
+      author: nvoxland
+      comment: Index queue_message_batch for efficient fetch
+      changes:
+        - createIndex:
+            tableName: queue_message_batch
+            indexName: idx_queue_message_fetch
+            columns:
+              - column:
+                  name: queue_name
+              - column:
+                  name: status
+              - column:
+                  name: id
+
+
+  - changeSet:
+      id: v62.g9dasg4
+      author: nvoxland
+      comment: Create topic_message_batch table for pub/sub messaging
+      changes:
+        - createTable:
+            tableName: topic_message_batch
+            remarks: Append-only message log for pub/sub fan-out messaging
+            columns:
+              - column:
+                  name: id
+                  type: BIGINT
+                  autoIncrement: true
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: topic_name
+                  type: varchar(254)
+                  remarks: The name of the topic this message was published to
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: The timestamp of when the message was created
+                  name: created_at
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: JSON-encoded array of messages
+                  name: messages
+                  type: ${text.type}
+                  constraints:
+                    nullable: false
+
+  - changeSet:
+      id: v62.igoiead
+      author: nvoxland
+      comment: Index topic_message_batch by topic_name and id for efficient polling
+      changes:
+        - createIndex:
+            tableName: topic_message_batch
+            indexName: idx_topic_message_batch_topic_id
+            columns:
+              - column:
+                  name: topic_name
+              - column:
+                  name: id
+
+  - changeSet:
+      id: v62.8867df
+      author: nvoxland
+      comment: Index topic_message_batch.created_at for efficient cleanup queries
+      changes:
+        - createIndex:
+            tableName: topic_message_batch
+            indexName: idx_topic_message_batch_created_at
+            columns:
+              - column:
+                  name: created_at

src/metabase/analytics/prometheus.clj

@@ -645,6 +645,54 @@
                           ;; 1KB -> 5MB
                           :buckets [1000 5000 10000 50000 100000 500000 1000000 5000000]})
 
+   ;; messaging metrics
+   (prometheus/counter :metabase-mq/appdb-cleanup-deleted
+                       {:description "Messages/batches deleted by cleanup."
+                        :labels [:transport :channel]})
+   (prometheus/gauge :metabase-mq/appdb-queue-depth
+                     {:description "Batch count per queue by status."
+                      :labels [:channel :status]})
+   (prometheus/counter :metabase-mq/appdb-queue-poll-results
+                       {:description "Queue poll results by outcome."
+                        :labels [:result]})
+   (prometheus/gauge :metabase-mq/appdb-topic-subscriber-lag
+                     {:description "Undelivered messages between subscriber offset and latest."
+                      :labels [:channel]})
+   (prometheus/counter :metabase-mq/batch-stale-recoveries
+                       {:description "Batches recovered from stale processing state."
+                        :labels [:transport :channel]})
+   (prometheus/counter :metabase-mq/batches-handled
+                       {:description "Batches handled by status."
+                        :labels [:transport :channel :status]})
+   (prometheus/counter :metabase-mq/dedup-messages-dropped
+                       {:description "Messages dropped by dedup before publishing."
+                        :labels [:channel]})
+   (prometheus/histogram :metabase-mq/handle-duration-ms
+                         {:description "Duration in milliseconds to process a batch."
+                          :labels [:transport :channel]
+                          :buckets [1 5 10 50 100 500 1000 5000 10000 30000]})
+   (prometheus/counter :metabase-mq/messages-published
+                       {:description "Total messages published."
+                        :labels [:transport :channel]})
+   (prometheus/gauge :metabase-mq/publish-buffer-depth
+                     {:description "Messages sitting in the publish buffer awaiting flush."
+                      :labels [:channel]})
+   (prometheus/counter :metabase-mq/publish-buffer-flush-errors
+                       {:description "Publish buffer flush failures where messages were re-buffered for retry."
+                        :labels [:channel]})
+   (prometheus/counter :metabase-mq/queue-batch-permanent-failures
+                       {:description "Queue batches that exhausted retries."
+                        :labels [:channel]})
+   (prometheus/counter :metabase-mq/queue-batch-retries
+                       {:description "Queue batches retried after transient failure."
+                        :labels [:channel]})
+   (prometheus/counter :metabase-mq/messages-received
+                       {:description "Individual messages delivered to handlers."
+                        :labels [:transport :channel]})
+   (prometheus/counter :metabase-mq/handler-errors
+                       {:description "Errors thrown by queue or topic message handlers."
+                        :labels [:transport :channel]})
+
    ;; release dashboard metrics
    (prometheus/counter :metabase-sync/failures
                        {:description "Number of sync operation failures."

src/metabase/app_db/connection.clj

@@ -134,23 +134,66 @@
 
 (def ^:private ^:dynamic *transaction-depth* 0)
 
+(def ^:dynamic *after-commit*
+  "When non-nil, an atom containing a vector of 0-arity fns to call after the outermost transaction commits.
+  Bound to a fresh atom at the outermost transaction boundary."
+  nil)
+
+(def ^:dynamic *transaction-state*
+  "When non-nil, an atom containing a map for arbitrary per-transaction data.
+  Any subsystem can store namespaced keys here. Bound to a fresh atom at the outermost transaction boundary."
+  nil)
+
 (defn in-transaction?
   "Whether we are currently in a transaction."
   []
   (pos? *transaction-depth*))
 
+(defn after-commit!
+  "Register `f` (a 0-arity fn) to run after the current outermost transaction commits.
+  Returns true if registered, false if not currently in a transaction."
+  [f]
+  (if *after-commit*
+    (do (swap! *after-commit* conj f) true)
+    false))
+
+(defn transaction-state
+  "Returns the current transaction state atom, or nil if not in a transaction."
+  []
+  *transaction-state*)
+
 (defn- do-transaction [^java.sql.Connection connection f]
   (letfn [(thunk []
-            (let [savepoint (.setSavepoint connection)]
+            (let [savepoint          (.setSavepoint connection)
+                  commit-snapshot    (when (and *after-commit* (> *transaction-depth* 1))
+                                       @*after-commit*)
+                  state-snapshot     (when (and *transaction-state* (> *transaction-depth* 1))
+                                       @*transaction-state*)]
               (try
                 (let [result (f connection)]
                   (when (= *transaction-depth* 1)
                     ;; top-level transaction, commit
-                    (.commit connection))
+                    (.commit connection)
+                    ;; drain and execute after-commit callbacks
+                    (loop []
+                      (when-let [callbacks (seq (first (reset-vals! *after-commit* [])))]
+                        (doseq [cb callbacks]
+                          (try
+                            (cb)
+                            (catch Throwable t
+                              (log/error t "Error in after-commit callback"))))
+                        ;; If callbacks registered new callbacks, drain those too
+                        (recur))))
                   result)
                 (catch Throwable txn-e
                   (try
                     (.rollback connection savepoint)
+                    ;; Restore after-commit and transaction-state to pre-savepoint values
+                    ;; so callbacks/state from the rolled-back savepoint are discarded
+                    (when commit-snapshot
+                      (reset! *after-commit* commit-snapshot))
+                    (when state-snapshot
+                      (reset! *transaction-state* state-snapshot))
                     (catch Exception rollback-e
                       (throw (ex-info
                               (str "Error rolling back after previous error: " (ex-message txn-e))
@@ -202,8 +245,11 @@
                     {:options options}))
 
     :else
-    (binding [*transaction-depth* (inc *transaction-depth*)]
-      (do-transaction connection f))))
+    (let [outermost? (zero? *transaction-depth*)]
+      (binding [*transaction-depth* (inc *transaction-depth*)
+                *after-commit*      (if outermost? (atom []) *after-commit*)
+                *transaction-state* (if outermost? (atom {}) *transaction-state*)]
+        (do-transaction connection f)))))
 
 (methodical/defmethod t2.pipeline/transduce-query :before :default
   "Make sure application database calls are not done inside core.async dispatch pool threads. This is done relatively

src/metabase/app_db/core.clj

@@ -37,7 +37,9 @@
   db-type
   in-transaction?
   quoting-style
-  unique-identifier]
+  unique-identifier
+  after-commit!
+  transaction-state]
 
  [mdb.connection-pool-setup
   recent-activity?]

src/metabase/core/core.clj

@@ -21,6 +21,7 @@
    [metabase.initialization-status.core :as init-status]
    [metabase.llm.startup :as llm.startup]
    [metabase.logger.core :as logger]
+   [metabase.mq.core :as mq]
    [metabase.notification.core :as notification]
    [metabase.permissions.core :as perms]
    [metabase.plugins.core :as plugins]
@@ -105,6 +106,7 @@
   []
   (log/info "Metabase Shutting Down ...")
   (queue/stop-listeners!)
+  (startup/run-shutdown-logic!)
   (task/stop-scheduler!)
   (server/stop-web-server!)
   (tracing/shutdown!)
@@ -186,6 +188,9 @@
   ;; and the test suite can take 2x longer. this is really unfortunate because it could lead to some false
   ;; negatives, but for now there's not much we can do
   (mdb/setup-db! :create-sample-content? (not config/is-test?))
+  ;; Capture topic starting offsets as early as possible so messages published during the rest of
+  ;; startup are delivered once polling begins.
+  (mq/start-receiving!)
   ;; In OSS, convert any Data Analysts group with members to a normal visible group
   (perms/sync-data-analyst-group-for-oss!)
   ;; Disable read-only mode if its on during startup.

src/metabase/core/init.clj

@@ -43,6 +43,7 @@
    [metabase.metabot.init]
    [metabase.model-persistence.init]
    [metabase.models.init]
+   [metabase.mq.init]
    [metabase.notification.init]
    [metabase.oauth-server.init]
    [metabase.parameters.init]

src/metabase/mq/README.md

@@ -0,0 +1,92 @@
+# metabase.mq
+
+A persistent message queue and pub/sub topic system for Metabase. Supports both durable (database-backed) and in-memory
+transports with a unified API.
+
+## Quick Start
+
+```clojure
+;; Register a listener
+(mq/def-listener! :queue/my-task
+  [msg]
+  (do-work! msg))
+
+;; Publish a message (inside a transaction — delivers after commit)
+(t2/with-transaction [_]
+  (mq/with-queue :queue/my-task [q]
+    (mq/put q {:key "value"})))
+
+;; Publish to a topic (fan-out, at-most-once)
+(mq/with-topic :topic/settings-changed [t]
+  (mq/put t {:setting "site-name"}))
+```
+
+## Publishing Semantics
+
+`with-queue` and `with-topic` are designed to read like a direct publish — you write a message and move on. Under the hood, delivery is intentionally deferred through several layers:
+
+1. **Macro body must succeed.** Messages are only enqueued if the body of `with-queue` / `with-topic` returns normally. An exception discards them.
+
+2. **Surrounding transaction must commit.** If the call is inside a `t2/with-transaction` block (directly or transitively), messages are held until the transaction commits. A rollback discards them. This means a message will never be delivered for a database change that didn't happen.
+
+3. **Batching window.** After the transaction commits, messages enter a time-windowed buffer. Rapid-fire publishes to the same channel within a sliding window are coalesced into a single batch before hitting the transport layer. This amortizes per-batch overhead during bursts.
+
+4. **Deduplication.** Before dispatch, duplicate messages in the same batch are removed.
+
+## Queues vs Topics
+
+|               | Queue                                       | Topic                             |
+|---------------|---------------------------------------------|-----------------------------------|
+| Delivery      | At-least-once per listener                  | At-least-once per active node     |
+| Retry         | Yes — up to `queue-max-retries` (default 5) | No                                |
+| Backing table | `queue_message_batch`                       | `topic_message_batch`             |
+| Backlog       | Persists until processed or exhausted       | Cleaned up after 1 hour           |
+| Use when      | Work must complete reliably                 | All nodes must react to something |
+
+## Architecture
+
+```
+User Code
+  with-queue / with-topic
+        │
+        ▼
+Publish Pipeline (mq.publish)
+  collect inside tx → batch 100ms window → deduplicate
+        │
+        ▼
+Transport Dispatch (mq.transport)   ← multimethod on channel namespace
+  :queue → queue backends
+  :topic → topic backends
+        │
+        ▼
+Backends
+  appdb  — FOR UPDATE SKIP LOCKED (queue), offset polling (topic)
+  memory — LinkedBlockingQueue per channel
+        │
+        ▼
+Delivery Engine (mq.impl)
+  worker pool → handle! → listener fn
+```
+### Idempotency
+
+Queue listeners must be idempotent. A batch can return to `pending` after a partial failure, and the same messages will
+be redelivered. Use `dedup-fn` to filter already-processed messages on retry:
+
+```clojure
+(mq/def-listener! :queue/my-task
+  {:dedup-fn (fn [msgs] (remove already-processed? msgs))}
+  [msg]
+  (process! msg))
+```
+
+## Testing
+
+Use `with-test-mq` from `metabase.mq.test` to run tests with an isolated in-memory backend. It supports double-delivery
+chaos testing to verify listener idempotency.
+
+```clojure
+(deftest my-test
+  (mq.test/with-test-mq
+    (mq/with-queue :queue/my-task {:id 1})
+    (is (= 1 @processed-count))))
+```

src/metabase/mq/core.clj

@@ -0,0 +1,93 @@
+(ns metabase.mq.core
+  "Unified public API for the Metabase messaging subsystem.
+
+  Two patterns are supported:
+
+  **Queue** — single-consumer, at-least-once delivery.  Each message is processed by exactly
+  one listener and removed after successful processing.  Failed messages are retried up to a
+  configurable limit.
+
+  **Topic** — single-consumer pub/sub.  Each active node receives every published message.
+  There can be up to one listener per node for each topic. Failed messages are never retried.
+
+  Listeners for both types are declared with `def-listener!`. The macro adds the listener
+  to a registry; `mq.init/start!` later activates everything in that registry at the right
+  point in startup. The listener body always receives a vec of messages — use
+  `(doseq [m messages] ...)` for per-message processing.
+
+      (mq/def-listener! :queue/simple-task [messages]
+        (doseq [msg messages] (process msg)))
+
+      (mq/def-listener! :topic/my-events [messages]
+        (doseq [msg messages] (handle-event msg)))
+
+  Optional config controls batch size, exclusivity, and dedup:
+       (mq/def-listener! :queue/my-task
+         {:max-batch-messages 10 :exclusive true}
+         [messages]
+         (process-batch messages))
+
+  See [[metabase.mq.listener/def-listener!]] for the full set of options and the rationale
+  behind the deferred-registration design.
+
+  Publishing is done using `with-queue` or `with-topic` which binds a queue you can put messages on.
+  When the body finishes successfully, the message(s) in the bound queue actually publishes the messages.
+
+      (with-queue :queue/simple-task [q]
+        (put q message))
+
+      (with-topic :topic/my-events [t]
+        (put t payload))"
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.init :as mq.init]
+   [metabase.mq.listener :as mq.listener]
+   [metabase.mq.publish :as mq.publish]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.impl :as q.impl]
+   [metabase.mq.queue.memory :as q.memory]
+   [metabase.mq.queue.transport-impl :as q.transport-impl]
+   [metabase.mq.topic.appdb :as topic.appdb]
+   [metabase.mq.topic.impl :as topic.impl]
+   [metabase.mq.topic.memory :as topic.memory]
+   [metabase.mq.topic.transport-impl :as topic.transport-impl]
+   [potemkin :as p]))
+
+(set! *warn-on-reflection* true)
+
+(comment
+  mq.impl/keep-me
+  mq.init/keep-me
+  mq.listener/keep-me
+  mq.publish/keep-me
+  q.appdb/keep-me
+  q.impl/keep-me
+  q.memory/keep-me
+  q.transport-impl/keep-me
+  topic.appdb/keep-me
+  topic.impl/keep-me
+  topic.memory/keep-me
+  topic.transport-impl/keep-me)
+
+(p/import-vars
+ [mq.listener
+  def-listener!
+  register-listeners!
+  unlisten!]
+
+ [mq.publish
+  put]
+
+ [mq.impl
+  last-activity]
+
+ [mq.init
+  start!
+  start-receiving!
+  stop!]
+
+ [q.impl
+  with-queue]
+
+ [topic.impl
+  with-topic])

src/metabase/mq/impl.clj

@@ -0,0 +1,197 @@
+(ns metabase.mq.impl
+  "Backend coordination: message delivery, analytics helpers, and lifecycle management."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.transport :as transport]
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Callable ExecutorService Executors Future)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private active-handlers
+  "channel keyword → {:future Future, :metadata map}
+   Tracks which channels have an active handler running across all backends."
+  (atom {}))
+
+(def ^:private worker-pool
+  "Shared thread pool for processing messages off backend poll threads."
+  (atom nil))
+
+(def ^:private last-activity*
+  "channel keyword → nanoTime of last publish or handler completion."
+  (atom {}))
+
+(defn last-activity
+  "Returns the nanoTime of the last publish or handler completion for the given channel, or nil."
+  [channel]
+  (get @last-activity* channel))
+
+(defn record-publish-activity!
+  "Records a publish event for the given channel. Called by the publishing pipeline."
+  [channel]
+  (swap! last-activity* assoc channel (System/nanoTime)))
+
+(defn channel-busy?
+  "Returns true if the given channel has an active handler running."
+  [channel]
+  (contains? @active-handlers channel))
+
+(defn busy-channels
+  "Returns the set of channels that currently have an active handler."
+  []
+  (set (keys @active-handlers)))
+
+(defn active-handler-metadata
+  "Returns the metadata map for the active handler on the given channel, or nil.
+   Backends store backend-specific data here (e.g. queue backends store :batch-id for heartbeats)."
+  [channel]
+  (get-in @active-handlers [channel :metadata]))
+
+(defn invoke-listener!
+  "Common listener invocation skeleton for both queues and topics.
+  Looks up listener via `listener-fn`, times execution, logs errors,
+  and records metrics. Calls `on-success` / `on-error` for system-specific
+  side effects (e.g. queue ACK/NACK)."
+  [{:keys [channel listener-fn invoke-fn on-success on-error]}]
+  (let [transport (namespace channel)
+        listener    (listener-fn)
+        labels      {:transport transport :channel (name channel)}
+        start       (System/nanoTime)]
+    (try
+      (if-not listener
+        (log/debugf "No listener registered for %s %s, skipping message" transport (name channel))
+        (do
+          (invoke-fn listener)
+          (when on-success (on-success))
+          (analytics/inc! :metabase-mq/batches-handled (assoc labels :status "success"))))
+      (catch Exception e
+        (log/error e (str "Error handling " transport " message") labels)
+        (when on-error (on-error e))
+        (analytics/inc! :metabase-mq/batches-handled (assoc labels :status "error")))
+      (finally
+        (analytics/observe! :metabase-mq/handle-duration-ms labels
+                            (/ (double (- (System/nanoTime) start)) 1e6))))))
+
+(defn handle!
+  "Handles accumulated messages for a channel by invoking the registered listener.
+   Processes each message/batch with error isolation — one failure doesn't block others.
+   For queues, batches use all-or-nothing ACK semantics: if ANY message in the batch
+   fails, the ENTIRE batch is nacked and will be retried. This means successfully
+   processed messages may be re-delivered. Queue listeners MUST be idempotent.
+   The :dedup-fn option on listeners helps mitigate duplicate processing on retry."
+  [channel message-batches messages]
+  (swap! last-activity* assoc channel (System/nanoTime))
+  (let [{:keys [max-batch-messages]} (listener/get-listener channel)
+        batch-size                   (or max-batch-messages 1)
+        transport                    (namespace channel)
+        labels                       {:transport transport :channel (name channel)}]
+    (analytics/inc! :metabase-mq/messages-received labels (count messages))
+    (invoke-listener!
+     {:channel      channel
+      :listener-fn  #(:listener (listener/get-listener channel))
+      :invoke-fn    (fn [h]
+                      (let [error (atom nil)]
+                        (doseq [batch (partition-all batch-size messages)]
+                          (try (h (vec batch))
+                               (catch Exception e
+                                 (analytics/inc! :metabase-mq/handler-errors labels)
+                                 (reset! error e)
+                                 (log/errorf e "Error handling %s message for %s, continuing"
+                                             transport (name channel)))))
+                        (when-let [e @error]
+                          (throw (ex-info "One or more messages failed" {} e)))))
+      :on-success   #(doseq [[bid backend] message-batches]
+                       (q.backend/batch-successful! backend channel bid))
+      :on-error     (fn [_e]
+                      (doseq [[bid backend] message-batches]
+                        (q.backend/batch-failed! backend channel bid)))})))
+
+(defn deliver!
+  "Called by backends when messages are ready for delivery.
+   Passes messages directly to handle! for processing.
+   `batch-id` and `backend` are optional — queues pass them for ACK/NACK, topics pass nil."
+  [channel messages batch-id backend]
+  (handle! channel (if batch-id {batch-id backend} {}) messages))
+
+(defn submit-delivery!
+  "Submits a delivery to the shared worker pool for non-blocking processing.
+   Returns true if submitted, false if the channel already has an active handler.
+   `metadata` is an opaque map stored alongside the future — backends can use it for
+   backend-specific tracking (e.g. queues store {:batch-id id} for heartbeat updates).
+   Uses bound-fn to convey dynamic bindings to the worker thread.
+   The busy-check and registration are atomic via a single swap!.
+   A generation counter prevents a completed future's cleanup from clobbering a
+   re-submission that won the slot between the finally and the dissoc."
+  [channel messages batch-id backend metadata]
+  (let [gen     (Object.)
+        [old _] (swap-vals! active-handlers
+                            (fn [handlers]
+                              (if (contains? handlers channel)
+                                handlers
+                                (assoc handlers channel {:future nil :metadata metadata :gen gen}))))]
+    (if (contains? old channel)
+      false
+      (let [^Callable task (bound-fn []
+                             (try
+                               (deliver! channel messages batch-id backend)
+                               (finally
+                                 ;; Only remove if this generation still owns the slot
+                                 (swap! active-handlers
+                                        (fn [handlers]
+                                          (if (identical? gen (:gen (get handlers channel)))
+                                            (dissoc handlers channel)
+                                            handlers)))
+                                 (mq.polling/notify-all!))))
+            f    (.submit ^ExecutorService @worker-pool task)]
+        ;; Only set the future if this generation still owns the slot
+        (swap! active-handlers
+               (fn [handlers]
+                 (if (identical? gen (:gen (get handlers channel)))
+                   (assoc-in handlers [channel :future] f)
+                   handlers)))
+        true))))
+
+(defn start-worker-pool!
+  "Starts the shared worker pool for non-blocking message delivery. Idempotent.
+  Returns `true` if THIS call created the pool, `false` if one was already running."
+  []
+  (let [pool (Executors/newCachedThreadPool)]
+    (if (compare-and-set! worker-pool nil pool)
+      true
+      (do (.shutdown ^ExecutorService pool) false))))
+
+(defn shutdown-worker-pool!
+  "Cancels all active handler futures and shuts down the worker pool."
+  []
+  (doseq [[_ {:keys [^Future future]}] @active-handlers]
+    (when future
+      (.cancel future true)))
+  (reset! active-handlers {})
+  (when-let [^ExecutorService pool @worker-pool]
+    (.shutdownNow pool)
+    (reset! worker-pool nil)))
+
+(defn start-transports
+  "Starts the queue/topic backends."
+  []
+  (transport/start! :queue)
+  (transport/start! :topic))
+
+(defn shutdown-transports
+  "Shuts down all queue/topic backends."
+  []
+  (transport/shutdown! :queue)
+  (transport/shutdown! :topic))
+
+(defn shutdown!
+  "Shuts down all mq infrastructure: publish buffer, worker pool, backends, and listener state."
+  []
+  (publish-buffer/stop-publish-buffer-flush!)
+  (shutdown-transports)
+  (shutdown-worker-pool!)
+  (reset! listener/*listeners* {}))

src/metabase/mq/init.clj

@@ -0,0 +1,153 @@
+(ns metabase.mq.init
+  "Initializes the mq subsystem at startup."
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.memory :as q.memory]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.mq.topic.appdb :as topic.appdb]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.mq.topic.memory :as topic.memory]
+   [metabase.startup.core :as startup]
+   [metabase.util.log :as log]))
+
+(def ^:private queue-backends
+  {:queue.backend/appdb  q.appdb/backend
+   :queue.backend/memory q.memory/backend})
+
+(def ^:private topic-backends
+  {:topic.backend/appdb  topic.appdb/backend
+   :topic.backend/memory topic.memory/backend})
+
+(def ^:private valid-queue-backends (set (keys queue-backends)))
+(def ^:private valid-topic-backends (set (keys topic-backends)))
+
+(defn- resolve-backend [label table kw-or-instance]
+  (if (keyword? kw-or-instance)
+    (or (get table kw-or-instance)
+        (throw (ex-info (str "Unknown " label " backend: " kw-or-instance)
+                        {:backend kw-or-instance :valid (set (keys table))})))
+    kw-or-instance))
+
+(defn start!
+  "Initializes the MQ subsystem with the given backends. Returns a handle that must be
+   passed to [[stop!]] to shut down and restore the previous backend state.
+
+   `queue-be` and `topic-be` may be either keyword identifiers (e.g. `:queue.backend/appdb`)
+   or concrete backend instances.
+
+   Called by [[startup/def-startup-logic!]] with production backends (handle discarded;
+   production shutdown uses [[startup/def-shutdown-logic!]] instead) and by
+   `metabase.mq.test-util/with-test-mq` with isolated memory backends.
+
+   Backends are set globally via `alter-var-root`. For test isolation, bind
+   `listener/*listeners*` and `publish-buffer/*publish-buffer*` with fresh atoms.
+
+   The returned handle records whether THIS call started the global worker pool and
+   publish-buffer flush executor (`:owns-worker-pool?`, `:owns-buffer-flush?`). The
+   matching [[stop!]] only shuts those down if this call started them — so a test
+   fixture running against an already-live server can no-op the global teardown."
+  [queue-be topic-be]
+  (let [queue-instance     (resolve-backend "queue" queue-backends queue-be)
+        topic-instance     (resolve-backend "topic" topic-backends topic-be)
+        prev-queue-be      q.backend/*backend*
+        prev-topic-be      topic.backend/*backend*]
+    (alter-var-root #'q.backend/*backend* (constantly queue-instance))
+    (alter-var-root #'topic.backend/*backend* (constantly topic-instance))
+    (log/infof "Queue backend: %s, Topic backend: %s" queue-be topic-be)
+    (listener/register-listeners!)
+    (let [owns-buffer-flush? (publish-buffer/start-publish-buffer-flush!)
+          owns-worker-pool?  (mq.impl/start-worker-pool!)]
+      (mq.impl/start-transports)
+      {:prev-queue-be      prev-queue-be
+       :prev-topic-be      prev-topic-be
+       :queue-be           queue-instance
+       :topic-be           topic-instance
+       :owns-buffer-flush? owns-buffer-flush?
+       :owns-worker-pool?  owns-worker-pool?})))
+
+(defn stop!
+  "Shuts down the MQ subsystem.
+
+   Called with no arguments (production): shuts down all services using the current backend vars.
+
+   Called with a handle (the map returned by [[start!]]): additionally restores the previous
+   backend state and only shuts down globals (worker pool, publish-buffer flush) if this call
+   owned them. Used by `metabase.mq.test-util/with-test-mq` for test isolation — a test
+   that borrows an already-running server's globals leaves them alone on teardown."
+  ([]
+   (publish-buffer/stop-publish-buffer-flush!)
+   (when-let [qb q.backend/*backend*] (q.backend/shutdown! qb))
+   (when-let [tb topic.backend/*backend*] (topic.backend/shutdown! tb))
+   (mq.impl/shutdown-worker-pool!)
+   (reset! listener/*listeners* {}))
+  ([{:keys [prev-queue-be prev-topic-be queue-be topic-be
+            owns-buffer-flush? owns-worker-pool?]}]
+   ;; Always force-drain the (currently bound) publish buffer so test fixtures that bound a
+   ;; fresh `*publish-buffer*` don't lose buffered messages when their teardown runs without
+   ;; owning the global flush executor. The executor itself is only shut down if THIS call
+   ;; started it.
+   (publish-buffer/flush-publish-buffer! true)
+   (when owns-buffer-flush? (publish-buffer/stop-publish-buffer-flush!))
+   (when queue-be (q.backend/shutdown! queue-be))
+   (when topic-be (topic.backend/shutdown! topic-be))
+   (when owns-worker-pool? (mq.impl/shutdown-worker-pool!))
+   (reset! listener/*listeners* {})
+   (alter-var-root #'q.backend/*backend* (constantly prev-queue-be))
+   (alter-var-root #'topic.backend/*backend* (constantly prev-topic-be))))
+
+(defn- resolve-topic-be []
+  (let [topic-be (keyword "topic.backend" (mq.settings/topic-backend))]
+    (when-not (contains? valid-topic-backends topic-be)
+      (throw (ex-info (str "Invalid topic backend: " topic-be
+                           ". Valid backends: " valid-topic-backends)
+                      {:backend topic-be :valid valid-topic-backends})))
+    topic-be))
+
+(defn- resolve-queue-be []
+  (let [queue-be (keyword "queue.backend" (mq.settings/queue-backend))]
+    (when-not (contains? valid-queue-backends queue-be)
+      (throw (ex-info (str "Invalid queue backend: " queue-be
+                           ". Valid backends: " valid-queue-backends)
+                      {:backend queue-be :valid valid-queue-backends})))
+    queue-be))
+
+(defn start-receiving!
+  "Tells the configured backends to start retaining messages for every channel that has a
+   registered listener. Handlers will not run until [[start!]] later spins up the polling /
+   consumer threads — but from this point on, no published message will be missed.
+
+   Listeners are discovered via the [[listener/def-listener*]] multimethod, whose methods are
+   registered as a side effect of namespace loading — by the time `metabase.core.core/init!*`
+   runs, the `metabase.core.init` require chain has loaded them all.
+
+   For queues this is a no-op (the queue table retains rows regardless of consumer state).
+   For topics it pins the read position / registers the subscription with the broker via the
+   `TopicBackend/start-receiving!` protocol method, which is idempotent.
+
+   Called explicitly from `metabase.core.core/init!*` right after the app DB is set up, so
+   the rest of startup can publish messages without losing them."
+  []
+  (let [topic-be       (resolve-topic-be)
+        topic-instance (resolve-backend "topic" topic-backends topic-be)
+        topic-channels (filter #(= "topic" (namespace %))
+                               (keys (methods listener/def-listener*)))]
+    (alter-var-root #'topic.backend/*backend* (constantly topic-instance))
+    (doseq [ch topic-channels]
+      (topic.backend/start-receiving! topic-instance ch))
+    (log/infof "Started receiving on %d topic(s)" (count topic-channels))))
+
+(defmethod startup/def-startup-logic! ::MqStart [_]
+  (start! (resolve-queue-be) (resolve-topic-be)))
+
+;; Run after all other startup logic so listener registration, transports, and polling threads
+;; only come up once the rest of the system is ready. Topic starting offsets are pinned earlier
+;; by [[start-receiving!]], called explicitly from `metabase.core.core/init!*` right after the
+;; app DB is set up.
+(defmethod startup/startup-priority ::MqStart [_] Long/MAX_VALUE)
+
+(defmethod startup/def-shutdown-logic! ::MqShutdown [_]
+  (stop!))

src/metabase/mq/listener.clj

@@ -0,0 +1,130 @@
+(ns metabase.mq.listener
+  "Listener registry: registration, lookup, and the `def-listener!` macro."
+  (:require
+   [metabase.mq.queue.transport-impl :as q.transport-impl]
+   [metabase.mq.topic.transport-impl :as topic.transport-impl]
+   [metabase.mq.transport :as transport]
+   [metabase.util.log :as log]
+   [metabase.util.malli.registry :as mr]))
+
+(comment
+  q.transport-impl/keep-me
+  topic.transport-impl/keep-me)
+
+(set! *warn-on-reflection* true)
+
+(def default-max-batch-messages
+  "Default `:max-batch-messages` when the listener doesn't specify one.
+   Bounds both DB-row size at publish time and the consumer slice size at handle time."
+  100)
+
+(def ^:dynamic *listeners*
+  "channel → {:listener fn :max-batch-messages int ...} for all channels."
+  (atom {}))
+
+(defn queue-names
+  "Returns the seq of queue channel names currently registered in `*listeners*`."
+  []
+  (filter #(= "queue" (namespace %)) (keys @*listeners*)))
+
+(defn topic-names
+  "Returns the seq of topic channel names currently registered in `*listeners*`."
+  []
+  (filter #(= "topic" (namespace %)) (keys @*listeners*)))
+
+(defn get-listener
+  "Returns the listener config map for `channel`, or nil if not registered."
+  [channel]
+  (get @*listeners* channel))
+
+(defn register-listener!
+  "Atomically registers a listener for the given channel, throwing if one already exists."
+  [channel listener-map]
+  (let [[old _] (swap-vals! *listeners*
+                            (fn [m]
+                              (if (contains? m channel)
+                                m
+                                (assoc m channel listener-map))))]
+    (when (contains? old channel)
+      (throw (ex-info (str "Listener already registered for " (namespace channel) " " (name channel))
+                      {:channel channel})))))
+
+(defn batch-listen!
+  "Low-level listener registration. Prefer [[def-listener!]] for production code — the macro
+   is the supported way to wire a handler so it gets activated through `register-listeners!`
+   at the correct point in startup, and discovered early by `mq/start-receiving!`.
+
+   Use this directly only when you need a runtime-dynamic registration (e.g. plugins, ad-hoc
+   tests). The listener is invoked with a vec of messages, sized up to `:max-batch-messages`
+   (defaults to `default-max-batch-messages`). Queues support `{:exclusive true}`."
+  [channel listener config]
+  (let [defaults (transport/on-listen! channel config)
+        config   (merge {:max-batch-messages default-max-batch-messages} config)]
+    (register-listener! channel
+                        (merge defaults config {:listener listener}))))
+
+(defn unlisten!
+  "Removes the listener for a channel."
+  [channel]
+  (swap! *listeners* dissoc channel))
+
+(mr/def ::channel
+  [:and :keyword [:fn {:error/message "Channel must be namespaced to 'queue' or 'topic'"}
+                  #(#{"queue" "topic"} (namespace %))]])
+
+(mr/def ::listen-opts
+  [:map [:exclusive {:optional true} :boolean]])
+
+(defmulti def-listener*
+  "Multimethod backing [[def-listener!]]."
+  {:arglists '([channel])}
+  identity)
+
+(defmacro def-listener!
+  "Declares a listener for a queue or topic.
+
+   The listener body receives a vec of messages; for per-message handling write
+   `(doseq [m messages] ...)` inside the body. The keyword's namespace decides the transport:
+   `:queue/*` for at-least-once queues, `:topic/*` for fire-and-forget pub/sub.
+
+   Optional config keys:
+   - `:max-batch-messages` — slice size (defaults to [[default-max-batch-messages]]).
+   - `:exclusive` (queues only) — when true, at most one batch is in-flight cluster-wide.
+   - `:dedup-fn` — function that filters duplicates from a batch before delivery.
+
+   Examples:
+
+       (mq/def-listener! :topic/settings-cache-invalidated [messages]
+         (doseq [_ messages] (restore-cache!)))
+
+       (mq/def-listener! :queue/simple-task {:exclusive true} [messages]
+         (doseq [msg messages] (process msg)))
+
+       (mq/def-listener! :queue/search-reindex
+         {:max-batch-messages 50 :exclusive true}
+         [messages]
+         (process-batch messages))"
+  {:arglists '([channel bindings & body]
+               [channel config bindings & body])}
+  [channel & args]
+  (let [[config & args]  (if (map? (first args)) args (cons nil args))
+        [bindings & body] args]
+    `(defmethod def-listener* ~channel [~'_]
+       (batch-listen!
+        ~channel
+        (fn [~@bindings] ~@body)
+        ~(or (select-keys config [:max-batch-messages :exclusive :dedup-fn]) {})))))
+
+(defn register-listeners!
+  "Call all [[def-listener!]] implementations to register their listeners.
+   Called at startup and in test setup (from `with-test-mq`).
+   Throws on the first registration failure so broken listeners are caught early."
+  []
+  (doseq [[k f] (methods def-listener*)]
+    (try
+      (f k)
+      (catch Throwable e
+        (log/errorf e "Failed to register listener %s" k)
+        (throw (ex-info (str "Failed to register listener " k)
+                        {:channel k}
+                        e))))))

src/metabase/mq/memory.clj

@@ -0,0 +1,110 @@
+(ns metabase.mq.memory
+  "Shared in-memory message store used by the memory queue and topic backends.
+  Each `layer` is a self-contained bundle of channel state, batch-registry, and a
+  poll thread — tests can construct isolated layers with [[make-layer]] instead of
+  rebinding dynamic vars."
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as mq.polling])
+  (:import
+   (java.util ArrayList)
+   (java.util.concurrent LinkedBlockingQueue)))
+
+(set! *warn-on-reflection* true)
+
+(declare poll-once!)
+
+(defn make-layer
+  "Creates a fresh memory layer. `:channels` is a map of channel-name →
+  LinkedBlockingQueue, `:batch-registry` tracks queue retries, `:queue-backend` is
+  populated when the queue backend's `start!` runs so the poll loop knows which
+  backend instance to hand to [[mq.impl/submit-delivery!]]."
+  []
+  {:channels         (atom {})
+   :batch-registry   (atom {})
+   :channel-failures (atom {})
+   :queue-backend    (atom nil)
+   :poll-state       (mq.polling/make-poll-state)})
+
+(defonce ^{:doc "Process-wide singleton layer shared by the production memory backends."}
+  default-layer
+  (make-layer))
+
+;;; ------------------------------------------- Channel state -------------------------------------------
+
+(defn- ensure-channel!
+  "Ensures a LinkedBlockingQueue exists for the channel, creating one if needed. Returns the queue."
+  ^LinkedBlockingQueue [{:keys [channels]} channel-name]
+  (or (get @channels channel-name)
+      (let [new-q (LinkedBlockingQueue.)]
+        (-> (swap! channels (fn [chs]
+                              (if (contains? chs channel-name)
+                                chs
+                                (assoc chs channel-name new-q))))
+            (get channel-name)))))
+
+;;; ------------------------------------------- Publish / Drain -------------------------------------------
+
+(defn publish!
+  "Adds messages to the channel's queue and wakes the polling thread."
+  [{:keys [poll-state] :as layer} channel-name messages]
+  (let [^LinkedBlockingQueue q (ensure-channel! layer channel-name)]
+    (doseq [msg messages]
+      (.offer q msg)))
+  (when-not (mq.impl/channel-busy? channel-name)
+    (mq.polling/notify! poll-state)))
+
+(defn- drain!
+  "Removes and returns up to `max-elements` messages currently in the channel's queue,
+   or nil if empty. If `max-elements` is nil or non-positive, drains everything."
+  [{:keys [channels]} channel-name max-elements]
+  (when-let [^LinkedBlockingQueue q (get @channels channel-name)]
+    (when-not (.isEmpty q)
+      (let [batch (ArrayList.)]
+        (if (and max-elements (pos? (int max-elements)))
+          (.drainTo q batch (int max-elements))
+          (.drainTo q batch))
+        (when-not (.isEmpty batch)
+          (vec batch))))))
+
+(defn- register-batch!
+  "Registers a batch in the layer's batch registry for retry tracking.
+  Inherits the failure count tracked for the channel — `batch-failed!` re-publishes
+  the messages and bumps the counter, so a new batch on the same channel must see
+  the accumulated count rather than starting from zero."
+  [{:keys [batch-registry channel-failures]} batch-id channel-name messages]
+  (let [failures (get @channel-failures channel-name 0)]
+    (swap! batch-registry assoc batch-id {:messages messages :failures failures})))
+
+;;; ------------------------------------------- Polling -------------------------------------------
+
+(defn- poll-once!
+  "Drains all non-busy channels and submits messages for delivery.
+  For queue channels, generates a batch-id and passes the layer's registered
+  queue backend. For topic channels, passes nil batch-id and nil backend."
+  [{:keys [queue-backend] :as layer}]
+  (doseq [channel-name (remove mq.impl/channel-busy?
+                               (concat (listener/queue-names) (listener/topic-names)))]
+    (let [max-elements (:max-batch-messages (listener/get-listener channel-name))]
+      (when-let [messages (drain! layer channel-name max-elements)]
+        (if (= "queue" (namespace channel-name))
+          (let [batch-id (str (random-uuid))]
+            (register-batch! layer batch-id channel-name messages)
+            (mq.impl/submit-delivery! channel-name messages batch-id @queue-backend
+                                      {:batch-id batch-id}))
+          (mq.impl/submit-delivery! channel-name messages nil nil nil))))))
+
+(defn start!
+  "Starts the layer's polling thread. Idempotent — second call is a no-op."
+  [{:keys [poll-state] :as layer}]
+  (mq.polling/start-polling! poll-state "Memory" 50 #(poll-once! layer)))
+
+(defn shutdown!
+  "Stops the polling thread and clears channel state."
+  [{:keys [poll-state channels batch-registry channel-failures queue-backend]}]
+  (mq.polling/stop-polling! poll-state "Memory")
+  (reset! channels {})
+  (reset! batch-registry {})
+  (reset! channel-failures {})
+  (reset! queue-backend nil))

src/metabase/mq/polling.clj

@@ -0,0 +1,80 @@
+(ns metabase.mq.polling
+  "Shared polling infrastructure for appdb and memory mq backends:
+  time-gated execution and polling thread lifecycle."
+  (:require
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Future)))
+
+(set! *warn-on-reflection* true)
+
+(defn periodically!
+  "Runs `f` at most once per `interval-ms`. Uses `state-atom` (an atom holding a long timestamp)
+   to track the last run time. Catches and logs exceptions using `label` for context."
+  [state-atom interval-ms label f]
+  (let [now (System/currentTimeMillis)]
+    (when (> (- now @state-atom) interval-ms)
+      (reset! state-atom now)
+      (try
+        (f)
+        (catch Exception e
+          (log/error e (str "Error in " label)))))))
+
+(def ^:private active-poll-states
+  "Set of poll states that currently have a running polling thread."
+  (atom #{}))
+
+(defn make-poll-state
+  "Creates a polling state map with an atom for the background process and a notify object."
+  []
+  {:process  (atom nil)
+   :running? (atom false)
+   :notify   (Object.)})
+
+(defn start-polling!
+  "Starts a polling thread that calls `poll-fn` in a loop, waiting on the notify object
+   for `wait-ms` between iterations. If `poll-fn` returns a truthy value (indicating work
+   was found), the next iteration runs immediately without waiting. Idempotent — second
+   call is a no-op."
+  [{:keys [process running? notify] :as poll-state} label wait-ms poll-fn]
+  (when (compare-and-set! running? false true)
+    (log/infof "Starting %s polling thread" label)
+    (swap! active-poll-states conj poll-state)
+    (reset! process
+            (future
+              (try
+                (loop []
+                  (when @running?
+                    (let [found-work? (try
+                                        (poll-fn)
+                                        (catch InterruptedException e (throw e))
+                                        (catch Exception e
+                                          (log/errorf e "Error in %s polling loop" label)
+                                          false))]
+                      (when-not found-work?
+                        (locking notify (.wait ^Object notify (long wait-ms)))))
+                    (recur)))
+                (catch InterruptedException _
+                  (log/infof "%s polling thread interrupted" label)))))))
+
+(defn stop-polling!
+  "Stops a polling thread."
+  [{:keys [process running?] :as poll-state} label]
+  (swap! active-poll-states disj poll-state)
+  (reset! running? false)
+  (when-let [f @process]
+    (reset! process nil)
+    (when (instance? Future f)
+      (.cancel ^Future f true))
+    (log/infof "%s polling thread stopped" label)))
+
+(defn notify!
+  "Wakes the polling thread so it runs immediately."
+  [{:keys [notify]}]
+  (locking notify (.notifyAll ^Object notify)))
+
+(defn notify-all!
+  "Wakes all active polling threads so they run immediately."
+  []
+  (doseq [ps @active-poll-states]
+    (notify! ps)))

src/metabase/mq/publish.clj

@@ -0,0 +1,101 @@
+(ns metabase.mq.publish
+  "Publishing pipeline: buffering, transaction support, and the `with-buffer` macro."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.app-db.core :as mdb]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *defer-in-transaction?*
+  "When true (the default), messages published inside a transaction are deferred
+  until after the transaction commits. Bind to false in sync/test mode where
+  listeners must process inline."
+  true)
+
+(defprotocol MessageBuffer
+  "Protocol for buffering messages before publishing."
+  (put [this msg]
+    "Put a message on the buffer."))
+
+(defn publish!
+  "Publishes messages to a channel. Applies dedup-fn if registered, buffers,
+   publishes to the appropriate backend, and records analytics."
+  [channel messages]
+  (let [{:keys [dedup-fn]} (listener/get-listener channel)
+        before-count (count messages)
+        messages     (if dedup-fn (dedup-fn messages) messages)]
+    (when (and dedup-fn (< (count messages) before-count))
+      (analytics/inc! :metabase-mq/dedup-messages-dropped
+                      {:channel (name channel)}
+                      (- before-count (count messages))))
+    (when (seq messages)
+      (mq.impl/record-publish-activity! channel)
+      (publish-buffer/buffered-publish! channel messages)
+      (analytics/inc! :metabase-mq/messages-published
+                      {:transport (namespace channel) :channel (name channel)}
+                      (count messages)))))
+
+(defn flush-deferred-messages!
+  "Flushes all deferred messages accumulated during a transaction.
+   Called as an after-commit callback. Per-channel try/catch so one failure doesn't block others."
+  []
+  (when-let [state (mdb/transaction-state)]
+    (let [deferred (::deferred-messages @state)]
+      (doseq [[channel msgs] deferred]
+        (try
+          (publish! channel msgs)
+          (catch Exception e
+            (log/error e "Error flushing deferred messages" {:channel channel})))))))
+
+(defn defer-in-transaction!
+  "Accumulates msgs in *transaction-state* under [::deferred-messages channel].
+   Registers flush-deferred-messages! as an after-commit callback once per transaction.
+   If *defer-in-transaction?* is false (set in tests), immediately publishes"
+  [channel msgs]
+  (if *defer-in-transaction?*
+    (let [state     (mdb/transaction-state)
+          old-state (first (swap-vals! state
+                                       (fn [s]
+                                         (-> s
+                                             (update-in [::deferred-messages channel] (fnil into []) msgs)
+                                             (assoc ::flush-registered? true)))))]
+      (when-not (::flush-registered? old-state)
+        (mdb/after-commit! flush-deferred-messages!)))
+    (publish! channel msgs)))
+
+(defn run-with-buffer
+  "Runs `body-fn` with a MessageBuffer. On success, publishes collected messages —
+   deferred if inside a transaction, immediate via publish! otherwise.
+   Channels with `:sync-local-delivery true` in their listener config always publish
+   immediately, even inside a transaction.
+   On exception, discards buffered messages and rethrows."
+  [channel error-label body-fn]
+  (let [buffer     (atom [])
+        msg-buffer (reify MessageBuffer
+                     (put [_ msg] (swap! buffer conj msg)))]
+    (try
+      (let [result (body-fn msg-buffer)
+            msgs   @buffer]
+        (when (seq msgs)
+          (if (and (mdb/transaction-state)
+                   (not (:sync-local-delivery (listener/get-listener channel))))
+            (defer-in-transaction! channel msgs)
+            (publish! channel msgs)))
+        result)
+      (catch Exception e
+        (log/error e error-label)
+        (throw e)))))
+
+(defmacro with-buffer
+  "Runs the body with a MessageBuffer for the given channel.
+   Messages are buffered and only published if the body completes successfully.
+   Inside a transaction, messages are deferred until after commit."
+  [channel [binding] & body]
+  `(run-with-buffer
+    ~channel
+    (str "Error in " (namespace ~channel) " processing")
+    (fn [~binding] ~@body)))

src/metabase/mq/publish_buffer.clj

@@ -0,0 +1,140 @@
+(ns metabase.mq.publish-buffer
+  "Publish buffering: batches rapid-fire publishes into time-windowed groups
+   and flushes them on a background scheduled thread."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.transport :as transport]
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Executors ScheduledExecutorService ThreadFactory TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *publish-buffer-ms*
+  "Time window in ms to buffer rapid-fire publishes. 0 = immediate publish.
+   Resets on every new message (sliding window)."
+  100)
+
+(def ^:dynamic *publish-buffer-max-ms*
+  "Maximum time in ms since the first message before a flush is forced. 0 = no max."
+  5000)
+
+(def ^:dynamic *publish-buffer-max-retries*
+  "Maximum number of flush attempts before messages are dropped. 0 = unlimited retries."
+  10)
+
+(def ^:dynamic *publish-buffer*
+  "Global buffer for buffering publishes.
+   channel → {:messages [...] :deadline-ms long :created-ms long :retries int}"
+  (atom {}))
+
+(defn- max-batch-size [channel]
+  (let [n (:max-batch-messages (listener/get-listener channel))]
+    (when (and n (pos? n)) n)))
+
+(defn buffered-publish!
+  "Routes messages through *publish-buffer* or publishes immediately based on `*publish-buffer-ms*`.
+
+   The sync path (when `*publish-buffer-ms*` is 0) splits `messages` into chunks of
+   `:max-batch-messages` before handing them to the transport.
+
+   The buffered path appends to the buffer and, if buffering pushes the channel's queued
+   count to or past `:max-batch-messages`, trips the channel's deadline so the next tick
+   of the background flush executor drains it. The actual flush stays on the executor
+   thread — publisher threads never block on backend writes."
+  [channel messages]
+  (let [max-size (max-batch-size channel)]
+    (if (zero? *publish-buffer-ms*)
+      (if max-size
+        (doseq [chunk (partition-all max-size messages)]
+          (transport/publish! channel (vec chunk)))
+        (transport/publish! channel (vec messages)))
+      (let [[_ new] (swap-vals! *publish-buffer*
+                                (fn [buf]
+                                  (let [now (System/currentTimeMillis)]
+                                    (update buf channel
+                                            (fn [entry]
+                                              (-> (or entry {:messages [] :deadline-ms 0 :created-ms now})
+                                                  (update :messages into messages)
+                                                  (assoc :deadline-ms (+ now *publish-buffer-ms*))))))))]
+        (when (and max-size (>= (count (get-in new [channel :messages])) max-size))
+          ;; Buffer reached the per-channel max-batch limit. Trip its deadline so the next
+          ;; background flush tick drains it. Other channels' deadlines are left untouched.
+          ;; The publisher does not flush synchronously — that would block on a backend write.
+          (swap! *publish-buffer*
+                 (fn [buf] (cond-> buf (contains? buf channel) (assoc-in [channel :deadline-ms] 1)))))))))
+
+(defn flush-publish-buffer!
+  "Drains publish buffer entries past their deadline.
+   When `force?` is true (used by the graceful-shutdown path), every entry is drained
+   regardless of its deadline so no buffered messages are dropped on shutdown."
+  ([] (flush-publish-buffer! false))
+  ([force?]
+   (doseq [[channel entry] @*publish-buffer*]
+     (analytics/set-gauge! :metabase-mq/publish-buffer-depth
+                           {:channel (name channel)}
+                           (count (:messages entry))))
+   (doseq [channel (keys @*publish-buffer*)]
+     (let [drained (atom nil)]
+       (swap! *publish-buffer*
+              (fn [buf]
+                (if-let [{:keys [deadline-ms created-ms] :as entry} (get buf channel)]
+                  (let [now (System/currentTimeMillis)]
+                    (if (or force?
+                            (and (pos? deadline-ms) (>= now deadline-ms))
+                            (and (pos? *publish-buffer-max-ms*) (>= now (+ created-ms *publish-buffer-max-ms*))))
+                      (do (reset! drained entry) (dissoc buf channel))
+                      buf))
+                  buf)))
+       (when-let [{:keys [messages] :as entry} @drained]
+         (try (transport/publish! channel messages)
+              (catch Exception e
+                (let [retries (inc (:retries entry 0))]
+                  (analytics/inc! :metabase-mq/publish-buffer-flush-errors {:channel (name channel)})
+                  (if (and (pos? *publish-buffer-max-retries*) (>= retries *publish-buffer-max-retries*))
+                    (log/warnf "Dropping %d messages for %s after %d flush failures: %s"
+                               (count messages) channel retries (ex-message e))
+                    (do
+                      (log/error e "Error flushing publish buffer, re-buffering"
+                                 {:channel channel :retries retries})
+                     ;; Put messages back into the buffer for retry on next flush cycle
+                      (swap! *publish-buffer*
+                             (fn [buf]
+                               (update buf channel
+                                       (fn [existing]
+                                         (if existing
+                                           (-> existing
+                                              ;; Prepend the failed batch so retried (older) messages
+                                              ;; are published before messages that arrived during the
+                                              ;; flush — preserves publish order across retries.
+                                               (update :messages #(into (vec messages) %))
+                                               (update :retries (fnil max 0) retries))
+                                           (assoc entry
+                                                  :deadline-ms (+ (System/currentTimeMillis) *publish-buffer-ms*)
+                                                  :retries retries))))))))))))))))
+
+(defonce ^:private publish-buffer-executor (atom nil))
+
+(defn start-publish-buffer-flush!
+  "Starts a daemon thread that flushes the publish buffer every 100ms. Idempotent.
+  Returns `true` if THIS call created the executor, `false` if one was already running."
+  []
+  (let [exec (Executors/newSingleThreadScheduledExecutor
+              (reify ThreadFactory
+                (newThread [_ r]
+                  (doto (Thread. r "mq-publish-buffer-flush")
+                    (.setDaemon true)))))]
+    (if (compare-and-set! publish-buffer-executor nil exec)
+      (do (.scheduleAtFixedRate exec ^Runnable flush-publish-buffer! 100 100 TimeUnit/MILLISECONDS)
+          true)
+      (do (.shutdown exec) false))))
+
+(defn stop-publish-buffer-flush!
+  "Stops the publish buffer flush thread and force-drains all remaining entries
+   (graceful shutdown — buffered messages are flushed regardless of deadline)."
+  []
+  (when-let [^ScheduledExecutorService exec
+             (first (reset-vals! publish-buffer-executor nil))]
+    (.shutdownNow exec))
+  (flush-publish-buffer! true))

src/metabase/mq/queue/appdb.clj

@@ -0,0 +1,234 @@
+(ns metabase.mq.queue.appdb
+  "Database-backed implementation of the message queue using the application database."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.app-db.core :as mdb]
+   [metabase.models.interface :as mi]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.impl :as q.impl]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+;; Stale processing recovery
+(def ^:private stale-processing-timeout-ms
+  "Messages in 'processing' status for longer than this are considered stale and recovered."
+  (* 10 60 1000))
+
+(defn- for-update-clause
+  "Returns the FOR UPDATE clause for the current app DB.
+  Uses SKIP LOCKED on PostgreSQL and MySQL to avoid blocking on rows locked by other nodes.
+  Falls back to plain FOR UPDATE on H2."
+  []
+  (if (#{:postgres :mysql} (mdb/db-type))
+    [:update :skip-locked]
+    [:update]))
+
+;;; ------------------------------------------- Fetch -------------------------------------------
+
+(defn- fetch-one-pending!
+  "Selects and locks the oldest pending row for a single queue, marks it 'processing' with
+  the given owner, and returns the original row map. Returns nil if no pending row exists
+  or the row at the head of the queue is locked by another transaction (FOR UPDATE SKIP
+  LOCKED). Must be called inside an open transaction so the row lock survives the UPDATE."
+  [conn queue-name owner-id]
+  (when-let [row (first (t2/query conn {:select   [:*]
+                                        :from     [:queue_message_batch]
+                                        :where    [:and
+                                                   [:= :queue_name (name queue-name)]
+                                                   [:= :status "pending"]]
+                                        :order-by [[:id :asc]]
+                                        :limit    1
+                                        :for      (for-update-clause)}))]
+    (t2/query conn {:update [:queue_message_batch]
+                    :set    {:status_heartbeat (mi/now)
+                             :status           "processing"
+                             :owner            owner-id}
+                    :where  [:= :id (:id row)]})
+    row))
+
+(defn- fetch!
+  "Fetches the oldest pending row for each of the given queue names.
+  Returns a seq of maps with :batch-id, :queue, and :messages keys, or nil if no messages
+  are available.
+
+  Implemented as a per-queue loop inside one transaction rather than a single LATERAL query
+  for cross-database portability — Postgres, MySQL 8+, and H2 support LATERAL but MariaDB
+  does not. Each per-queue SELECT uses FOR UPDATE SKIP LOCKED so concurrent consumers each
+  get a different row instead of all contending on the same head-of-queue. The cost is N
+  small queries per iteration; in practice N is the number of registered queue listeners,
+  which is small."
+  [owner-id queue-names]
+  (when (seq queue-names)
+    (let [exclusive-names (q.impl/exclusive-queue-names)]
+      (t2/with-transaction [conn]
+        (let [blocked-queues  (when (seq exclusive-names)
+                                (into #{}
+                                      (map :queue_name)
+                                      (t2/query conn {:select   [:queue_name]
+                                                      :from     [:queue_message_batch]
+                                                      :where    [:and
+                                                                 [:in :queue_name (vec exclusive-names)]
+                                                                 [:= :status "processing"]]
+                                                      :group-by [:queue_name]})))
+              fetchable-names (if (seq blocked-queues)
+                                (remove #(contains? blocked-queues (name %)) queue-names)
+                                queue-names)]
+          (when (seq fetchable-names)
+            (let [batches (into []
+                                (keep (fn [qn]
+                                        (when-let [row (fetch-one-pending! conn qn owner-id)]
+                                          {:batch-id (:id row)
+                                           :queue     (keyword "queue" (:queue_name row))
+                                           :messages  (json/decode (:messages row))})))
+                                fetchable-names)]
+              (when (seq batches) batches))))))))
+
+(defn- recover-stale-processing-batches!
+  "Recovers processing batches whose heartbeat is older than the stale timeout. Returns the
+  number of batches THIS call actually transitioned (under concurrent recovery from another
+  node the guarded UPDATE rejects rows already recovered and the count drops).
+
+  Done in two bulk UPDATEs (retryable + permanent) instead of a per-row loop so a node-down
+  event with N stale batches does not cost O(N) round trips."
+  []
+  (let [max-retries (mq.settings/queue-max-retries)
+        threshold   (Timestamp/from (.minusMillis (Instant/now) stale-processing-timeout-ms))
+        candidates  (t2/select :queue_message_batch
+                               :status "processing"
+                               :status_heartbeat [:< threshold])
+        {retryable false permanent true}
+        (group-by #(>= (inc (:failures %)) max-retries) candidates)
+        bulk-update! (fn [rows new-status]
+                       (if (seq rows)
+                         (let [result (t2/query {:update :queue_message_batch
+                                                 :set    {:status           new-status
+                                                          :failures         [:+ :failures 1]
+                                                          :status_heartbeat (mi/now)
+                                                          :owner            nil}
+                                                 :where  [:and
+                                                          [:in :id (mapv :id rows)]
+                                                          [:= :status "processing"]
+                                                          [:< :status_heartbeat threshold]]})]
+                           ;; t2/query with :update returns the affected-row count; on some
+                           ;; drivers it comes back as `[n]` rather than `n`.
+                           (if (sequential? result) (first result) result))
+                         0))
+        n-retryable (bulk-update! retryable "pending")
+        n-permanent (bulk-update! permanent "failed")]
+    (when (pos? n-retryable)
+      (log/warnf "Recovered %d stale processing batch(es)" n-retryable)
+      (doseq [[queue-name rows] (group-by :queue_name retryable)]
+        (analytics/inc! :metabase-mq/batch-stale-recoveries
+                        {:transport "queue" :channel queue-name}
+                        (count rows))))
+    (when (pos? n-permanent)
+      (log/warnf "Marked %d stale processing batch(es) as failed (reached max-retries %d)"
+                 n-permanent max-retries)
+      (doseq [[queue-name rows] (group-by :queue_name permanent)]
+        (analytics/inc! :metabase-mq/queue-batch-permanent-failures
+                        {:channel queue-name}
+                        (count rows))))
+    (+ n-retryable n-permanent)))
+
+(defn- cleanup-failed-batches! []
+  (let [threshold (Timestamp/from (.minusMillis (Instant/now) (* 7 24 60 60 1000)))
+        deleted   (t2/delete! :queue_message_batch :status "failed" :status_heartbeat [:< threshold])]
+    (when (pos? deleted)
+      (log/infof "Cleaned up %d failed queue batches" deleted)
+      (analytics/inc! :metabase-mq/appdb-cleanup-deleted {:transport "queue" :channel "all"} deleted))))
+
+(defn- update-depth-gauges! []
+  (doseq [{:keys [queue_name status cnt]}
+          (t2/query {:select   [:queue_name :status [[:count :*] :cnt]]
+                     :from     [:queue_message_batch]
+                     :group-by [:queue_name :status]})]
+    (analytics/set-gauge! :metabase-mq/appdb-queue-depth {:channel queue_name :status status} cnt)))
+
+(defn- update-heartbeats! [owner-id]
+  (doseq [channel (filter #(= "queue" (namespace %)) (mq.impl/busy-channels))]
+    (when-let [{:keys [batch-id]} (mq.impl/active-handler-metadata channel)]
+      (try
+        (t2/update! :queue_message_batch
+                    {:id batch-id :owner owner-id :status "processing"}
+                    {:status_heartbeat (mi/now)})
+        (catch Exception e
+          (log/warnf e "Failed to update heartbeat for batch %d" batch-id))))))
+
+;;; ------------------------------------------- Polling -------------------------------------------
+
+(defn- poll-iteration!
+  "One iteration of the polling loop: run periodic tasks, then try to process batches.
+  Fetches one pending row per available queue and submits each for delivery.
+  Returns true if any work was found."
+  [{:keys [owner-id last-stale-check-ms last-heartbeat-ms last-cleanup-ms last-depth-gauge-ms] :as this}]
+  (mq.polling/periodically! last-stale-check-ms (* 60 1000)       "stale batch recovery"  recover-stale-processing-batches!)
+  (mq.polling/periodically! last-heartbeat-ms   (* 2 60 1000)     "heartbeat update"      #(update-heartbeats! owner-id))
+  (mq.polling/periodically! last-cleanup-ms     (* 24 60 60 1000) "failed batch cleanup"  cleanup-failed-batches!)
+  (mq.polling/periodically! last-depth-gauge-ms (* 30 1000)       "queue depth gauge"     update-depth-gauges!)
+  (boolean
+   (when-let [available-queues (seq (remove mq.impl/channel-busy? (listener/queue-names)))]
+     (when-let [batches (seq (fetch! owner-id available-queues))]
+       (doseq [{:keys [batch-id queue messages]} batches]
+         (mq.impl/submit-delivery! queue messages batch-id this {:batch-id batch-id}))
+       true))))
+
+(defrecord AppDbQueueBackend
+           [owner-id poll-state last-stale-check-ms last-heartbeat-ms last-cleanup-ms last-depth-gauge-ms]
+  q.backend/QueueBackend
+  (publish! [_this queue messages]
+    (t2/insert! :queue_message_batch
+                {:queue_name (name queue)
+                 :messages   (json/encode messages)})
+    (when-not (mq.impl/channel-busy? queue)
+      (mq.polling/notify! poll-state)))
+
+  (batch-successful! [_this _queue-name batch-id]
+    (let [deleted (t2/delete! :queue_message_batch :id batch-id :owner owner-id)]
+      (when (= 0 deleted)
+        (log/warnf "Message %d was already deleted from the queue." batch-id))))
+
+  (batch-failed! [_this queue-name batch-id]
+    (let [row     (t2/select-one :queue_message_batch :id batch-id :owner owner-id)
+          updated (when row
+                    (if (>= (inc (:failures row)) (mq.settings/queue-max-retries))
+                      (do
+                        (log/warnf "Message %d has reached max failures (%d), marking as failed" batch-id (mq.settings/queue-max-retries))
+                        (analytics/inc! :metabase-mq/queue-batch-permanent-failures {:channel (name queue-name)})
+                        (t2/update! :queue_message_batch
+                                    {:id batch-id :owner owner-id}
+                                    {:status "failed" :failures [:+ :failures 1] :status_heartbeat (mi/now) :owner nil}))
+                      (do
+                        (analytics/inc! :metabase-mq/queue-batch-retries {:channel (name queue-name)})
+                        (t2/update! :queue_message_batch
+                                    {:id batch-id :owner owner-id}
+                                    {:status "pending" :failures [:+ :failures 1] :status_heartbeat (mi/now) :owner nil}))))]
+      (when (and row (= 0 updated))
+        (log/warnf "Message %d was not found in the queue. Likely error in concurrency handling" batch-id))))
+
+  (start! [this]
+    (mq.polling/start-polling! poll-state "Queue" 5000 #(poll-iteration! this)))
+
+  (shutdown! [_this]
+    (mq.polling/stop-polling! poll-state "Queue")))
+
+(defn make-backend
+  "Constructs a fresh `AppDbQueueBackend` with its own owner-id, poll-state, and periodic-task
+   rate-limit atoms. The production [[backend]] singleton is one such instance; tests can build
+   isolated ones so their `start!`/`shutdown!` don't touch production state."
+  []
+  (->AppDbQueueBackend (str (random-uuid))
+                       (mq.polling/make-poll-state)
+                       (atom 0) (atom 0) (atom 0) (atom 0)))
+
+(def backend
+  "Singleton instance of the appdb queue backend."
+  (make-backend))

src/metabase/mq/queue/backend.clj

@@ -0,0 +1,22 @@
+(ns metabase.mq.queue.backend
+  "Backend abstraction for the message queue system.
+  Each concrete backend (appdb, memory, sync) is a record implementing `QueueBackend`.")
+
+(set! *warn-on-reflection* true)
+
+(defprotocol QueueBackend
+  "A queue backend handles durable storage, delivery, and retry semantics for queue messages."
+  (publish!          [this queue-name messages]
+    "Publishes messages to the given queue. `messages` is always a vector of messages.")
+  (batch-successful! [this queue-name batch-id]
+    "Marks a batch as successfully processed.")
+  (batch-failed!     [this queue-name batch-id]
+    "Marks a batch as failed and increments its failure count.")
+  (start!            [this]
+    "Starts the backend polling loop. Called once at init time.")
+  (shutdown!         [this]
+    "Shuts down all queue resources for this backend."))
+
+(def ^:dynamic *backend*
+  "The active `QueueBackend` instance. Set by `metabase.mq.init/start!`."
+  nil)

src/metabase/mq/queue/impl.clj

@@ -0,0 +1,36 @@
+(ns metabase.mq.queue.impl
+  "Internal implementation for the queue messaging system"
+  (:require
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish :as publish]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def :metabase.mq.queue/queue-name
+  [:and :keyword [:fn {:error/message "Queue name must be namespaced to 'queue'"}
+                  #(= "queue" (namespace %))]])
+
+(defn exclusive?
+  "Returns true if the given queue name is registered with `:exclusive true`."
+  [queue-name]
+  (:exclusive (listener/get-listener queue-name)))
+
+(defn exclusive-queue-names
+  "Returns the set of queue names (as strings) that are registered with `:exclusive true`."
+  []
+  (into #{}
+        (comp (filter (fn [[_k v]] (:exclusive v)))
+              (map (fn [[k _v]] (name k))))
+        @listener/*listeners*))
+
+(defmacro with-queue
+  "Runs the body with the ability to add messages to the given queue.
+  Messages are buffered and only published if the body completes successfully.
+  If an exception occurs, no messages are published and the exception is rethrown.
+
+  When called inside a database transaction, messages are accumulated and published
+  as a single batch after the transaction commits successfully. This prevents consumers
+  from reading uncommitted data."
+  [queue-name [queue-binding] & body]
+  `(publish/with-buffer ~queue-name [~queue-binding] ~@body))

src/metabase/mq/queue/memory.clj

@@ -0,0 +1,56 @@
+(ns metabase.mq.queue.memory
+  "In-memory queue backend. Delegates storage and polling to a shared memory layer.
+  The layer holds the batch registry for retry tracking — each `MemoryQueueBackend`
+  carries a reference to its layer so tests can construct isolated instances."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.mq.memory :as memory]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(defrecord MemoryQueueBackend [layer]
+  q.backend/QueueBackend
+
+  (publish! [_this queue-name messages]
+    (memory/publish! layer queue-name messages))
+
+  (batch-successful! [_this queue-name batch-id]
+    (swap! (:batch-registry layer) dissoc batch-id)
+    (swap! (:channel-failures layer) dissoc queue-name))
+
+  (batch-failed! [_this queue-name batch-id]
+    (let [registry      (:batch-registry layer)
+          failures-atom (:channel-failures layer)]
+      (when-let [{:keys [messages failures]} (get @registry batch-id)]
+        (swap! registry dissoc batch-id)
+        (let [new-failures (inc failures)]
+          (if (>= new-failures (mq.settings/queue-max-retries))
+            (do
+              (swap! failures-atom dissoc queue-name)
+              (log/warnf "Batch %s has reached max failures (%d), dropping" batch-id (mq.settings/queue-max-retries))
+              (analytics/inc! :metabase-mq/queue-batch-permanent-failures {:channel (name queue-name)}))
+            (do
+              (swap! failures-atom assoc queue-name new-failures)
+              (analytics/inc! :metabase-mq/queue-batch-retries {:channel (name queue-name)})
+              ;; Re-queue messages for retry — register-batch! will pick up the bumped count
+              (memory/publish! layer queue-name messages)))))))
+
+  (start! [this]
+    (reset! (:queue-backend layer) this)
+    (memory/start! layer))
+
+  (shutdown! [_this]
+    (memory/shutdown! layer)))
+
+(defn make-backend
+  "Constructs a `MemoryQueueBackend`. With no args, wraps the process-wide
+  `memory/default-layer`. Tests can pass their own layer for isolation."
+  ([] (make-backend memory/default-layer))
+  ([layer] (->MemoryQueueBackend layer)))
+
+(def backend
+  "Singleton `MemoryQueueBackend` backed by `memory/default-layer`."
+  (make-backend))

src/metabase/mq/queue/transport_impl.clj

@@ -0,0 +1,10 @@
+(ns metabase.mq.queue.transport-impl
+  "Queue implementations of the transport-level multimethods in
+  `metabase.mq.transport`."
+  (:require
+   [metabase.mq.transport :as transport]))
+
+(set! *warn-on-reflection* true)
+
+(defmethod transport/on-listen! :queue [_channel opts]
+  {:exclusive (boolean (:exclusive opts))})

src/metabase/mq/settings.clj

@@ -0,0 +1,33 @@
+(ns metabase.mq.settings
+  (:require
+   [metabase.app-db.core :as mdb]
+   [metabase.settings.core :as setting :refer [defsetting]]
+   [metabase.util.i18n :refer [deferred-tru]]))
+
+(defsetting queue-max-retries
+  (deferred-tru "Maximum number of times a failed queue message will be retried before being dropped.")
+  :type       :integer
+  :default    5
+  :visibility :internal
+  :encryption :no
+  :export?    false)
+
+(defsetting queue-backend
+  (deferred-tru "Which queue backend to use. Valid values: `appdb`, `memory`.")
+  :type       :string
+  :default    "appdb"
+  :visibility :internal
+  :encryption :no
+  :export?    false)
+
+(defsetting topic-backend
+  (deferred-tru "Which topic backend to use. Valid values: appdb, memory.")
+  :type       :string
+  :getter     (fn []
+                (or (setting/get-value-of-type :string :topic-backend)
+                    (if (= (mdb/db-type) :postgres)
+                      "appdb"
+                      "memory")))
+  :visibility :internal
+  :encryption :no
+  :export?    false)

src/metabase/mq/topic/appdb.clj

@@ -0,0 +1,125 @@
+(ns metabase.mq.topic.appdb
+  "Database-backed implementation of the pub/sub system using the application database.
+  Messages are stored in the `topic_message_batch` table. Each subscriber on each node polls
+  independently, tracking its read offset in memory."
+  (:require
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private batch-size
+  "Maximum number of messages to fetch in a single poll."
+  100)
+
+(defn- current-max-id
+  "Returns the current maximum id in topic_message_batch for the given topic, or 0."
+  [topic-name]
+  (or (t2/select-one-fn :max-id [:topic_message_batch [[:max :id] :max-id]]
+                        :topic_name (name topic-name))
+      0))
+
+(defn- poll-messages!
+  "Fetches rows from topic_message_batch with id > offset."
+  [topic-name offset]
+  (t2/query {:select   [:id :messages]
+             :from     [:topic_message_batch]
+             :where    [:and
+                        [:= :topic_name (name topic-name)]
+                        [:> :id offset]]
+             :order-by [[:id :asc]]
+             :limit    batch-size}))
+
+;;; ------------------------------------------- Periodic tasks (run on polling thread) -------------------------------------------
+
+(defn- cleanup-old-messages! []
+  (let [threshold (Timestamp/from (.minusMillis (Instant/now) (* 60 60 1000)))
+        deleted   (t2/delete! :topic_message_batch :created_at [:< threshold])]
+    (when (pos? deleted)
+      (log/infof "Cleaned up %d old topic messages" deleted)
+      (analytics/inc! :metabase-mq/appdb-cleanup-deleted {:transport "topic" :channel "all"} deleted))))
+
+(defn- update-lag-gauges! [offsets]
+  (doseq [[topic-name offset] @offsets]
+    (let [max-id (current-max-id topic-name)]
+      (analytics/set-gauge! :metabase-mq/appdb-topic-subscriber-lag
+                            {:channel (name topic-name)}
+                            (- max-id offset)))))
+
+(defn- poll-iteration!
+  "One iteration of the polling loop: run periodic tasks, then poll all topics.
+  Returns true if any topic had messages delivered.
+
+  Note on offset semantics: the offset is advanced as soon as the batch is claimed
+  by [[mq.impl/submit-delivery!]], not when the listener completes. Topics do not
+  retry, so a listener that throws does not roll the offset back, and a graceful
+  `stop!`/`start!` within the same process will not re-deliver an in-flight batch
+  (offsets live in memory)."
+  [{:keys [offsets last-cleanup-ms last-lag-gauge-ms]}]
+  (mq.polling/periodically! last-cleanup-ms   (* 10 60 1000) "topic cleanup"   cleanup-old-messages!)
+  (mq.polling/periodically! last-lag-gauge-ms (* 30 1000)    "topic lag gauge" #(update-lag-gauges! offsets))
+  (let [found-work? (atom false)]
+    (doseq [topic-name (remove mq.impl/channel-busy? (listener/topic-names))]
+      (let [offset (if (contains? @offsets topic-name)
+                     (get @offsets topic-name)
+                     (let [o (current-max-id topic-name)]
+                       (swap! offsets assoc topic-name o)
+                       o))
+            rows   (poll-messages! topic-name offset)]
+        (when (seq rows)
+          (let [all-messages (into [] (mapcat (comp json/decode :messages)) rows)
+                max-id       (:id (last rows))]
+            (when (mq.impl/submit-delivery! topic-name all-messages nil nil nil)
+              (swap! offsets assoc topic-name max-id)
+              (reset! found-work? true))))))
+    @found-work?))
+
+(defrecord AppDbTopicBackend [offsets poll-state last-cleanup-ms last-lag-gauge-ms]
+  topic.backend/TopicBackend
+  (publish! [_this topic-name messages]
+    (t2/insert! :topic_message_batch
+                {:topic_name (name topic-name)
+                 :messages   (json/encode messages)})
+    (when-not (mq.impl/channel-busy? topic-name)
+      (mq.polling/notify! poll-state)))
+
+  (start-receiving! [_this topic-name]
+    ;; Idempotent: first caller wins so an early offset pinned by
+    ;; [[metabase.mq.init/start-receiving!]] is not clobbered by the later call
+    ;; from `on-listen!` during `register-listeners!`.
+    (let [[old _] (swap-vals! offsets update topic-name
+                              (fn [existing]
+                                (if (some? existing) existing (current-max-id topic-name))))]
+      (when-not (contains? old topic-name)
+        (log/infof "Started receiving topic %s (starting offset %d)"
+                   (name topic-name) (get @offsets topic-name)))))
+
+  (unsubscribe! [_this topic-name]
+    (swap! offsets dissoc topic-name)
+    (log/infof "Unsubscribed from topic %s" (name topic-name)))
+
+  (start-handling! [this]
+    (mq.polling/start-polling! poll-state "Topic" 2000 #(poll-iteration! this)))
+
+  (shutdown! [_this]
+    (mq.polling/stop-polling! poll-state "Topic")
+    (reset! offsets {})))
+
+(defn make-backend
+  "Constructs a fresh `AppDbTopicBackend` with its own offsets atom, poll-state, and
+   periodic-task rate-limit atoms. Tests can build isolated ones so their `start-handling!` /
+   `shutdown!` don't touch production state."
+  []
+  (->AppDbTopicBackend (atom {}) (mq.polling/make-poll-state) (atom 0) (atom 0)))
+
+(def backend
+  "Singleton instance of the appdb topic backend."
+  (make-backend))

src/metabase/mq/topic/backend.clj

@@ -0,0 +1,39 @@
+(ns metabase.mq.topic.backend
+  "Backend abstraction for the pub/sub system.
+  Each concrete backend (appdb, memory, sync) is a record implementing `TopicBackend`.
+
+  Topics are fire-and-forget: there is no retry logic. If a listener throws, the error
+  is logged and the batch is skipped. This matches the semantics of the LISTEN/NOTIFY
+  backend where re-delivery is not possible.")
+
+(set! *warn-on-reflection* true)
+
+(defprotocol TopicBackend
+  "A topic backend handles fire-and-forget pub/sub delivery.
+
+  The lifecycle is two-phased:
+   1. `start-receiving!` is called per topic, very early in startup. It pins the read
+      position / registers the subscription with the broker. From this point forward,
+      messages published to this topic are retained for this consumer.
+   2. `start-handling!` is called once per backend, very late in startup. It begins
+      polling / consuming and delivering retained messages to registered handlers."
+  (publish!         [this topic-name messages]
+    "Publishes messages to the given topic. Messages are a vector.")
+  (start-receiving! [this topic-name]
+    "Idempotent. Pin the read position / register the subscription with the broker for
+     `topic-name`. After this call, messages published to the topic are retained for
+     this consumer but are not yet delivered to handlers.")
+  (start-handling!  [this]
+    "Begins polling / consuming. Delivers retained messages — from the read position
+     pinned by `start-receiving!` onward — to registered handlers. Called once at init
+     time, after `start-receiving!` has been called for all topics that have listeners.")
+  (unsubscribe!     [this topic-name]
+    "Stops receiving for the given topic and tears down the handler-side state — releases
+     resources, forgets the read position, and removes any associated subscription with the
+     broker..")
+  (shutdown!        [this]
+    "Shuts down all topic resources for this backend."))
+
+(def ^:dynamic *backend*
+  "The active `TopicBackend` instance. Set by `metabase.mq.init/start!`."
+  nil)

src/metabase/mq/topic/impl.clj

@@ -0,0 +1,25 @@
+(ns metabase.mq.topic.impl
+  "Topic-specific implementation: on-listen! hook and topic schemas."
+  (:require
+   [metabase.mq.publish :as publish]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def :metabase.mq.topic/topic-name
+  [:and :keyword [:fn {:error/message "Topic name must be namespaced to 'topic'"}
+                  #(= "topic" (namespace %))]])
+
+(defmacro with-topic
+  "Runs the body with the ability to add messages to the given topic.
+  Messages are buffered and only published if the body completes successfully.
+  If an exception occurs, no messages are published and the exception is rethrown.
+
+  Outside a transaction, publishing is best-effort and fire-and-forget — if the body
+  succeeds but `publish!` throws, the body's side effects will have already occurred.
+
+  Inside a database transaction, messages are deferred and published as a single batch
+  after the transaction commits successfully. This prevents consumers from seeing
+  uncommitted data. If the transaction rolls back, the messages are discarded."
+  [topic-name [buffer-binding] & body]
+  `(publish/with-buffer ~topic-name [~buffer-binding] ~@body))

src/metabase/mq/topic/memory.clj

@@ -0,0 +1,34 @@
+(ns metabase.mq.topic.memory
+  "In-memory topic backend. Delegates storage and polling to a shared memory layer."
+  (:require
+   [metabase.mq.memory :as memory]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.util.log :as log])
+  (:import (java.util.concurrent LinkedBlockingQueue)))
+
+(set! *warn-on-reflection* true)
+
+(defrecord MemoryTopicBackend [layer]
+  topic.backend/TopicBackend
+  (publish! [_this topic-name messages]
+    (memory/publish! layer topic-name messages))
+  (start-receiving! [_this topic-name]
+    ;; Match the appdb topic semantic: receivers don't see the prior backlog.
+    ;; Clear any queued messages for this topic at this point.
+    (when-let [^LinkedBlockingQueue q (get @(:channels layer) topic-name)]
+      (.clear q))
+    (log/infof "Memory receiving topic %s" (name topic-name)))
+  (start-handling! [_this] (memory/start! layer))
+  (unsubscribe! [_this topic-name]
+    (log/infof "Memory unsubscribed from topic %s" (name topic-name)))
+  (shutdown! [_this] (memory/shutdown! layer)))
+
+(defn make-backend
+  "Constructs a `MemoryTopicBackend`. With no args, wraps the process-wide
+  `memory/default-layer`. Tests can pass their own layer for isolation."
+  ([] (make-backend memory/default-layer))
+  ([layer] (->MemoryTopicBackend layer)))
+
+(def backend
+  "Singleton `MemoryTopicBackend` backed by `memory/default-layer`."
+  (make-backend))

src/metabase/mq/topic/transport_impl.clj

@@ -0,0 +1,12 @@
+(ns metabase.mq.topic.transport-impl
+  "Topic implementation of the transport-level multimethods in
+  `metabase.mq.transport`."
+  (:require
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.mq.transport :as transport]))
+
+(set! *warn-on-reflection* true)
+
+(defmethod transport/on-listen! :topic [channel _opts]
+  (topic.backend/start-receiving! topic.backend/*backend* channel)
+  {})

src/metabase/mq/transport.clj

@@ -0,0 +1,41 @@
+(ns metabase.mq.transport
+  "Thin dispatcher that routes channel operations to the queue or topic backend.
+  Selection is based on the channel keyword's namespace (`:queue/*` or `:topic/*`)."
+  (:require
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.topic.backend :as topic.backend]))
+
+(set! *warn-on-reflection* true)
+
+(defn transport-type
+  "Returns :queue or :topic for a namespaced channel keyword."
+  [channel]
+  (keyword (namespace channel)))
+
+(defmulti on-listen!
+  "Transport-specific hook invoked during listener registration. Returns a map of
+  default opts for the transport and may perform side effects (e.g. subscribing to
+  a topic backend). Dispatched on `transport-type`."
+  {:arglists '([channel opts])}
+  (fn [channel _opts] (transport-type channel)))
+
+(defn publish!
+  "Publishes messages to the appropriate backend for the channel's transport type."
+  [channel messages]
+  (case (transport-type channel)
+    :queue (q.backend/publish!     q.backend/*backend*     channel messages)
+    :topic (topic.backend/publish! topic.backend/*backend* channel messages)))
+
+(defn start!
+  "Starts the backend for the given transport type (`:queue` or `:topic`)."
+  [transport-type]
+  (case transport-type
+    :queue (q.backend/start!             q.backend/*backend*)
+    :topic (topic.backend/start-handling! topic.backend/*backend*)))
+
+(defn shutdown!
+  "Shuts down the backend for the given transport type (`:queue` or `:topic`)."
+  [transport-type]
+  (case transport-type
+    :queue (q.backend/shutdown!     q.backend/*backend*)
+    :topic (topic.backend/shutdown! topic.backend/*backend*)))

src/metabase/startup/core.clj

@@ -1,5 +1,5 @@
 (ns metabase.startup.core
-  "Defines the `def-startup-logic!` multimethod, which is used to run initialization logic when the server starts up."
+  "Defines the `def-startup-logic!` and `def-shutdown-logic!` multimethods for server lifecycle hooks."
   (:require
    [metabase.util :as u]
    [metabase.util.log :as log]))
@@ -15,17 +15,49 @@
 
   For logic that should run on only one node at a time, use `cluster-lock/with-cluster-lock` in your function.
 
+  To control ordering, add a [[startup-priority]] method for the same dispatch value (lower
+  priorities run first; default 0).
+
     (defmethod startup/init-logic! ::ExampleLogic [_]
       (future (println \"Running example logic...\")))"
   {:arglists '([job-name-string])}
   keyword)
 
+(defmulti startup-priority
+  "Ordering priority for [[def-startup-logic!]] implementations. Lower runs first; higher runs
+  later. Default is 0. Use `Long/MAX_VALUE` for logic that must run after everything else."
+  {:arglists '([job-name-string])}
+  keyword)
+
+(defmethod startup-priority :default [_] 500)
+
 (defn run-startup-logic!
-  "Call all implementations of `def-startup-logic!`. Called by metabase.core/init!"
+  "Call all implementations of `def-startup-logic!`. Called by metabase.core/init!
+  Methods run in ascending [[startup-priority]] order, with ties broken by dispatch-value name
+  for determinism."
   []
-  (doseq [[k f] (methods def-startup-logic!)]
+  (doseq [[k f] (sort-by (fn [[k _]] [(startup-priority k) (name k)])
+                         (methods def-startup-logic!))]
     (try
       (log/infof "Running setup logic %s %s" (u/format-color 'green (name k)) (u/emoji "☑\uFE0F"))
       (f k)
       (catch Throwable e
         (log/errorf e "Error initializing startup logic %s" k)))))
+
+(defmulti def-shutdown-logic!
+  "Runs shutdown logic with a given name. All implementations are called during graceful server
+  shutdown. Counterpart to [[def-startup-logic!]].
+
+  The dispatch value can be any unique keyword and is used for logging."
+  {:arglists '([job-name-string])}
+  keyword)
+
+(defn run-shutdown-logic!
+  "Call all implementations of `def-shutdown-logic!`. Called during graceful server shutdown."
+  []
+  (doseq [[k f] (methods def-shutdown-logic!)]
+    (try
+      (log/infof "Running shutdown logic %s" (name k))
+      (f k)
+      (catch Throwable e
+        (log/errorf e "Error running shutdown logic %s" k)))))

test/metabase/app_db/connection_test.clj

@@ -134,6 +134,95 @@
             (when-not (is-transaction-exception? e)
               (throw e))))))))
 
+(deftest after-commit-returns-false-outside-transaction-test
+  (testing "after-commit! returns false when not in a transaction"
+    (is (false? (mdb.connection/after-commit! (fn []))))))
+
+(deftest after-commit-returns-true-inside-transaction-test
+  (testing "after-commit! returns true when inside a transaction"
+    (t2/with-transaction []
+      (is (true? (mdb.connection/after-commit! (fn [])))))))
+
+(deftest after-commit-callbacks-execute-after-commit-test
+  (testing "Callbacks registered via after-commit! execute after successful commit"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit! (fn [] (swap! result conj :callback-1)))
+        (mdb.connection/after-commit! (fn [] (swap! result conj :callback-2)))
+        (is (= [] @result) "Callbacks should not have executed yet"))
+      (is (= [:callback-1 :callback-2] @result) "Callbacks should execute after commit"))))
+
+(deftest after-commit-callbacks-not-executed-on-rollback-test
+  (testing "Callbacks are not executed when a transaction is rolled back"
+    (let [result (atom [])]
+      (try
+        (t2/with-transaction []
+          (mdb.connection/after-commit! (fn [] (swap! result conj :should-not-run)))
+          (throw (ex-info "force rollback" {})))
+        (catch Exception _))
+      (is (= [] @result) "Callbacks should not execute on rollback"))))
+
+(deftest after-commit-callback-registers-new-callback-test
+  (testing "A callback that registers another callback — both execute"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit!
+         (fn []
+           (swap! result conj :first)
+           ;; Register a new callback from within a callback
+           (mdb.connection/after-commit! (fn [] (swap! result conj :second))))))
+      (is (= [:first :second] @result) "Both callbacks should execute")))
+
+  (testing "Deeply chained callbacks all execute"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit!
+         (fn []
+           (swap! result conj :a)
+           (mdb.connection/after-commit!
+            (fn []
+              (swap! result conj :b)
+              (mdb.connection/after-commit!
+               (fn [] (swap! result conj :c))))))))
+      (is (= [:a :b :c] @result)))))
+
+(deftest nested-transactions-share-after-commit-and-state-test
+  (testing "Nested transactions share the same *after-commit* and *transaction-state*"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit! (fn [] (swap! result conj :outer)))
+        (let [outer-state mdb.connection/*transaction-state*]
+          (t2/with-transaction []
+            (is (identical? outer-state mdb.connection/*transaction-state*)
+                "*transaction-state* should be the same atom in nested transactions")
+            (mdb.connection/after-commit! (fn [] (swap! result conj :inner)))))
+        (is (= [] @result) "Callbacks should not have executed yet"))
+      (is (= [:outer :inner] @result) "Both callbacks should execute after outermost commit"))))
+
+(deftest after-commit-discarded-on-nested-rollback-test
+  (testing "Callbacks registered in a nested transaction that rolls back are NOT executed"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit! (fn [] (swap! result conj :outer)))
+        (try
+          (t2/with-transaction []
+            (mdb.connection/after-commit! (fn [] (swap! result conj :inner-should-not-run)))
+            (throw (Exception. "force savepoint rollback")))
+          (catch Exception _)))
+      (is (= [:outer] @result)
+          "Only the outer callback should run; the inner one from the rolled-back savepoint should be discarded")))
+
+  (testing "Transaction state from rolled-back nested transaction is also discarded"
+    (t2/with-transaction []
+      (swap! mdb.connection/*transaction-state* assoc :outer-key "outer-val")
+      (try
+        (t2/with-transaction []
+          (swap! mdb.connection/*transaction-state* assoc :inner-key "inner-val")
+          (throw (Exception. "force savepoint rollback")))
+        (catch Exception _))
+      (is (= {:outer-key "outer-val"} @mdb.connection/*transaction-state*)
+          "State from rolled-back nested transaction should be discarded"))))
+
 (deftest ^:parallel transaction-isolation-level-test
   (testing "We should always use READ_COMMITTED for the app DB (#44505)"
     (with-open [conn (.getConnection mdb.connection/*application-db*)]

test/metabase/metabot/api_test.clj

@@ -917,20 +917,21 @@
                                                             (reset! captured-args args)
                                                             ;; Return a minimal streaming response
                                                             nil)]
-        (api/streaming-request {:metabot_id      test-metabot-id
-                                :profile_id      nil
-                                :message         "test message"
-                                :context         {}
-                                :history         []
-                                :conversation_id (str (random-uuid))
-                                :state           {}
-                                :debug           false}
-                               {:origin nil :referer nil :user-agent nil :ip-address nil})
-        (testing "metabot-id is included in the arguments"
-          (is (some? (:metabot-id @captured-args))
-              "metabot-id should not be nil")
-          (is (= test-metabot-id (:metabot-id @captured-args))
-              "metabot-id should match the input metabot_id"))))))
+        (mt/with-test-user :rasta
+          (api/streaming-request {:metabot_id      test-metabot-id
+                                  :profile_id      nil
+                                  :message         "test message"
+                                  :context         {}
+                                  :history         []
+                                  :conversation_id (str (random-uuid))
+                                  :state           {}
+                                  :debug           false}
+                                 {:origin nil :referer nil :user-agent nil :ip-address nil})
+          (testing "metabot-id is included in the arguments"
+            (is (some? (:metabot-id @captured-args))
+                "metabot-id should not be nil")
+            (is (= test-metabot-id (:metabot-id @captured-args))
+                "metabot-id should match the input metabot_id")))))))
 
 (deftest streaming-request-ip-address-test
   (mt/with-model-cleanup [:model/MetabotMessage

test/metabase/mq/backend_parity_test.clj

@@ -0,0 +1,151 @@
+(ns metabase.mq.backend-parity-test
+  "Contract-level parity between the queue/topic backends.
+
+  Each scenario is written once and run against every backend kind so that
+  semantic drift between the memory and appdb implementations is caught at
+  the source rather than discovered later in production.
+
+  Every scenario is ALSO run with `:duplicate-delivery? true` so that
+  listeners in the parity suite are forced to tolerate the MQ's at-least-
+  once contract. Assertions use multiset / containment checks rather than
+  exact-sequence equality so that a single logical message can legitimately
+  be delivered more than once.
+
+  Scenarios use `mq.tu/eventually!` rather than `mq.tu/flush!` for their
+  assertions, because `wait-for-idle!` cannot see pending rows inside the
+  appdb tables — it only checks the publish buffer, the worker-pool busy
+  set, and memory channels."
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.test-util :as mq.tu]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private backend-kinds
+  "Backend kinds exercised by the parity scenarios. `:memory` covers the
+  fast async in-process backend; `:appdb` covers the durable DB-backed one."
+  [:memory :appdb])
+
+(def ^:private delivery-modes
+  "Delivery modes exercised by the parity scenarios. `:normal` is the
+  production path; `:duplicate` wraps both backends with a decorator that
+  publishes every message twice, forcing listeners to tolerate at-least-
+  once delivery."
+  [:normal :duplicate])
+
+(def ^:private scenario-timeout-ms
+  "Budget for waiting on eventual delivery in parity scenarios. appdb runs
+  through the publish buffer → DB insert → 5s-default poll loop, but the
+  fixture's `wait-for-idle!` calls `notify-all!` each tick so the effective
+  poll latency is much lower."
+  10000)
+
+(defn- cleanup-appdb-channel!
+  "Removes any rows left in the message tables for the given channel so a
+  parity scenario does not pollute later tests."
+  [channel]
+  (case (namespace channel)
+    "queue" (t2/delete! :queue_message_batch :queue_name (name channel))
+    "topic" (t2/delete! :topic_message_batch :topic_name (name channel))))
+
+(defn- unique-channel [transport-ns suffix]
+  (keyword transport-ns (str suffix "-" (random-uuid))))
+
+(defn- run-parity!
+  "Runs `scenario-fn` once per (kind × delivery-mode) combination. Each
+  invocation gets a freshly-minted unique channel keyword so scenarios do
+  not interfere with each other. On completion the scenario's channel is
+  unlistened (if still present) and any appdb rows for it are cleaned up."
+  ([channel-prefix scenario-fn]
+   (run-parity! backend-kinds delivery-modes channel-prefix scenario-fn))
+  ([kinds modes channel-prefix scenario-fn]
+   (doseq [kind  kinds
+           mode  modes]
+     (testing (str kind " backend / " mode " delivery")
+       (let [channel (unique-channel (namespace channel-prefix) (name channel-prefix))]
+         (try
+           (mq.tu/with-test-mq [ctx {:backend                kind
+                                     :duplicate-delivery? (= mode :duplicate)}]
+             (scenario-fn ctx channel))
+           (finally
+             (when (= :appdb kind)
+               (cleanup-appdb-channel! channel)))))))))
+
+(deftest queue-delivers-published-messages-test
+  (run-parity!
+   :queue/parity-delivery
+   (fn [ctx queue-name]
+     (let [received (atom [])]
+       (mq.tu/listen! queue-name {} #(swap! received conj %))
+       (mq/with-queue queue-name [q]
+         (mq/put q "a")
+         (mq/put q "b")
+         (mq/put q "c"))
+       (mq.tu/eventually! ctx
+                          #(= #{"a" "b" "c"} (set @received))
+                          scenario-timeout-ms)
+       (is (= #{"a" "b" "c"} (set @received))
+           "Every unique message was delivered at least once")
+       (is (every? pos? (vals (frequencies @received)))
+           "No spurious messages appeared in @received")
+       (mq/unlisten! queue-name)))))
+
+(deftest topic-delivers-published-messages-test
+  (run-parity!
+   :topic/parity-delivery
+   (fn [ctx topic-name]
+     (let [received (atom [])]
+       (mq.tu/listen! topic-name {} #(swap! received conj %))
+       (mq/with-topic topic-name [t]
+         (mq/put t "x")
+         (mq/put t "y"))
+       (mq.tu/eventually! ctx
+                          #(= #{"x" "y"} (set @received))
+                          scenario-timeout-ms)
+       (is (= #{"x" "y"} (set @received))
+           "Every unique topic message was delivered at least once")
+       (mq/unlisten! topic-name)))))
+
+(deftest topic-error-isolation-test
+  (run-parity!
+   :topic/parity-errors
+   (fn [ctx topic-name]
+     (let [received (atom [])]
+       (mq.tu/listen! topic-name {}
+                      (fn [msg]
+                        (when (= "bad" msg)
+                          (throw (ex-info "handler error" {})))
+                        (swap! received conj msg)))
+       (mq/with-topic topic-name [t]
+         (mq/put t "good-1")
+         (mq/put t "bad")
+         (mq/put t "good-2"))
+       (mq.tu/eventually! ctx
+                          #(= #{"good-1" "good-2"} (set @received))
+                          scenario-timeout-ms)
+       (is (= #{"good-1" "good-2"} (set @received))
+           "Good messages delivered; bad message isolated from neighbours")
+       (is (not (contains? (set @received) "bad"))
+           "The throwing message never ends up in @received")
+       (mq/unlisten! topic-name)))))
+
+(deftest queue-retries-failed-batch-test
+  (run-parity!
+   :queue/parity-retry
+   (fn [ctx queue-name]
+     (let [calls-by-msg (atom {})]
+       (mq.tu/listen! queue-name {}
+                      (fn [msg]
+                        (let [n (get (swap! calls-by-msg update msg (fnil inc 0)) msg)]
+                          (when (= 1 n)
+                            (throw (ex-info "first attempt" {:msg msg}))))))
+       (mq/with-queue queue-name [q]
+         (mq/put q "retry-me"))
+       (mq.tu/eventually! ctx
+                          #(>= (get @calls-by-msg "retry-me" 0) 2)
+                          scenario-timeout-ms)
+       (is (>= (get @calls-by-msg "retry-me" 0) 2)
+           "Message delivered at least twice (retry-after-failure or chaos duplication)")
+       (mq/unlisten! queue-name)))))

test/metabase/mq/publish_buffer_test.clj

@@ -0,0 +1,89 @@
+(ns metabase.mq.publish-buffer-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.transport :as transport]))
+
+(set! *warn-on-reflection* true)
+
+(deftest flush-re-buffers-on-failure-test
+  (testing "When transport/publish! throws, messages are re-buffered (not lost)"
+    (let [published (atom [])
+          call-count (atom 0)]
+      (binding [publish-buffer/*publish-buffer* (atom {})
+                publish-buffer/*publish-buffer-ms* 0
+                publish-buffer/*publish-buffer-max-ms* 0]
+        ;; Directly populate the buffer with an entry past its deadline
+        (reset! publish-buffer/*publish-buffer*
+                {:queue/test {:messages ["msg1" "msg2"]
+                              :deadline-ms 1
+                              :created-ms 1}})
+        (with-redefs [transport/publish! (fn [channel messages]
+                                           (swap! call-count inc)
+                                           (if (= 1 @call-count)
+                                             (throw (ex-info "publish failed" {}))
+                                             (swap! published conj {:channel channel :messages messages})))]
+          ;; First flush — publish throws, messages should be re-buffered
+          (publish-buffer/flush-publish-buffer!)
+          (testing "Messages are back in the buffer after failure"
+            (is (contains? @publish-buffer/*publish-buffer* :queue/test))
+            (is (= ["msg1" "msg2"]
+                   (get-in @publish-buffer/*publish-buffer* [:queue/test :messages]))))
+
+          ;; Second flush — publish succeeds
+          (publish-buffer/flush-publish-buffer!)
+          (testing "Messages are delivered on retry"
+            (is (= [{:channel :queue/test :messages ["msg1" "msg2"]}] @published))
+            (is (empty? @publish-buffer/*publish-buffer*))))))))
+
+(deftest flush-drops-messages-after-max-retries-test
+  (testing "Messages are dropped after exceeding max retries"
+    (binding [publish-buffer/*publish-buffer*             (atom {})
+              publish-buffer/*publish-buffer-ms*          0
+              publish-buffer/*publish-buffer-max-ms*      0
+              publish-buffer/*publish-buffer-max-retries* 3]
+      (with-redefs [transport/publish! (fn [_channel _messages]
+                                         (throw (ex-info "always fails" {})))]
+        ;; Populate buffer with messages past their deadline
+        (reset! publish-buffer/*publish-buffer*
+                {:queue/test {:messages   ["msg1" "msg2"]
+                              :deadline-ms 1
+                              :created-ms  1
+                              :retries     0}})
+        ;; Flush 1: retries=1, re-buffered
+        (publish-buffer/flush-publish-buffer!)
+        (is (contains? @publish-buffer/*publish-buffer* :queue/test)
+            "Messages should be re-buffered after first failure")
+
+        ;; Flush 2: retries=2, re-buffered
+        (publish-buffer/flush-publish-buffer!)
+        (is (contains? @publish-buffer/*publish-buffer* :queue/test)
+            "Messages should be re-buffered after second failure")
+
+        ;; Flush 3: retries=3 >= max-retries=3, dropped
+        (publish-buffer/flush-publish-buffer!)
+        (is (empty? @publish-buffer/*publish-buffer*)
+            "Messages should be dropped after reaching max retries")))))
+
+(deftest flush-re-buffer-merges-with-new-messages-test
+  (testing "Re-buffered messages merge with new messages that arrived during flush"
+    (binding [publish-buffer/*publish-buffer* (atom {})
+              publish-buffer/*publish-buffer-ms* 100
+              publish-buffer/*publish-buffer-max-ms* 0]
+      ;; Populate buffer with messages past deadline
+      (reset! publish-buffer/*publish-buffer*
+              {:queue/test {:messages ["old1" "old2"]
+                            :deadline-ms 1
+                            :created-ms 1}})
+      (with-redefs [transport/publish! (fn [_channel _messages]
+                                         ;; Simulate new messages arriving during flush
+                                         (swap! publish-buffer/*publish-buffer*
+                                                assoc :queue/test
+                                                {:messages ["new1"]
+                                                 :deadline-ms (+ (System/currentTimeMillis) 100)
+                                                 :created-ms (System/currentTimeMillis)})
+                                         (throw (ex-info "publish failed" {})))]
+        (publish-buffer/flush-publish-buffer!)
+        (testing "Re-buffered (older) messages are prepended so publish order is preserved across retries"
+          (let [messages (get-in @publish-buffer/*publish-buffer* [:queue/test :messages])]
+            (is (= ["old1" "old2" "new1"] messages))))))))

test/metabase/mq/queue/appdb_test.clj

@@ -0,0 +1,320 @@
+(ns metabase.mq.queue.appdb-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(deftest publish-test
+  (let [queue-name (keyword "queue" (str (gensym "publish-test-")))]
+    (try
+      (q.backend/publish! q.appdb/backend queue-name ["test message"])
+      (testing "Messages are persisted in the queue"
+        (let [row (t2/select-one :queue_message_batch :queue_name (name queue-name))]
+          (is (= ["test message"] (json/decode (:messages row))))
+          (is (= (name queue-name) (:queue_name row)))
+          (is (= "pending" (:status row)))
+          (is (not (nil? (:status_heartbeat row))))
+          (is (= (:created_at row) (:status_heartbeat row)))
+          (is (= 0 (:failures row)))
+          (is (nil? (:owner row)))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest fetch-test
+  (let [queue1 (keyword "queue" (str "queue1-" (gensym)))
+        queue2 (keyword "queue" (str "queue2-" (gensym)))
+        invalid-queue (keyword "queue" (str "queue-invalid-" (gensym)))]
+    (binding [listener/*listeners* (atom {queue1 {:listener identity} queue2 {:listener identity}})]
+      (try
+        (t2/with-connection [_conn]
+          (testing "Returns nil if no rows are found"
+            (is (nil? (#'q.appdb/fetch! (:owner-id q.appdb/backend) (listener/queue-names)))))
+
+          (t2/insert! :queue_message_batch
+                      {:queue_name (name invalid-queue)
+                       :messages   (json/encode ["invalid"])})
+          (t2/insert! :queue_message_batch
+                      {:queue_name (name queue1)
+                       :messages   (json/encode ["data1"])})
+          (t2/insert! :queue_message_batch
+                      {:queue_name (name queue2)
+                       :messages   (json/encode ["data2"])})
+
+          (testing "Returns nil if no queues are defined"
+            (binding [listener/*listeners* (atom {})]
+              (is (nil? (#'q.appdb/fetch! (:owner-id q.appdb/backend) (listener/queue-names))))))
+          (testing "Fetches one row per queue in a single call"
+            (let [results (#'q.appdb/fetch! (:owner-id q.appdb/backend) (listener/queue-names))
+                  by-queue (into {} (map (juxt :queue identity)) results)]
+              (is (= 2 (count results)))
+              (testing "queue1 row"
+                (let [{:keys [batch-id messages]} (get by-queue queue1)]
+                  (is (pos-int? batch-id))
+                  (is (= ["data1"] messages))
+                  (testing "Fetched row gets marked as processing"
+                    (let [updated-row (t2/select-one :queue_message_batch :id batch-id)]
+                      (is (= "processing" (:status updated-row)))
+                      (is (not= (:created_at updated-row) (:status_heartbeat updated-row)))
+                      (is (= 0 (:failures updated-row)))
+                      (is (= (:owner-id q.appdb/backend) (:owner updated-row)))))))
+              (testing "queue2 row"
+                (let [{:keys [batch-id messages]} (get by-queue queue2)]
+                  (is (pos-int? batch-id))
+                  (is (= ["data2"] messages))))))
+          (testing "When everything valid is processing, return nil"
+            (is (nil? (#'q.appdb/fetch! (:owner-id q.appdb/backend) (listener/queue-names))))))
+        (finally
+          (t2/delete! :queue_message_batch :queue_name [:in [(name queue1) (name queue2) (name invalid-queue)]]))))))
+
+(deftest exclusive-fetch-test
+  (let [exclusive-q (keyword "queue" (str "exclusive-" (random-uuid)))
+        normal-q    (keyword "queue" (str "normal-" (random-uuid)))]
+    (binding [listener/*listeners* (atom {exclusive-q {:listener identity :exclusive true}
+                                          normal-q    {:listener identity :exclusive false}})]
+      (try
+        (t2/with-connection [_conn]
+          (testing "With no processing rows, exclusive queue messages are fetched normally"
+            (t2/insert! :queue_message_batch
+                        {:queue_name (name exclusive-q)
+                         :messages   (json/encode ["exclusive-msg1"])})
+            (let [results (#'q.appdb/fetch! (:owner-id q.appdb/backend) (listener/queue-names))
+                  {:keys [queue messages]} (first results)]
+              (is (= 1 (count results)))
+              (is (= exclusive-q queue))
+              (is (= ["exclusive-msg1"] messages))))
+
+          (testing "With a processing row on exclusive queue, fetch skips it"
+            (t2/insert! :queue_message_batch
+                        {:queue_name (name exclusive-q)
+                         :messages   (json/encode ["exclusive-msg2"])})
+            ;; The first message is now 'processing', so the second should be skipped
+            (is (nil? (#'q.appdb/fetch! (:owner-id q.appdb/backend) (listener/queue-names)))
+                "Should return nil because exclusive queue has a processing row"))
+
+          (testing "Non-exclusive queue is still fetchable even when exclusive queue is blocked"
+            (t2/insert! :queue_message_batch
+                        {:queue_name (name normal-q)
+                         :messages   (json/encode ["normal-msg1"])})
+            (let [results (#'q.appdb/fetch! (:owner-id q.appdb/backend) (listener/queue-names))
+                  {:keys [queue messages]} (first results)]
+              (is (= 1 (count results)))
+              (is (= normal-q queue))
+              (is (= ["normal-msg1"] messages))))
+
+          (testing "After exclusive processing completes, next message can be fetched"
+            ;; Mark the processing row as done by deleting it (simulating batch-successful!)
+            (t2/delete! :queue_message_batch :queue_name (name exclusive-q) :status "processing")
+            (let [results (#'q.appdb/fetch! (:owner-id q.appdb/backend) (listener/queue-names))
+                  {:keys [queue messages]} (first results)]
+              (is (= exclusive-q queue))
+              (is (= ["exclusive-msg2"] messages)))))
+        (finally
+          (t2/delete! :queue_message_batch :queue_name [:in [(name exclusive-q) (name normal-q)]]))))))
+
+(deftest message-successful-test
+  (let [queue-name (keyword "queue" (str (gensym "successful-test-")))]
+    (testing "Message successful deletes the message"
+      (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                {:queue_name (name queue-name)
+                                                 :messages (json/encode ["test-message"])
+                                                 :status "processing"
+                                                 :owner (:owner-id q.appdb/backend)})]
+        (q.backend/batch-successful! q.appdb/backend queue-name message-id)
+        (is (nil? (t2/select-one :queue_message_batch :id message-id)))))
+
+    (testing "Message successful on non-existent message does not fail"
+      (is (nil? (q.backend/batch-successful! q.appdb/backend queue-name 99999))))))
+
+;; Note: no cleanup needed — batch-successful! deletes the row
+
+(deftest message-failed-test
+  (let [queue-name (keyword "queue" (str (gensym "failed-test-")))]
+    (try
+      (testing "Message failed resets to pending and increments failures"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name (name queue-name)
+                                                   :messages (json/encode ["test-message"])
+                                                   :status "processing"
+                                                   :failures 0
+                                                   :owner (:owner-id q.appdb/backend)})]
+          (q.backend/batch-failed! q.appdb/backend queue-name message-id)
+          (let [updated-message (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "pending" (:status updated-message)))
+            (is (= 1 (:failures updated-message)))
+            (is (nil? (:owner updated-message)))
+            (is (not (nil? (:status_heartbeat updated-message)))))))
+
+      (testing "Message failed increments failures multiple times"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name (name queue-name)
+                                                   :messages (json/encode ["test-message"])
+                                                   :status "processing"
+                                                   :failures 2
+                                                   :owner (:owner-id q.appdb/backend)})]
+          (q.backend/batch-failed! q.appdb/backend queue-name message-id)
+          (let [updated-message (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "pending" (:status updated-message)))
+            (is (= 3 (:failures updated-message)))
+            (is (nil? (:owner updated-message))))))
+
+      (testing "Message moved to failed status after max failures"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name (name queue-name)
+                                                   :messages (json/encode ["test-message"])
+                                                   :status "processing"
+                                                   :failures (dec (mq.settings/queue-max-retries))
+                                                   :owner (:owner-id q.appdb/backend)})]
+          (q.backend/batch-failed! q.appdb/backend queue-name message-id)
+          (let [updated-message (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "failed" (:status updated-message)))
+            (is (= (mq.settings/queue-max-retries) (:failures updated-message)))
+            (is (nil? (:owner updated-message))))))
+
+      (testing "Message failed on non-existent message does not fail"
+        (is (nil? (q.backend/batch-failed! q.appdb/backend queue-name 99999))))
+
+      (testing "Message failed when being processed by another node is a no-op"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name (name queue-name)
+                                                   :messages (json/encode ["test-message"])
+                                                   :status "processing"
+                                                   :owner "another-node"
+                                                   :failures 0})]
+          (q.backend/batch-failed! q.appdb/backend queue-name message-id)
+          (let [updated-message (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "processing" (:status updated-message)))
+            (is (= 0 (:failures updated-message)))
+            (is (= "another-node" (:owner updated-message))))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest recover-stale-processing-batches-test
+  (let [queue-name (keyword "queue" (str "stale-test-" (random-uuid)))
+        stale-heartbeat (Timestamp/from (.minusMillis (Instant/now) (+ @#'q.appdb/stale-processing-timeout-ms 60000)))]
+    (try
+      (testing "Stale processing batch is reset to pending with incremented failures"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :messages         (json/encode ["stale-msg"])
+                                                   :status           "processing"
+                                                   :status_heartbeat stale-heartbeat
+                                                   :failures         1
+                                                   :owner            "dead-owner"})]
+          (is (pos? (#'q.appdb/recover-stale-processing-batches!)))
+          (let [row (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "pending" (:status row)))
+            (is (= 2 (:failures row)))
+            (is (nil? (:owner row))))))
+
+      (testing "Stale batch at max retries is marked as failed"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :messages         (json/encode ["stale-msg-max"])
+                                                   :status           "processing"
+                                                   :status_heartbeat stale-heartbeat
+                                                   :failures         (dec (mq.settings/queue-max-retries))
+                                                   :owner            "dead-owner"})]
+          (is (pos? (#'q.appdb/recover-stale-processing-batches!)))
+          (let [row (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "failed" (:status row)))
+            (is (= (mq.settings/queue-max-retries) (:failures row)))
+            (is (nil? (:owner row))))))
+
+      (testing "Recent processing batch is not recovered by stale check"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :messages         (json/encode ["recent-msg"])
+                                                   :status           "processing"
+                                                   :failures         0
+                                                   :owner            "active-owner"})]
+          (#'q.appdb/recover-stale-processing-batches!)
+          (let [row (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "processing" (:status row)))
+            (is (= 0 (:failures row))))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest heartbeat-test
+  (let [queue-name (keyword "queue" (str "heartbeat-test-" (random-uuid)))]
+    (try
+      (let [message-id         (t2/insert-returning-pk! :queue_message_batch
+                                                        {:queue_name (name queue-name)
+                                                         :messages   (json/encode ["msg"])
+                                                         :status     "processing"
+                                                         :owner      (:owner-id q.appdb/backend)})
+            original-heartbeat (:status_heartbeat (t2/select-one :queue_message_batch :id message-id))]
+        (Thread/sleep 100) ; Ensure enough time passes for timestamp to differ
+        (with-redefs [mq.impl/busy-channels          (constantly #{queue-name})
+                      mq.impl/active-handler-metadata (fn [_ch] {:batch-id message-id})]
+          (#'q.appdb/update-heartbeats! (:owner-id q.appdb/backend)))
+        (testing "Heartbeat is updated after update-heartbeats!"
+          (let [updated-heartbeat (:status_heartbeat (t2/select-one :queue_message_batch :id message-id))]
+            (is (not= original-heartbeat updated-heartbeat)
+                "Heartbeat timestamp should be updated"))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest cleanup-failed-batches-test
+  (let [queue-name (keyword "queue" (str "cleanup-test-" (random-uuid)))
+        old-ts     (Timestamp/from (.minusMillis (Instant/now) (+ (* 7 24 60 60 1000) 60000)))]
+    (try
+      (testing "Failed batch older than 7 days is deleted"
+        (let [old-id (t2/insert-returning-pk! :queue_message_batch
+                                              {:queue_name       (name queue-name)
+                                               :messages         (json/encode ["old-msg"])
+                                               :status           "failed"
+                                               :status_heartbeat old-ts})]
+          (#'q.appdb/cleanup-failed-batches!)
+          (is (nil? (t2/select-one :queue_message_batch :id old-id))
+              "Old failed batch should be deleted")))
+
+      (testing "Recent failed batch is not deleted"
+        (let [recent-id (t2/insert-returning-pk! :queue_message_batch
+                                                 {:queue_name (name queue-name)
+                                                  :messages   (json/encode ["recent-msg"])
+                                                  :status     "failed"})]
+          (#'q.appdb/cleanup-failed-batches!)
+          (is (some? (t2/select-one :queue_message_batch :id recent-id))
+              "Recent failed batch should not be deleted")
+          (t2/delete! :queue_message_batch :id recent-id)))
+
+      (testing "Non-failed batch with old heartbeat is not cleaned up"
+        (let [pending-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :messages         (json/encode ["pending-msg"])
+                                                   :status           "pending"
+                                                   :status_heartbeat old-ts})]
+          (#'q.appdb/cleanup-failed-batches!)
+          (is (some? (t2/select-one :queue_message_batch :id pending-id))
+              "Non-failed batch should not be deleted")
+          (t2/delete! :queue_message_batch :id pending-id)))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest update-depth-gauges-test
+  (let [queue-name  (keyword "queue" (str "depth-gauge-test-" (random-uuid)))
+        gauge-calls (atom [])]
+    (try
+      (t2/insert! :queue_message_batch {:queue_name (name queue-name) :messages (json/encode ["m1"]) :status "pending"})
+      (t2/insert! :queue_message_batch {:queue_name (name queue-name) :messages (json/encode ["m2"]) :status "failed"})
+      (with-redefs [analytics/set-gauge! (fn [metric labels value]
+                                           (when (= metric :metabase-mq/appdb-queue-depth)
+                                             (swap! gauge-calls conj {:labels labels :value value})))]
+        (#'q.appdb/update-depth-gauges!))
+      (testing "Gauge is emitted for each queue/status combination"
+        (let [calls-for-queue (filter #(= (name queue-name) (-> % :labels :channel)) @gauge-calls)
+              by-status       (into {} (map (juxt #(-> % :labels :status) :value)) calls-for-queue)]
+          (is (= 1 (get by-status "pending")) "Should count 1 pending message")
+          (is (= 1 (get by-status "failed")) "Should count 1 failed message")))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))

test/metabase/mq/queue/core_test.clj

@@ -0,0 +1,285 @@
+(ns metabase.mq.queue.core-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.app-db.connection :as app-db.conn]
+   [metabase.mq.core :as mq]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish :as mq.publish]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.test-util :as mq.tu])
+  (:import (clojure.lang ExceptionInfo)
+           (java.util.concurrent CountDownLatch CyclicBarrier)))
+
+(set! *warn-on-reflection* true)
+
+(deftest e2e-test
+  (let [heard-messages (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [message] (swap! heard-messages conj message))}
+      (testing "The messages are heard and processed"
+        (mq/with-queue :queue/test [q]
+          (mq/put q "test message 1")
+          (mq/put q "test message 2"))
+        (mq.tu/flush! test-mq)
+        (is (= ["test message 1" "test message 2"] @heard-messages))))))
+
+(deftest publish-to-unlistened-queue-test
+  (mq.tu/with-test-mq [_test-mq]
+    (testing "with-queue on a queue with no listener still succeeds"
+      (mq/with-queue :queue/nonexistent [q]
+        (mq/put q "msg")))))
+
+(deftest with-queue-success-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (testing "with-queue publishes buffered messages on success"
+        (let [result (mq/with-queue :queue/test [q]
+                       (mq/put q "a")
+                       (mq/put q "b")
+                       :done)]
+          (is (= :done result))
+          (mq.tu/flush! test-mq)
+          (is (= ["a" "b"] @heard)))))))
+
+(deftest with-queue-exception-discards-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (testing "with-queue discards buffered messages on exception"
+        (is (thrown? Exception
+                     (mq/with-queue :queue/test [q]
+                       (mq/put q "should-be-discarded")
+                       (throw (ex-info "boom" {})))))
+        (mq.tu/flush! test-mq)
+        (is (empty? @heard))))))
+
+(deftest with-queue-no-listener-test
+  (mq.tu/with-test-mq [_test-mq]
+    (testing "with-queue on queue with no listener still succeeds"
+      (mq/with-queue :queue/nonexistent [q]
+        (mq/put q "msg")))))
+
+(deftest double-listen-throws-test
+  (mq.tu/with-test-mq [_test-mq]
+    {:queue/test (fn [_] nil)}
+    (testing "Registering a second listener on the same queue throws"
+      (is (thrown-with-msg? ExceptionInfo #"Listener already registered"
+                            (mq.tu/listen! :queue/test {} (fn [_] nil)))))))
+
+(deftest concurrent-listen-throws-test
+  (mq.tu/with-test-mq [_test-mq]
+    (let [queue-name :queue/test
+          n          10
+          barrier    (CyclicBarrier. n)
+          results    (atom [])]
+      (testing "Concurrent listen! calls: exactly one succeeds, rest throw"
+        (let [threads (mapv (fn [_]
+                              (let [f (bound-fn []
+                                        (.await barrier)
+                                        (try
+                                          (mq.tu/listen! queue-name {} (fn [_] nil))
+                                          (swap! results conj :ok)
+                                          (catch ExceptionInfo _
+                                            (swap! results conj :error))))]
+                                (Thread. ^Runnable f)))
+                            (range n))]
+          (run! (fn [^Thread t] (.start t)) threads)
+          (run! (fn [^Thread t] (.join t 5000)) threads)
+          (is (= 1 (count (filter #{:ok} @results))))
+          (is (= (dec n) (count (filter #{:error} @results))))))
+      (mq/unlisten! queue-name))))
+
+(deftest exclusive-listen-test
+  (let [heard-messages (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/exclusive-test {:listener           (fn [messages]
+                                                    (run! #(swap! heard-messages conj %) messages))
+                              :max-batch-messages 1
+                              :exclusive          true}}
+      (testing "Exclusive queue processes messages via the async memory backend"
+        (mq/with-queue :queue/exclusive-test [q]
+          (mq/put q "msg1")
+          (mq/put q "msg2"))
+        (mq.tu/flush! test-mq)
+        (is (= ["msg1" "msg2"] @heard-messages))))))
+
+(deftest batch-listen-exclusive-test
+  (let [heard-batches (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      (listener/batch-listen! :queue/batch-exclusive
+                              (fn [batch] (swap! heard-batches conj batch))
+                              {:max-batch-messages 10 :exclusive true})
+      (testing "Exclusive batch-listen! registers and processes"
+        (mq/with-queue :queue/batch-exclusive [q]
+          (mq/put q "a")
+          (mq/put q "b"))
+        (mq.tu/flush! test-mq)
+        (is (= [["a" "b"]] @heard-batches))))))
+
+(deftest transaction-defers-publish-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (binding [app-db.conn/*after-commit*  (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (testing "Inside a transaction, messages are accumulated, not published immediately"
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg1")
+            (mq/put q "msg2"))
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg3"))
+          (testing "Messages are deferred in transaction state"
+            (is (= ["msg1" "msg2" "msg3"]
+                   (get-in @app-db.conn/*transaction-state*
+                           [::mq.publish/deferred-messages :queue/test]))))
+          (testing "Listener has not been called yet"
+            (is (= [] @heard))))
+        (testing "After flush (simulating commit), all messages are delivered"
+          (mq.publish/flush-deferred-messages!)
+          (mq.tu/flush! test-mq)
+          (is (= ["msg1" "msg2" "msg3"] @heard)))))))
+
+(deftest transaction-rollback-discards-messages-test
+  (mq.tu/with-test-mq [test-mq]
+    {:queue/test (fn [_] nil)}
+    (binding [app-db.conn/*after-commit*  (atom [])
+              app-db.conn/*transaction-state* (atom {})]
+      (testing "Messages accumulated during a failed transaction are discarded"
+        (is (thrown? Exception
+                     (mq/with-queue :queue/test [q]
+                       (mq/put q "should-be-discarded")
+                       (throw (ex-info "boom" {})))))
+        (is (nil? (get-in @app-db.conn/*transaction-state*
+                          [::mq.publish/deferred-messages :queue/test]))
+            "No messages should be in transaction state after exception")
+        (mq.tu/flush! test-mq)))))
+
+(deftest outside-transaction-publishes-immediately-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (testing "Outside a transaction, messages are published"
+        (is (nil? app-db.conn/*transaction-state*))
+        (mq/with-queue :queue/test [q]
+          (mq/put q "msg1"))
+        (mq.tu/flush! test-mq)
+        (is (= ["msg1"] @heard))))))
+
+(deftest multiple-queues-in-transaction-test
+  (let [heard-a (atom [])
+        heard-b (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test-a (fn [msg] (swap! heard-a conj msg))
+       :queue/test-b (fn [msg] (swap! heard-b conj msg))}
+      (binding [app-db.conn/*after-commit*  (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (mq/with-queue :queue/test-a [q]
+          (mq/put q "a1")
+          (mq/put q "a2"))
+        (mq/with-queue :queue/test-b [q]
+          (mq/put q "b1"))
+        (testing "Both queues have deferred messages"
+          (is (= ["a1" "a2"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :queue/test-a])))
+          (is (= ["b1"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :queue/test-b]))))
+        (testing "After flush, both queues receive their messages"
+          (mq.publish/flush-deferred-messages!)
+          (mq.tu/flush! test-mq)
+          (is (= ["a1" "a2"] @heard-a))
+          (is (= ["b1"] @heard-b)))))))
+
+(deftest buffering-combines-messages-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      (listener/batch-listen! :queue/test
+                              (fn [batch] (swap! heard into batch))
+                              {:max-batch-messages 50})
+      (binding [publish-buffer/*publish-buffer-ms* 100
+                publish-buffer/*publish-buffer*    (atom {})]
+        (testing "buffered-publish! buffers when *publish-buffer-ms* > 0"
+          (mq.publish/publish! :queue/test ["msg1"])
+          (mq.publish/publish! :queue/test ["msg2"])
+          (is (empty? @heard)
+              "Nothing delivered yet because buffer window hasn't elapsed")
+          (is (= ["msg1" "msg2"]
+                 (:messages (get @publish-buffer/*publish-buffer* :queue/test)))
+              "Messages are buffered"))
+        (testing "After flush, all messages are delivered"
+          (swap! publish-buffer/*publish-buffer*
+                 update :queue/test assoc :deadline-ms 1)
+          (publish-buffer/flush-publish-buffer!)
+          (mq.tu/flush! test-mq)
+          (is (= ["msg1" "msg2"] @heard)
+              "Both messages delivered after flush"))))))
+
+(deftest buffering-immediate-when-zero-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [msg] (swap! heard conj msg))}
+      (binding [publish-buffer/*publish-buffer-ms* 0
+                publish-buffer/*publish-buffer*    (atom {})]
+        (testing "When *publish-buffer-ms* is 0, messages publish without buffering"
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg1"))
+          (mq.tu/flush! test-mq)
+          (is (= ["msg1"] @heard))
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg2"))
+          (mq.tu/flush! test-mq)
+          (is (= ["msg1" "msg2"] @heard)))))))
+
+(deftest buffering-flush-delivers-test
+  (let [heard (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      (listener/batch-listen! :queue/test
+                              (fn [batch] (swap! heard into batch))
+                              {:max-batch-messages 3})
+      (binding [publish-buffer/*publish-buffer-ms*  1
+                publish-buffer/*publish-buffer-max-ms* 0
+                publish-buffer/*publish-buffer*     (atom {})]
+        (testing "Messages are buffered until flushed"
+          (mq.publish/publish! :queue/test ["a" "b"])
+          (is (empty? @heard) "not yet flushed")
+          (mq.publish/publish! :queue/test ["c"])
+          (is (empty? @heard) "still buffered, no proactive flush")
+          (Thread/sleep 5)
+          (publish-buffer/flush-publish-buffer!)
+          (mq.tu/flush! test-mq)
+          (is (= ["a" "b" "c"] @heard) "all delivered after flush"))))))
+
+(deftest fifo-ordering-test
+  (let [received (atom [])]
+    (mq.tu/with-test-mq [test-mq]
+      {:queue/test (fn [message] (swap! received conj message))}
+      (doseq [i (range 10)]
+        (mq/with-queue :queue/test [q]
+          (mq/put q i)))
+      (mq.tu/flush! test-mq)
+      (testing "All messages are delivered in order"
+        (is (= (range 10) @received))))))
+
+(deftest exclusive-queue-single-active-handler-test
+  "Verifies that submit-delivery! returns false for a channel that already has an
+   active handler, enforcing at-most-one concurrent delivery per channel."
+  (mq.impl/start-worker-pool!)
+  (mq.tu/with-test-mq [_test-mq]
+    (let [queue-name :queue/exclusive-concurrency-test
+          latch      (CountDownLatch. 1)]
+      (mq.tu/listen! queue-name {:exclusive true}
+                     (fn [_] (.await latch))) ; Block listener until released
+      (testing "First submission succeeds and marks the channel as busy"
+        (is (true? (mq.impl/submit-delivery! queue-name ["msg1"] nil nil nil)))
+        (Thread/sleep 50) ; Give worker thread time to start and block on latch
+        (is (true? (mq.impl/channel-busy? queue-name))))
+      (testing "Second submission returns false while channel is busy"
+        (is (false? (mq.impl/submit-delivery! queue-name ["msg2"] nil nil nil))))
+      (.countDown latch) ; Release the listener
+      (Thread/sleep 100) ; Wait for delivery to complete and active-handlers to clear
+      (testing "Channel is no longer busy after delivery completes"
+        (is (false? (mq.impl/channel-busy? queue-name))))
+      (mq/unlisten! queue-name))))

test/metabase/mq/queue/memory_test.clj

@@ -0,0 +1,36 @@
+(ns metabase.mq.queue.memory-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.test-util :as mq.tu]))
+
+(set! *warn-on-reflection* true)
+
+(deftest publish-test
+  (mq.tu/with-test-mq [ctx]
+    (let [queue-name (keyword "queue" (str "publish-test-" (gensym)))
+          received   (atom [])]
+      (mq.tu/listen! queue-name {} (fn [msg] (swap! received conj msg)))
+      (testing "Publish lazily creates channel and delivers messages"
+        (mq/with-queue queue-name [q]
+          (mq/put q "test-message")
+          (mq/put q "test-message2"))
+        (mq.tu/flush! ctx)
+        (is (= ["test-message" "test-message2"] @received))))))
+
+(deftest exclusive-memory-test
+  (mq.tu/with-test-mq [ctx]
+    (let [queue-name :queue/exclusive-test
+          received   (atom [])]
+      (mq.tu/listen! queue-name
+                     {:exclusive true}
+                     (fn [msg] (swap! received conj msg)))
+      (testing "Exclusive queue processes all messages"
+        (mq/with-queue queue-name [q]
+          (mq/put q "msg1")
+          (mq/put q "msg2")
+          (mq/put q "msg3")
+          (mq/put q "msg4")
+          (mq/put q "msg5"))
+        (mq.tu/flush! ctx)
+        (is (= ["msg1" "msg2" "msg3" "msg4" "msg5"] @received))))))

test/metabase/mq/shutdown_drain_test.clj

@@ -0,0 +1,64 @@
+(ns metabase.mq.shutdown-drain-test
+  "Tests that `mq.init/stop!` performs a graceful drain of the publish buffer
+  before shutting down backends, so buffered messages still land in the appdb
+  tables rather than being lost."
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.init :as mq.init]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest queue-shutdown-drains-publish-buffer-test
+  (testing "stop! drains the publish buffer for queues so buffered messages land in the appdb table"
+    (let [queue-name (keyword "queue" (str "shutdown-drain-" (gensym)))]
+      (try
+        (binding [listener/*listeners*                   (atom {})
+                  publish-buffer/*publish-buffer*        (atom {})
+                  publish-buffer/*publish-buffer-ms*     60000
+                  publish-buffer/*publish-buffer-max-ms* 0]
+          (let [handle (mq.init/start! :queue.backend/appdb :topic.backend/appdb)]
+            (mq/with-queue queue-name [q]
+              (mq/put q "drain-me"))
+            (is (= ["drain-me"]
+                   (get-in @publish-buffer/*publish-buffer* [queue-name :messages]))
+                "Message should be sitting in the publish buffer before stop!")
+            (is (empty? (t2/select :queue_message_batch :queue_name (name queue-name)))
+                "Buffer drain hasn't happened yet — table should be empty")
+            (mq.init/stop! handle))
+          (let [rows (t2/select :queue_message_batch :queue_name (name queue-name))]
+            (is (= 1 (count rows))
+                "Exactly one row should exist after the graceful shutdown drain")
+            (is (= ["drain-me"] (json/decode (:messages (first rows))))
+                "The buffered message should have been persisted on shutdown")))
+        (finally
+          (t2/delete! :queue_message_batch :queue_name (name queue-name)))))))
+
+(deftest topic-shutdown-drains-publish-buffer-test
+  (testing "stop! drains the publish buffer for topics so buffered messages land in the appdb table"
+    (let [topic-name (keyword "topic" (str "shutdown-drain-" (gensym)))]
+      (try
+        (binding [listener/*listeners*                   (atom {})
+                  publish-buffer/*publish-buffer*        (atom {})
+                  publish-buffer/*publish-buffer-ms*     60000
+                  publish-buffer/*publish-buffer-max-ms* 0]
+          (let [handle (mq.init/start! :queue.backend/appdb :topic.backend/appdb)]
+            (mq/with-topic topic-name [t]
+              (mq/put t "drain-me"))
+            (is (= ["drain-me"]
+                   (get-in @publish-buffer/*publish-buffer* [topic-name :messages]))
+                "Message should be sitting in the publish buffer before stop!")
+            (is (empty? (t2/select :topic_message_batch :topic_name (name topic-name)))
+                "Buffer drain hasn't happened yet — table should be empty")
+            (mq.init/stop! handle))
+          (let [rows (t2/select :topic_message_batch :topic_name (name topic-name))]
+            (is (= 1 (count rows))
+                "Exactly one row should exist after the graceful shutdown drain")
+            (is (= ["drain-me"] (json/decode (:messages (first rows))))
+                "The buffered message should have been persisted on shutdown")))
+        (finally
+          (t2/delete! :topic_message_batch :topic_name (name topic-name)))))))

test/metabase/mq/test_util.clj

@@ -0,0 +1,308 @@
+(ns metabase.mq.test-util
+  "Test fixture for exercising the MQ subsystem in tests. `with-test-mq` starts a
+  fresh async memory backend and exposes a fixture context that carries the layer,
+  the backend instances, and convenience fns (`flush!`, `wait-for-idle!`,
+  `eventually!`) for driving delivery and waiting for quiescence.
+
+  `do-with-test-mq!` also accepts `:backend :appdb` for running the real database-
+  backed backend in parity tests, and `:duplicate-delivery? true` to wrap the
+  backends with a decorator that publishes every message twice. The duplicate-
+  delivery mode exists to force listeners under test to prove they handle the
+  MQ's at-least-once contract — any scenario that silently relies on exactly-
+  once delivery will fail loudly under this mode."
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.init :as mq.init]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.memory :as memory]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.memory :as q.memory]
+   [metabase.mq.topic.appdb :as topic.appdb]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.mq.topic.memory :as topic.memory])
+  (:import
+   (java.util.concurrent LinkedBlockingQueue)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------- Quiescence -------------------------------------------
+
+(defn- layer-channels-empty?
+  "Returns true if every memory channel that currently has a listener is empty.
+  Channels with no listener are not counted — messages published after an
+  `unlisten!` are orphaned by design and would otherwise block teardown.
+  Returns true for non-memory fixtures (appdb) since there is no layer."
+  [{:keys [layer]}]
+  (if (nil? layer)
+    true
+    (let [listeners @listener/*listeners*]
+      (every? (fn [[channel-name ^LinkedBlockingQueue q]]
+                (or (not (contains? listeners channel-name))
+                    (.isEmpty q)))
+              @(:channels layer)))))
+
+(defn- unmet-idle-conditions
+  "Returns a set of the idle conditions that currently do NOT hold. Empty set == idle."
+  [ctx]
+  (cond-> #{}
+    (seq @publish-buffer/*publish-buffer*) (conj :publish-buffer-nonempty)
+    (seq (mq.impl/busy-channels))          (conj :busy-channels)
+    (not (layer-channels-empty? ctx))      (conj :memory-channels-nonempty)))
+
+(defn wait-for-idle!
+  "Blocks until the MQ fixture reaches quiescence or `timeout-ms` elapses. Throws
+  an ex-info whose `:unmet` key names the conditions still holding on timeout.
+
+  Because `poll-once!` drains memory channels *before* setting `active-handlers`
+  on the next submit-delivery, a single idle observation can occur in the gap
+  between the two. We require two consecutive idle observations 20 ms apart."
+  ([ctx] (wait-for-idle! ctx 5000))
+  ([ctx timeout-ms]
+   (let [deadline (+ (System/currentTimeMillis) timeout-ms)]
+     (loop [consecutive-idle 0]
+       ;; Wake every active polling thread on each tick so appdb backends drain
+       ;; immediately instead of waiting on their 5s default poll interval.
+       (mq.polling/notify-all!)
+       (let [unmet (unmet-idle-conditions ctx)]
+         (cond
+           (and (empty? unmet) (>= consecutive-idle 1)) :idle
+           (empty? unmet) (do (Thread/sleep 20) (recur (inc consecutive-idle)))
+           (< (System/currentTimeMillis) deadline) (do (Thread/sleep 10) (recur 0))
+           :else (throw (ex-info "MQ did not reach idle state before timeout"
+                                 {:timeout-ms timeout-ms :unmet unmet}))))))))
+
+(defn- force-expire-publish-buffer!
+  "Marks every entry in the publish buffer as past its deadline so the next
+  `flush-publish-buffer!` call drains it immediately."
+  []
+  (swap! publish-buffer/*publish-buffer*
+         (fn [buf]
+           (into {} (map (fn [[k v]] [k (assoc v :deadline-ms 1)])) buf))))
+
+(defn flush!
+  "Force-drains the publish buffer and waits until the MQ is idle. This is the
+  standard 'move time forward' primitive for tests: publish things, call `flush!`,
+  then assert on the resulting state."
+  ([ctx] (flush! ctx 5000))
+  ([ctx timeout-ms]
+   (force-expire-publish-buffer!)
+   (publish-buffer/flush-publish-buffer!)
+   (wait-for-idle! ctx timeout-ms)))
+
+(defn eventually!
+  "Polls `pred` until truthy or `timeout-ms` elapses. Returns the final value of
+  `pred`. Use when the thing you're waiting on can't be expressed as 'MQ is idle'
+  — for example, when waiting for a specific count to be reached or for
+  appdb-backed delivery to round-trip through the database."
+  ([_ctx pred] (eventually! _ctx pred 2000))
+  ([_ctx pred timeout-ms]
+   (let [deadline (+ (System/currentTimeMillis) timeout-ms)]
+     (loop []
+       ;; Wake any polling threads so appdb backends drain promptly instead of
+       ;; blocking on their production poll interval.
+       (mq.polling/notify-all!)
+       (let [v (pred)]
+         (if v
+           v
+           (if (< (System/currentTimeMillis) deadline)
+             (do (Thread/sleep 10) (recur))
+             v)))))))
+
+;;; ------------------------------------------- Fixture -------------------------------------------
+
+(defn listen!
+  "Test helper — registers a per-message listener by wrapping `listener-fn` so it's called
+  once per message in each batch, with error isolation. Production code should use
+  [[metabase.mq.core/def-listener!]] instead; this is for ad-hoc test registration where
+  declaring at namespace-load time isn't convenient."
+  [channel opts listener-fn]
+  (listener/batch-listen! channel
+                          (fn [messages]
+                            (let [error (volatile! nil)]
+                              (doseq [m messages]
+                                (try (listener-fn m)
+                                     (catch Throwable e (vreset! error e))))
+                              (when-let [e @error] (throw e))))
+                          (or opts {})))
+
+(defn- merge-listeners!
+  "Merges a user-supplied listener map on top of whatever listeners
+  `register-listeners!` already registered. Values can be plain single-message
+  fns (wrapped to match the vec-of-messages contract that `handle!` expects)
+  or full config maps with `:listener` etc. Map-form values are passed through
+  as-is — their `:listener` must already accept a vec of messages."
+  [listeners]
+  (when (seq listeners)
+    (swap! listener/*listeners*
+           (fn [current]
+             (reduce-kv (fn [m k v]
+                          (assoc m k (if (fn? v)
+                                       (if-let [existing (get current k)]
+                                         (assoc existing :listener (partial run! v))
+                                         {:listener           (partial run! v)
+                                          :max-batch-messages 1})
+                                       v)))
+                        current
+                        listeners)))))
+
+(defn- make-fixture-backends
+  "Constructs backend instances for the given fixture kind. Memory gets a fresh
+  isolated layer; appdb shares the DB table but gets its own per-instance state
+  (poll-state, owner-id, offsets, periodic-task rate-limit atoms) so that
+  test teardown doesn't tear down the production singleton's polling thread."
+  [kind]
+  (case kind
+    :memory (let [layer (memory/make-layer)]
+              {:backend  :memory
+               :layer    layer
+               :queue-be (q.memory/make-backend layer)
+               :topic-be (topic.memory/make-backend layer)})
+    :appdb  {:backend  :appdb
+             :queue-be (q.appdb/make-backend)
+             :topic-be (topic.appdb/make-backend)}))
+
+(defn- double-delivery-queue-backend!
+  "Wraps an inner `QueueBackend` so that `publish!` calls the inner backend's
+  `publish!` twice with the same payload. Used by `do-with-test-mq!` when
+  `:duplicate-delivery?` is set, to force listeners under test to handle the
+  MQ's at-least-once contract."
+  [inner]
+  (reify q.backend/QueueBackend
+    (publish! [_ queue-name messages]
+      (q.backend/publish! inner queue-name messages)
+      (q.backend/publish! inner queue-name messages))
+    (batch-successful! [_ queue-name batch-id]
+      (q.backend/batch-successful! inner queue-name batch-id))
+    (batch-failed! [_ queue-name batch-id]
+      (q.backend/batch-failed! inner queue-name batch-id))
+    (start! [_] (q.backend/start! inner))
+    (shutdown! [_] (q.backend/shutdown! inner))))
+
+(defn- double-delivery-topic-backend!
+  "Wraps an inner `TopicBackend` so that `publish!` calls the inner backend's
+  `publish!` twice with the same payload. Used by `do-with-test-mq!` when
+  `:duplicate-delivery?` is set."
+  [inner]
+  (reify topic.backend/TopicBackend
+    (publish! [_ topic-name messages]
+      (topic.backend/publish! inner topic-name messages)
+      (topic.backend/publish! inner topic-name messages))
+    (start-receiving! [_ topic-name]
+      (topic.backend/start-receiving! inner topic-name))
+    (unsubscribe! [_ topic-name]
+      (topic.backend/unsubscribe! inner topic-name))
+    (start-handling! [_] (topic.backend/start-handling! inner))
+    (shutdown! [_] (topic.backend/shutdown! inner))))
+
+(defn do-with-test-mq!
+  "Function form of [[with-test-mq]]. `opts` may include:
+
+  - `:backend`                 `:memory` (default) or `:appdb`
+  - `:listeners`            listener map, same shape as `with-sync-mq` had
+  - `:duplicate-delivery?`  when true, wraps both backends with a decorator
+                            that publishes every message twice — used to
+                            verify listeners handle at-least-once delivery
+
+  The fixture forces `*publish-buffer-ms*` to 0 so publishes go straight to the
+  backend without waiting for the background flush scheduler. Tests that want to
+  exercise buffering behaviour must rebind it explicitly in the body."
+  ([f] (do-with-test-mq! {} f))
+  ([opts f]
+   (let [{:keys [backend listeners duplicate-delivery?] :or {backend :memory}} opts]
+     (binding [listener/*listeners*               (atom {})
+               publish-buffer/*publish-buffer*    (atom {})
+               publish-buffer/*publish-buffer-ms* 0]
+       (let [backends (make-fixture-backends backend)
+             queue-be (cond-> (:queue-be backends)
+                        duplicate-delivery? double-delivery-queue-backend!)
+             topic-be (cond-> (:topic-be backends)
+                        duplicate-delivery? double-delivery-topic-backend!)
+             handle   (mq.init/start! queue-be topic-be)
+             ctx      (assoc backends :queue-be queue-be :topic-be topic-be :handle handle)]
+         (try
+           (merge-listeners! listeners)
+           (let [result (f ctx)]
+             (flush! ctx)
+             result)
+           (finally
+             (mq.init/stop! handle))))))))
+
+(defmacro with-test-mq
+  "Starts an MQ subsystem, runs body with `ctx-sym` bound to a fixture context
+  map, then drains and shuts down.
+
+  The binding vector is `[ctx-sym]` or `[ctx-sym opts-map]` where `opts-map`
+  may contain `:backend`, `:duplicate-delivery?`, etc. — anything
+  [[do-with-test-mq!]] accepts.
+
+  An optional listener map may follow the binding vector (same shape as the
+  old `with-sync-mq`). It is merged into the opts under `:listeners`.
+
+  Use `(mq.tu/flush! ctx)` between publishes and assertions, and
+  `(mq.tu/wait-for-idle! ctx)` / `(mq.tu/eventually! ctx pred)` where finer
+  control is needed.
+
+  Examples:
+
+      (with-test-mq [ctx]
+        body)
+
+      (with-test-mq [ctx]
+        {:queue/foo (fn [msg] ...)}
+        body)
+
+      (with-test-mq [ctx {:backend :appdb :duplicate-delivery? true}]
+        body)"
+  {:style/indent 1}
+  [[ctx-sym & [opts-expr]] & body]
+  (let [[listeners body] (if (map? (first body))
+                           [(first body) (rest body)]
+                           [nil body])]
+    `(do-with-test-mq! (merge ~opts-expr {:listeners ~listeners})
+                       (fn [~ctx-sym] ~@body))))
+
+;;; ------------------------------------------- Fixture self-tests -------------------------------------------
+
+(deftest duplicate-delivery-doubles-queue-messages-test
+  (testing "With :duplicate-delivery? true, the queue backend delivers each message twice"
+    (with-test-mq [ctx {:duplicate-delivery? true}]
+      (let [received (atom [])]
+        (listen! :queue/chaos-smoke {} #(swap! received conj %))
+        (mq/with-queue :queue/chaos-smoke [q]
+          (mq/put q "hello"))
+        (eventually! ctx #(= 2 (count @received)) 5000)
+        (is (= ["hello" "hello"] @received)
+            "Single published message is delivered to the listener twice")
+        (mq/unlisten! :queue/chaos-smoke)))))
+
+(deftest duplicate-delivery-doubles-topic-messages-test
+  (testing "With :duplicate-delivery? true, the topic backend delivers each message twice"
+    (with-test-mq [ctx {:duplicate-delivery? true}]
+      (let [received (atom [])]
+        (listen! :topic/chaos-smoke {} #(swap! received conj %))
+        (mq/with-topic :topic/chaos-smoke [t]
+          (mq/put t "ping"))
+        (eventually! ctx #(= 2 (count @received)) 5000)
+        (is (= ["ping" "ping"] @received)
+            "Single published topic message is delivered to the subscriber twice")
+        (mq/unlisten! :topic/chaos-smoke)))))
+
+(deftest duplicate-delivery-off-by-default-test
+  (testing "The default fixture delivers each message exactly once"
+    (with-test-mq [ctx]
+      (let [received (atom [])]
+        (listen! :queue/chaos-default {} #(swap! received conj %))
+        (mq/with-queue :queue/chaos-default [q]
+          (mq/put q "once"))
+        (eventually! ctx #(>= (count @received) 1) 5000)
+        ;; Give the poll thread an extra moment in case a second (unexpected)
+        ;; delivery is on the way, then confirm exactly one happened.
+        (Thread/sleep 100)
+        (is (= ["once"] @received)
+            "Default fixture delivers exactly once")
+        (mq/unlisten! :queue/chaos-default)))))

test/metabase/mq/topic/appdb_test.clj

@@ -0,0 +1,126 @@
+(ns metabase.mq.topic.appdb-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.analytics-interface.core :as analytics]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.test-util :as mq.tu]
+   [metabase.mq.topic.appdb :as topic.appdb]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(deftest publish-test
+  (let [topic-name :topic/publish-test]
+    (topic.backend/publish! topic.appdb/backend topic-name ["test message"])
+    (testing "Message is persisted in topic_message"
+      (let [row (t2/select-one :topic_message_batch :topic_name (name topic-name))]
+        (is (some? row))
+        (is (= (name topic-name) (:topic_name row)))
+        (is (= ["test message"] (json/decode (:messages row))))
+        (is (some? (:created_at row)))))
+    ;; cleanup
+    (t2/delete! :topic_message_batch :topic_name (name topic-name))))
+
+(deftest batch-publish-test
+  (let [topic-name :topic/batch-publish-test]
+    (topic.backend/publish! topic.appdb/backend topic-name ["msg-1" "msg-2" "msg-3"])
+    (testing "Batch of messages stored in single row"
+      (let [row (t2/select-one :topic_message_batch :topic_name (name topic-name))]
+        (is (= ["msg-1" "msg-2" "msg-3"] (json/decode (:messages row))))))
+    ;; cleanup
+    (t2/delete! :topic_message_batch :topic_name (name topic-name))))
+
+(deftest subscribe-and-receive-test
+  (let [topic-name (keyword "topic" (str "sub-receive-test-" (random-uuid)))
+        received   (atom [])]
+    (try
+      ;; Register listener (this also subscribes, setting the offset via the current *backend*)
+      (binding [topic.backend/*backend* topic.appdb/backend]
+        (mq.tu/listen! topic-name
+                       {}
+                       (fn [message]
+                         (swap! received conj message))))
+      ;; Publish then deliver synchronously via poll-iteration! + deliver!
+      (topic.backend/publish! topic.appdb/backend topic-name ["hello-appdb"])
+      ;; poll-iteration! fetches rows and calls submit-delivery! which is async.
+      ;; Instead, poll the messages and deliver synchronously on this thread.
+      (let [offset (or (get @(:offsets topic.appdb/backend) topic-name) 0)
+            rows   (#'topic.appdb/poll-messages! topic-name offset)]
+        (when (seq rows)
+          (let [all-messages (into [] (mapcat (comp json/decode :messages)) rows)]
+            (mq.impl/deliver! topic-name all-messages nil nil))))
+
+      (testing "Subscriber receives the published message"
+        (is (= ["hello-appdb"] @received)))
+      (finally
+        (listener/unlisten! topic-name)
+        (t2/delete! :topic_message_batch :topic_name (name topic-name))))))
+
+(deftest no-messages-without-subscribers-test
+  (let [topic-name :topic/no-sub-test]
+    (topic.backend/publish! topic.appdb/backend topic-name ["orphan-msg"])
+    (testing "Message exists in table even with no subscribers"
+      (is (pos? (t2/count :topic_message_batch :topic_name (name topic-name)))))
+    ;; cleanup
+    (t2/delete! :topic_message_batch :topic_name (name topic-name))))
+
+(deftest cleanup-old-messages-test
+  (let [topic-name (str "cleanup-test-" (random-uuid))
+        old-ts     (Timestamp/from (.minusMillis (Instant/now) (+ (* 60 60 1000) 60000)))]
+    (try
+      (testing "Topic messages older than 1 hour are deleted"
+        (t2/query {:insert-into :topic_message_batch
+                   :values      [{:topic_name topic-name
+                                  :messages   (json/encode ["old-msg"])
+                                  :created_at old-ts}]})
+        (let [old-row (t2/select-one :topic_message_batch :topic_name topic-name)]
+          (#'topic.appdb/cleanup-old-messages!)
+          (is (nil? (t2/select-one :topic_message_batch :id (:id old-row)))
+              "Old topic message should be deleted")))
+
+      (testing "Recent topic messages are not deleted"
+        (t2/insert! :topic_message_batch {:topic_name topic-name :messages (json/encode ["recent-msg"])})
+        (let [recent-row (t2/select-one :topic_message_batch :topic_name topic-name)]
+          (#'topic.appdb/cleanup-old-messages!)
+          (is (some? (t2/select-one :topic_message_batch :id (:id recent-row)))
+              "Recent topic message should not be deleted")))
+      (finally
+        (t2/delete! :topic_message_batch :topic_name topic-name)))))
+
+(deftest lag-gauge-test
+  (let [topic-name  (keyword "topic" (str "lag-gauge-test-" (random-uuid)))
+        gauge-calls (atom [])]
+    (try
+      ;; Publish two messages and look up their IDs from the DB
+      (topic.backend/publish! topic.appdb/backend topic-name ["m1"])
+      (let [id1 (:id (first (t2/query {:select   [:id]
+                                       :from     [:topic_message_batch]
+                                       :where    [:= :topic_name (name topic-name)]
+                                       :order-by [[:id :desc]]
+                                       :limit    1})))]
+        ;; Subscriber has read up to id1
+        (swap! (:offsets topic.appdb/backend) assoc topic-name id1)
+        ;; Publish a second message (unread)
+        (topic.backend/publish! topic.appdb/backend topic-name ["m2"])
+        (let [id2 (:id (first (t2/query {:select   [:id]
+                                         :from     [:topic_message_batch]
+                                         :where    [:= :topic_name (name topic-name)]
+                                         :order-by [[:id :desc]]
+                                         :limit    1})))]
+          (with-redefs [analytics/set-gauge! (fn [metric labels value]
+                                               (when (= metric :metabase-mq/appdb-topic-subscriber-lag)
+                                                 (swap! gauge-calls conj {:labels labels :value value})))]
+            (#'topic.appdb/update-lag-gauges! (:offsets topic.appdb/backend)))
+          (testing "Lag gauge is emitted with the correct unread count"
+            (let [call (first (filter #(= (name topic-name) (-> % :labels :channel)) @gauge-calls))]
+              (is (some? call) "Gauge should be recorded for the test topic")
+              (is (= (- id2 id1) (:value call))
+                  "Lag should equal the difference between max-id and current offset")))))
+      (finally
+        (swap! (:offsets topic.appdb/backend) dissoc topic-name)
+        (t2/delete! :topic_message_batch :topic_name (name topic-name))))))

test/metabase/mq/topic/core_test.clj

@@ -0,0 +1,271 @@
+(ns metabase.mq.topic.core-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.app-db.connection :as app-db.conn]
+   [metabase.mq.core :as mq]
+   [metabase.mq.publish :as mq.publish]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.test-util :as mq.tu])
+  (:import (clojure.lang ExceptionInfo)
+           (java.util.concurrent CyclicBarrier)))
+
+(set! *warn-on-reflection* true)
+
+(deftest e2e-publish-subscribe-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/e2e {}
+                     (fn [message]
+                       (swap! received conj message)))
+
+      (testing "Messages are received by subscriber"
+        (mq/with-topic :topic/e2e [t]
+          (mq/put t "message-1"))
+        (mq/with-topic :topic/e2e [t]
+          (mq/put t "message-2"))
+        (mq.tu/flush! ctx)
+        (is (= ["message-1" "message-2"] @received)))
+
+      (testing "Unsubscribe stops delivery"
+        (mq/unlisten! :topic/e2e)
+        (mq/with-topic :topic/e2e [t]
+          (mq/put t "message-3"))
+        (mq.tu/flush! ctx)
+        (is (= ["message-1" "message-2"] @received))))))
+
+(deftest batch-publish-e2e-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/batch {}
+                     (fn [message]
+                       (swap! received conj message)))
+
+      (mq/with-topic :topic/batch [t]
+        (mq/put t "a")
+        (mq/put t "b")
+        (mq/put t "c"))
+      (mq.tu/flush! ctx)
+
+      (testing "Batch of messages delivered together"
+        (is (= ["a" "b" "c"] @received)))
+
+      (mq/unlisten! :topic/batch))))
+
+(deftest error-handling-e2e-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/errors {}
+                     (fn [message]
+                       (when (= "fail" message)
+                         (throw (ex-info "Handler error" {})))
+                       (swap! received conj message)))
+
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "ok-1"))
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "fail"))
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "ok-2"))
+      (mq.tu/flush! ctx)
+
+      (testing "Non-error messages are delivered"
+        (is (= ["ok-1" "ok-2"] @received)))
+
+      (mq/unlisten! :topic/errors))))
+
+(deftest batch-partial-failure-test
+  (testing "When listener throws on one message in a batch, remaining messages are still delivered"
+    (mq.tu/with-test-mq [ctx]
+      (let [received (atom [])]
+        (mq.tu/listen! :topic/batch-fail {}
+                       (fn [message]
+                         (when (= "boom" message)
+                           (throw (ex-info "Handler error" {})))
+                         (swap! received conj message)))
+
+        (mq/with-topic :topic/batch-fail [t]
+          (mq/put t "msg-1")
+          (mq/put t "boom")
+          (mq/put t "msg-3")
+          (mq/put t "msg-4"))
+        (mq.tu/flush! ctx)
+
+        (is (= ["msg-1" "msg-3" "msg-4"] @received))
+
+        (mq/unlisten! :topic/batch-fail)))))
+
+(deftest concurrent-subscribe-throws-test
+  (mq.tu/with-test-mq [_ctx]
+    (let [topic-name :topic/concurrent-sub-test
+          n          10
+          barrier    (CyclicBarrier. n)
+          results    (atom [])]
+      (testing "Concurrent listen! calls: exactly one succeeds, rest throw"
+        (let [threads (mapv (fn [_]
+                              (let [f (bound-fn []
+                                        (.await barrier)
+                                        (try
+                                          (mq.tu/listen! topic-name {} (fn [_] nil))
+                                          (swap! results conj :ok)
+                                          (catch ExceptionInfo _
+                                            (swap! results conj :error))))]
+                                (Thread. ^Runnable f)))
+                            (range n))]
+          (run! (fn [^Thread t] (.start t)) threads)
+          (run! (fn [^Thread t] (.join t 5000)) threads)
+          (is (= 1 (count (filter #{:ok} @results))))
+          (is (= (dec n) (count (filter #{:error} @results))))))
+      (mq/unlisten! topic-name))))
+
+(deftest late-subscriber-test
+  (mq.tu/with-test-mq [ctx]
+    (mq/with-topic :topic/late [t]
+      (mq/put t "old-message"))
+    (mq.tu/flush! ctx)
+
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/late {}
+                     (fn [message]
+                       (swap! received conj message)))
+
+      (mq/with-topic :topic/late [t]
+        (mq/put t "new-message"))
+      (mq.tu/flush! ctx)
+
+      (testing "Late subscriber only sees new messages"
+        (is (= ["new-message"] @received)))
+
+      (mq/unlisten! :topic/late))))
+
+(deftest transaction-defers-topic-publish-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/txn-test {}
+                     (fn [message]
+                       (swap! received conj message)))
+      (binding [app-db.conn/*after-commit*      (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (testing "Inside a transaction, messages are accumulated, not published immediately"
+          (mq/with-topic :topic/txn-test [t]
+            (mq/put t "msg1")
+            (mq/put t "msg2"))
+          (mq/with-topic :topic/txn-test [t]
+            (mq/put t "msg3"))
+          (testing "Messages are deferred in transaction state"
+            (is (= ["msg1" "msg2" "msg3"]
+                   (get-in @app-db.conn/*transaction-state*
+                           [::mq.publish/deferred-messages :topic/txn-test]))))
+          (testing "Listener has not been called yet"
+            (is (= [] @received))))
+        (testing "After flush (simulating commit), all messages are delivered"
+          (mq.publish/flush-deferred-messages!)
+          (mq.tu/flush! ctx)
+          (is (= ["msg1" "msg2" "msg3"] @received))))
+      (mq/unlisten! :topic/txn-test))))
+
+(deftest transaction-rollback-discards-topic-messages-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/txn-rollback {}
+                     (fn [message]
+                       (swap! received conj message)))
+      (binding [app-db.conn/*after-commit*      (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (testing "Messages accumulated during a failed transaction are discarded"
+          (is (thrown? Exception
+                       (mq/with-topic :topic/txn-rollback [t]
+                         (mq/put t "should-be-discarded")
+                         (throw (ex-info "boom" {})))))
+          (is (nil? (get-in @app-db.conn/*transaction-state*
+                            [::mq.publish/deferred-messages :topic/txn-rollback]))
+              "No messages should be in transaction state after exception")
+          (mq.tu/flush! ctx)))
+      (mq/unlisten! :topic/txn-rollback))))
+
+(deftest outside-transaction-publishes-topic-immediately-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/txn-immediate {}
+                     (fn [message]
+                       (swap! received conj message)))
+      (testing "Outside a transaction, messages are published"
+        (is (nil? app-db.conn/*transaction-state*))
+        (mq/with-topic :topic/txn-immediate [t]
+          (mq/put t "msg1"))
+        (mq.tu/flush! ctx)
+        (is (= ["msg1"] @received)))
+      (mq/unlisten! :topic/txn-immediate))))
+
+(deftest multiple-topics-in-transaction-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received-a (atom [])
+          received-b (atom [])]
+      (mq.tu/listen! :topic/txn-multi-a {}
+                     (fn [message] (swap! received-a conj message)))
+      (mq.tu/listen! :topic/txn-multi-b {}
+                     (fn [message] (swap! received-b conj message)))
+      (binding [app-db.conn/*after-commit*      (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (mq/with-topic :topic/txn-multi-a [t]
+          (mq/put t "a1")
+          (mq/put t "a2"))
+        (mq/with-topic :topic/txn-multi-b [t]
+          (mq/put t "b1"))
+        (testing "Both topics have deferred messages"
+          (is (= ["a1" "a2"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :topic/txn-multi-a])))
+          (is (= ["b1"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :topic/txn-multi-b]))))
+        (testing "After flush, both topics receive their messages"
+          (mq.publish/flush-deferred-messages!)
+          (mq.tu/flush! ctx)
+          (is (= ["a1" "a2"] @received-a))
+          (is (= ["b1"] @received-b))))
+      (mq/unlisten! :topic/txn-multi-a)
+      (mq/unlisten! :topic/txn-multi-b))))
+
+(deftest buffering-combines-topic-messages-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/buffer-test {}
+                     (fn [message] (swap! received conj message)))
+      (binding [publish-buffer/*publish-buffer-ms* 100
+                publish-buffer/*publish-buffer*    (atom {})]
+        (testing "buffered-publish! buffers topic messages when *publish-buffer-ms* > 0"
+          (mq/with-topic :topic/buffer-test [t]
+            (mq/put t "msg1"))
+          (mq/with-topic :topic/buffer-test [t]
+            (mq/put t "msg2"))
+          (is (empty? @received)
+              "Nothing delivered yet because buffer window hasn't elapsed")
+          (is (= ["msg1" "msg2"]
+                 (:messages (get @publish-buffer/*publish-buffer* :topic/buffer-test)))
+              "Messages are buffered"))
+        (testing "After flush, all messages are delivered"
+          (swap! publish-buffer/*publish-buffer*
+                 update :topic/buffer-test assoc :deadline-ms 1)
+          (publish-buffer/flush-publish-buffer!)
+          (mq.tu/flush! ctx)
+          (is (= ["msg1" "msg2"] @received)
+              "Both messages delivered after flush")))
+      (mq/unlisten! :topic/buffer-test))))
+
+(deftest buffering-immediate-when-zero-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/buffer-zero {}
+                     (fn [message] (swap! received conj message)))
+      (binding [publish-buffer/*publish-buffer-ms* 0
+                publish-buffer/*publish-buffer*    (atom {})]
+        (testing "With *publish-buffer-ms* 0, topic messages publish without buffering"
+          (mq/with-topic :topic/buffer-zero [t]
+            (mq/put t "msg1"))
+          (mq.tu/flush! ctx)
+          (is (= ["msg1"] @received))
+          (mq/with-topic :topic/buffer-zero [t]
+            (mq/put t "msg2"))
+          (mq.tu/flush! ctx)
+          (is (= ["msg1" "msg2"] @received))))
+      (mq/unlisten! :topic/buffer-zero))))

test/metabase/mq/topic/memory_test.clj

@@ -0,0 +1,145 @@
+(ns metabase.mq.topic.memory-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.test-util :as mq.tu])
+  (:import
+   (clojure.lang ExceptionInfo)
+   (java.util.concurrent CyclicBarrier)))
+
+(set! *warn-on-reflection* true)
+
+(deftest publish-and-subscribe-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/test {}
+                     (fn [message]
+                       (swap! received conj message)))
+      (mq/with-topic :topic/test [t]
+        (mq/put t "hello"))
+      (mq/with-topic :topic/test [t]
+        (mq/put t "world"))
+      (mq.tu/flush! ctx)
+
+      (testing "Subscriber receives published messages"
+        (is (= ["hello" "world"] @received)))
+
+      (mq/unlisten! :topic/test))))
+
+(deftest batch-publish-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/batch {}
+                     (fn [message]
+                       (swap! received conj message)))
+      (mq/with-topic :topic/batch [t]
+        (mq/put t "msg-1")
+        (mq/put t "msg-2")
+        (mq/put t "msg-3"))
+      (mq.tu/flush! ctx)
+
+      (testing "Batch of messages received in one row"
+        (is (= ["msg-1" "msg-2" "msg-3"] @received)))
+
+      (mq/unlisten! :topic/batch))))
+
+(deftest unsubscribe-stops-delivery-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/unsub {}
+                     (fn [message]
+                       (swap! received conj message)))
+
+      (mq/with-topic :topic/unsub [t]
+        (mq/put t "before"))
+      (mq.tu/flush! ctx)
+      (is (= ["before"] @received))
+
+      (mq/unlisten! :topic/unsub)
+
+      (mq/with-topic :topic/unsub [t]
+        (mq/put t "after"))
+      (mq.tu/flush! ctx)
+
+      (testing "No messages received after unsubscribe"
+        (is (= ["before"] @received))))))
+
+(deftest error-handling-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received (atom [])]
+      (mq.tu/listen! :topic/errors {}
+                     (fn [message]
+                       (when (= "error!" message)
+                         (throw (ex-info "Test error" {})))
+                       (swap! received conj message)))
+
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "good"))
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "error!"))
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "also-good"))
+      (mq.tu/flush! ctx)
+
+      (testing "Good messages are received"
+        (is (= ["good" "also-good"] @received)))
+
+      (mq/unlisten! :topic/errors))))
+
+(deftest double-subscribe-throws-test
+  (mq.tu/with-test-mq [_ctx]
+    (mq.tu/listen! :topic/double {} (fn [_] nil))
+    (testing "Subscribing twice to the same topic throws"
+      (is (thrown-with-msg? ExceptionInfo #"Listener already registered"
+                            (mq.tu/listen! :topic/double {} (fn [_] nil)))))
+    (mq/unlisten! :topic/double)))
+
+(deftest concurrent-publish-ordering-test
+  (testing "Concurrent publishes are all delivered"
+    (mq.tu/with-test-mq [ctx]
+      (let [received (atom [])
+            n        20
+            barrier  (CyclicBarrier. n)]
+        (mq.tu/listen! :topic/concurrent-order {}
+                       (fn [message]
+                         (swap! received conj message)))
+        ;; Publish concurrently from n threads
+        (let [threads (mapv (fn [i]
+                              (let [f (bound-fn []
+                                        (.await barrier)
+                                        (mq/with-topic :topic/concurrent-order [t]
+                                          (mq/put t (str "msg-" i))))]
+                                (Thread. ^Runnable f)))
+                            (range n))]
+          (run! (fn [^Thread t] (.start t)) threads)
+          (run! (fn [^Thread t] (.join t 5000)) threads))
+        (mq.tu/flush! ctx)
+        (testing "All messages delivered exactly once"
+          (is (= n (count @received))))
+        (testing "No duplicate messages"
+          (is (= (count @received) (count (distinct @received)))))
+        (mq/unlisten! :topic/concurrent-order)))))
+
+(deftest topic-isolation-test
+  (mq.tu/with-test-mq [ctx]
+    (let [received-a (atom [])
+          received-b (atom [])]
+      (mq.tu/listen! :topic/isolated-a {}
+                     (fn [message]
+                       (swap! received-a conj message)))
+      (mq.tu/listen! :topic/isolated-b {}
+                     (fn [message]
+                       (swap! received-b conj message)))
+
+      (mq/with-topic :topic/isolated-a [t]
+        (mq/put t "for-a"))
+      (mq/with-topic :topic/isolated-b [t]
+        (mq/put t "for-b"))
+      (mq.tu/flush! ctx)
+
+      (testing "Messages on topic A don't appear on topic B"
+        (is (= ["for-a"] @received-a))
+        (is (= ["for-b"] @received-b)))
+
+      (mq/unlisten! :topic/isolated-a)
+      (mq/unlisten! :topic/isolated-b))))

test/metabase/test/initialize.clj

@@ -93,9 +93,17 @@
   ((resolve 'metabase.test.initialize.plugins/init-test-drivers!)
    [:driver-deprecation-test-legacy :driver-deprecation-test-new :secret-test-driver]))
 
+;; Initialize the MQ subsystem: sync backends, no buffering, register listeners.
+;; Must happen before :db so that listeners (e.g. connection-pool-invalidated) are available
+;; when database initialization triggers events.
+(define-initialization :mq
+  (classloader/require 'metabase.test.initialize.mq)
+  ((resolve 'metabase.test.initialize.mq/init!)))
+
 ;; initializing the DB also does setup needed so the scheduler will work correctly. (Remember that the scheduler uses
 ;; a JDBC backend!)
 (define-initialization :db
+  (initialize-if-needed! :mq)
   (classloader/require 'metabase.test.initialize.db)
   ((resolve 'metabase.test.initialize.db/init!)))
 

test/metabase/test/initialize/mq.clj

@@ -0,0 +1,17 @@
+(ns metabase.test.initialize.mq
+  "Initializes the MQ subsystem for test mode: memory backends, zero publish
+  buffering, and registering all `def-listener!` implementations."
+  (:require
+   [metabase.mq.init :as mq.init]
+   [metabase.mq.publish-buffer :as publish-buffer]))
+
+(defn init! []
+  (alter-var-root #'publish-buffer/*publish-buffer-ms* (constantly 0))
+  ;; Load driver.init so namespaces containing def-listener! calls are loaded
+  ;; before register-listeners! (called inside mq.init/start!) iterates them.
+  ;; In production, core/init.clj loads these.
+  (require 'metabase.driver.init)
+  ;; Drive through mq.init/start! so the worker pool, publish-buffer flush thread,
+  ;; and the memory backend's polling loop all come up — otherwise MQ-driven behaviour
+  ;; outside `with-test-mq` would silently drop messages onto a non-polling queue.
+  (mq.init/start! :queue.backend/memory :topic.backend/memory))