fixes

- ProposalCard.tsx: drop the unreachable "rewrite+index" case in
  kindLabel — ProposalKind was narrowed to "rewrite" | "index" |
  "precompute" earlier in the branch.
- TargetIndexesSection.tsx: wrap RTK Query's unknown-typed error in
  Boolean(error) so JSX accepts the conditional render.
- RunStatus.tsx: render the duration parens as plain JSX instead of
  trying to translate "(${duration})". ttag rejects messages whose
  translatable surface is just punctuation around an interpolation.
  Drop the now-unused `c` import.

.clj-kondo/config/modules/config.edn

@@ -2251,6 +2251,7 @@
   transforms
   {:team "Graphy"
    :api  #{metabase.transforms.core
+           metabase.transforms.crud
            metabase.transforms.feature-gating
            metabase.transforms.init
            metabase.transforms.schema
@@ -3403,7 +3404,11 @@
 
   enterprise/transforms-inspector
   {:team "Gadget"
-   :api  #{metabase-enterprise.transforms-inspector.api}
+   :api  #{metabase-enterprise.transforms-inspector.api
+           ;; Exposed for transform-optimizer (Phase 1 context builder reuses
+           ;; build-context + analyze-native-query). Keep API surface narrow.
+           metabase-enterprise.transforms-inspector.context
+           metabase-enterprise.transforms-inspector.query-analysis}
    :uses #{analytics-interface
            api
            driver
@@ -3422,6 +3427,27 @@
                     :model/Table
                     :model/Transform}}
 
+  enterprise/transform-optimizer
+  {:team "Gadget"
+   :api  #{metabase-enterprise.transform-optimizer.api
+           metabase-enterprise.transform-optimizer.init}
+   :uses #{api
+           driver
+           enterprise/transforms-inspector
+           events
+           lib
+           metabot
+           query-processor
+           server
+           transforms
+           transforms-base
+           util}
+   :model-imports #{:model/Database
+                    :model/Field
+                    :model/Table
+                    :model/Transform
+                    :model/TransformRun}}
+
   enterprise/transforms-python
   {:team "Graphy"
    :api  #{metabase-enterprise.transforms-python.api

enterprise/backend/src/metabase_enterprise/api_routes/routes.clj

@@ -39,6 +39,7 @@
    [metabase-enterprise.stale.api]
    [metabase-enterprise.support-access-grants.api]
    [metabase-enterprise.tenants.api]
+   [metabase-enterprise.transform-optimizer.api]
    [metabase-enterprise.transforms-python.api]
    [metabase-enterprise.transforms.api]
    [metabase-enterprise.upload-management.api]
@@ -129,6 +130,7 @@
    ;; TODO (Ngoc 2026-03-25) -- use :transforms-advanced feature flag once it exists
    "/transforms"                   (premium-handler metabase-enterprise.transforms.api/routes :transforms-python)
    "/transforms-python"            (premium-handler metabase-enterprise.transforms-python.api/routes :transforms-python)
+   "/transform-optimizer"          (premium-handler metabase-enterprise.transform-optimizer.api/routes :transforms-python)
    "/scim"                         (premium-handler metabase-enterprise.scim.routes/routes :scim)
    "/semantic-search"              (premium-handler metabase-enterprise.semantic-search.api/routes :semantic-search)
    "/security-center"              (premium-handler metabase-enterprise.security-center.api/routes :admin-security-center)

enterprise/backend/src/metabase_enterprise/core/init.clj

@@ -23,4 +23,5 @@
    [metabase-enterprise.sso.init]
    [metabase-enterprise.stale.init]
    [metabase-enterprise.support-access-grants.init]
+   [metabase-enterprise.transform-optimizer.init]
    [metabase-enterprise.transforms-python.init]))

enterprise/backend/src/metabase_enterprise/metabot/usage.clj

@@ -27,6 +27,7 @@
     "slackbot"
     "oss-sql-gen"
     "sql-gen"
+    "transform-optimizer"
     "unknown"
     "user-intent-classification"})
 

enterprise/backend/src/metabase_enterprise/transform_optimizer/accept.clj

@@ -0,0 +1,340 @@
+(ns metabase-enterprise.transform-optimizer.accept
+  "Apply a proposal set. One proposal does **one** thing:
+
+    - `:rewrite` / `:precompute` — has a `body`, no DDL. We create a new
+      transform from the body.
+    - `:index` — has a single `ddl_statement`, no body. We either run it
+      immediately (when `target = 'source-db'`) or attach it to the
+      `:target.post_run_ddl` of the transform created by the proposal it
+      `depends_on` (when `target = 'transform-target'`).
+
+  Mixing a rewrite with an index in one proposal is no longer allowed at
+  the prelude / schema level, which makes the accept routing
+  straightforward."
+  (:require
+   [clojure.string :as str]
+   [metabase-enterprise.transform-optimizer.ddl.execute :as ddl.execute]
+   [metabase.lib.core :as lib]
+   [metabase.transforms.crud :as transforms.crud]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;; ---------------------------------------------------------------------------
+;; Target name sanitisation
+;;
+;; Postgres allows up to 63-byte identifiers. The proposal `name` field is
+;; human-readable, so we normalise to snake_case ASCII and clamp the length
+;; to leave room for a numeric suffix if a collision happens.
+
+(def ^:private max-name-bytes 56)
+
+(defn- sanitize-name [s]
+  (let [base (-> (or s "transform")
+                 u/lower-case-en
+                 (str/replace #"[^a-z0-9_]+" "_")
+                 (str/replace #"^_+|_+$" "")
+                 (str/replace #"_+" "_"))]
+    (cond
+      (str/blank? base)              "transform"
+      (Character/isDigit (.charAt ^String base 0)) (str "t_" base)
+      :else                          base)))
+
+(defn- clamp-name [s]
+  (let [s (sanitize-name s)]
+    (if (<= (count s) max-name-bytes)
+      s
+      (subs s 0 max-name-bytes))))
+
+;; ---------------------------------------------------------------------------
+;; Proposal → transform body
+;;
+;; Same as before: anchor the new query's `:database` to the original
+;; transform's `source_database_id` column, in case the embedded query has
+;; drifted.
+
+(defn- proposal->create-body
+  [{original-source :source :keys [target source_database_id] :as original}
+   {:keys [body name id target_table] :as _proposal}
+   collection-id]
+  (when (and body (not (str/blank? body)))
+    (let [original-query (:query original-source)
+          new-query      (cond-> (lib/with-native-query original-query body)
+                           (some? source_database_id)
+                           (assoc :database source_database_id))
+          ;; The LLM emits `target_table` for `:precompute` proposals so a
+          ;; dependent rewrite can FROM the exact same table name. Fall
+          ;; back to deriving it from the proposal's :name only when the
+          ;; LLM forgot — older preludes didn't have this field.
+          target-name    (clamp-name (or target_table name))]
+      {:name          (str (:name original) " — " (or name id "proposal"))
+       :description   (str "Auto-generated by transform-optimizer (proposal id: "
+                           (or id "?") ")")
+       :source        {:type  :query
+                       :query new-query}
+       :target        {:schema   (or (:schema target) "public")
+                       :name     target-name
+                       :type     :table
+                       :database source_database_id}
+       :collection_id (or collection-id (:collection_id original))})))
+
+;; ---------------------------------------------------------------------------
+;; DDL routing
+;;
+;; Each `:index` proposal owns exactly one statement. Routing key off the
+;; proposal's `ddl_statement.target`:
+;;
+;;   "source-db"        → run immediately against the source DB.
+;;   "transform-target" → attach to the `target.post_run_ddl` of the
+;;                        new transform created by the proposal this one
+;;                        `depends_on`. (We require exactly one body-bearing
+;;                        dependency for transform-target indices.)
+
+(defn- proposal-ddl
+  "Returns the accepted ddl_statement of an `:index` proposal, or nil."
+  [{:keys [ddl_statement] :as _proposal}]
+  (when (and ddl_statement (= :accepted (:validation ddl_statement)))
+    ddl_statement))
+
+(defn- group-deferred-ddl-by-dep
+  "For every `:index` proposal whose `ddl_statement.target` is
+  `transform-target`, attach its statement to the *first* body-bearing
+  proposal in its `depends_on` list. Returns `{dep-pid [ddl …]}`."
+  [proposals]
+  (let [body-bearing-ids (into #{}
+                               (keep (fn [{:keys [id body]}]
+                                       (when (and id (not (str/blank? (or body "")))) id)))
+                               proposals)]
+    (reduce
+     (fn [acc p]
+       (if-let [ddl (proposal-ddl p)]
+         (if (= "transform-target" (:target ddl))
+           (if-let [dep (some body-bearing-ids (:depends_on p))]
+             (update acc dep (fnil conj []) ddl)
+             ;; transform-target with no body-bearing dep → can't apply;
+             ;; surface as failed later (we keep the ddl on the proposal so
+             ;; the UI can show the rejection).
+             acc)
+           acc)
+         acc))
+     {}
+     proposals)))
+
+;; ---------------------------------------------------------------------------
+;; Execution
+
+(defn- run-source-db-ddl!
+  "Runs every accepted `source-db` DDL. Returns a `{pid → result-map}` keyed
+  by the originating proposal id."
+  [original-transform proposals]
+  (let [db-id     (:source_database_id original-transform)
+        database  (when db-id (t2/select-one :model/Database :id db-id))
+        driver-kw (some-> database :engine keyword)]
+    (reduce
+     (fn [acc p]
+       (let [ddl (proposal-ddl p)]
+         (if (and ddl (= "source-db" (:target ddl)))
+           (assoc acc (:id p) (ddl.execute/execute! driver-kw database (:statement ddl)))
+           acc)))
+     {}
+     proposals)))
+
+(defn- ddl-row
+  "Build the response row for a proposal's `ddl_statement` based on
+  validation + execution outcome."
+  [{:keys [id ddl_statement] :as _proposal} exec-result]
+  (when ddl_statement
+    (let [base (-> (select-keys ddl_statement
+                                [:target :statement :rationale
+                                 :validation :index_name :rejection])
+                   (assoc :proposal_id id))]
+      (cond
+        (= :rejected (:validation ddl_statement))
+        base                                            ;; no status — rejected at validation
+
+        exec-result
+        (merge base
+               {:status (:status exec-result)}
+               (when-let [m (:error-message exec-result)] {:error_message m}))
+
+        (= "transform-target" (:target ddl_statement))
+        (assoc base :status :pending)
+
+        :else
+        (assoc base :status :skipped :error_message "no dependency to attach to")))))
+
+;; ---------------------------------------------------------------------------
+;; Public API
+
+;; ---------------------------------------------------------------------------
+;; Replace mode — swap the original transform's source in place
+
+(defn- replace-in-place!
+  "Update `original-transform`'s `:source.query` to the proposal's body
+  (anchored to the original's `source_database_id`) and merge any
+  deferred DDL into its existing `:target.post_run_ddl`. Returns the
+  updated transform map."
+  [original-transform proposal deferred-ddl]
+  (let [original-query (-> original-transform :source :query)
+        db-id          (:source_database_id original-transform)
+        new-query      (cond-> (lib/with-native-query original-query (:body proposal))
+                         (some? db-id) (assoc :database db-id))
+        existing-ddl   (get-in original-transform [:target :post_run_ddl] [])
+        merged-ddl     (vec (concat existing-ddl deferred-ddl))
+        new-target     (cond-> (:target original-transform)
+                         (seq merged-ddl) (assoc :post_run_ddl merged-ddl))
+        body           (cond-> {:source {:type :query :query new-query}}
+                         (seq merged-ddl) (assoc :target new-target))]
+    (transforms.crud/update-transform! (:id original-transform) body)))
+
+(defn- body-bearing? [p]
+  (and (:body p) (not (str/blank? (:body p)))))
+
+(defn- one-rewrite-and-precomputes-or-throw
+  "Replace mode requires **exactly one** body-bearing `:rewrite` proposal in
+  the batch (that's the one whose body replaces the original transform's
+  source). Any number of `:precompute` proposals are also OK — those are
+  still materialised as *new* intermediate transforms; replace only ever
+  targets the original. Returns `[the-rewrite the-precomputes]`."
+  [proposals]
+  (let [bodies      (filter body-bearing? proposals)
+        rewrites    (filter #(= "rewrite" (some-> (:kind %) name)) bodies)
+        precomputes (filter #(= "precompute" (some-> (:kind %) name)) bodies)
+        unknown     (remove (fn [p] (#{"rewrite" "precompute"}
+                                     (some-> (:kind p) name)))
+                            bodies)]
+    (cond
+      (zero? (count rewrites))
+      (throw (ex-info "replace mode needs exactly one body-bearing :rewrite proposal; got none"
+                      {:status-code 400 :error "replace_needs_rewrite"}))
+
+      (> (count rewrites) 1)
+      (throw (ex-info (str "replace mode needs exactly one body-bearing :rewrite proposal; got "
+                           (count rewrites))
+                      {:status-code 400 :error "replace_multiple_rewrites"}))
+
+      (seq unknown)
+      (throw (ex-info (str "replace mode only accepts :rewrite + :precompute body-bearing "
+                           "proposals; got " (vec (map (comp name :kind) unknown)))
+                      {:status-code 400 :error "replace_unsupported_kind"}))
+
+      :else
+      [(first rewrites) precomputes])))
+
+;; ---------------------------------------------------------------------------
+;; Public API
+
+(defn accept!
+  "Apply a proposal set. `mode` selects how body-bearing proposals are
+  materialised:
+
+    :new (default) — Each body-bearing proposal becomes a *new* transform.
+                     Pure-index proposals contribute DDL only.
+    :replace       — The single body-bearing proposal in the batch
+                     replaces the original transform's source in place
+                     instead of creating a new transform. Any
+                     `transform-target` DDL attached via `depends_on` is
+                     merged into the original transform's
+                     `target.post_run_ddl`. Source-db DDL runs as in
+                     `:new`. Only valid when the batch contains exactly
+                     one body-bearing proposal of kind=rewrite.
+
+  Returns (mode :new):
+    {:created_transforms [{id, name, proposal_id, kind, depends_on, …}]
+     :ddl_statements     [{proposal_id, target, status, …}]
+     :skipped_proposals  [proposal-id …]}
+
+  Returns (mode :replace):
+    {:replaced_transform {id, name, proposal_id, kind}
+     :ddl_statements     [{proposal_id, target, status, …}]
+     :skipped_proposals  [proposal-id …]}"
+  ([original-transform proposals collection-id]
+   (accept! original-transform proposals collection-id :new))
+  ([original-transform proposals collection-id mode]
+   (let [proposals       (vec (or proposals []))
+         mode            (or mode :new)
+         deferred-by-dep (group-deferred-ddl-by-dep proposals)
+         source-db-out   (run-source-db-ddl! original-transform proposals)
+         ddl-rows        (vec (keep #(ddl-row % (get source-db-out (:id %))) proposals))
+         skipped         (->> proposals
+                              (remove body-bearing?)
+                              (mapv :id))]
+     (case mode
+       :replace
+       (let [[rewrite-proposal precompute-proposals]
+             (one-rewrite-and-precomputes-or-throw proposals)
+             ;; First: create every precompute as a new transform (the
+             ;; rewrite's body will reference their target tables). Any
+             ;; deferred transform-target DDL routes to its precompute,
+             ;; same as in :new mode.
+             created-precomputes
+             (->> precompute-proposals
+                  (keep
+                   (fn [p]
+                     (when-let [body (proposal->create-body original-transform p collection-id)]
+                       (let [deferred (get deferred-by-dep (:id p))
+                             body+ddl (cond-> body
+                                        (seq deferred)
+                                        (assoc-in [:target :post_run_ddl] (vec deferred)))]
+                         (try
+                           (let [new (transforms.crud/create-transform! body+ddl)]
+                             {:id          (:id new)
+                              :name        (:name new)
+                              :proposal_id (:id p)
+                              :kind        "precompute"
+                              :depends_on  (or (:depends_on p) [])
+                              :pending_ddl (count deferred)})
+                           (catch Exception e
+                             (log/warnf e "accept! replace: precompute create failed for proposal %s" (:id p))
+                             {:proposal_id (:id p)
+                              :error       (or (ex-message e) "precompute create failed")}))))))
+                  vec)
+             ;; Then: replace the original's source with the rewrite body.
+             rewrite-deferred (get deferred-by-dep (:id rewrite-proposal))
+             updated          (try
+                                (replace-in-place! original-transform rewrite-proposal rewrite-deferred)
+                                (catch Exception e
+                                  (log/warnf e "accept! replace failed for proposal %s"
+                                             (:id rewrite-proposal))
+                                  (throw (ex-info (or (ex-message e) "replace failed")
+                                                  {:status-code 500
+                                                   :error "replace_failed"
+                                                   :proposal_id (:id rewrite-proposal)}))))]
+         (cond-> {:replaced_transform {:id          (:id updated)
+                                       :name        (:name updated)
+                                       :proposal_id (:id rewrite-proposal)
+                                       :kind        "rewrite"
+                                       :pending_ddl (count (or rewrite-deferred []))}
+                  :ddl_statements    ddl-rows
+                  :skipped_proposals skipped}
+           (seq created-precomputes)
+           (assoc :created_transforms created-precomputes)))
+
+       :new
+       (let [created
+             (->> proposals
+                  (keep
+                   (fn [p]
+                     (when-let [body (proposal->create-body original-transform p collection-id)]
+                       (let [deferred (get deferred-by-dep (:id p))
+                             body+ddl (cond-> body
+                                        (seq deferred)
+                                        (assoc-in [:target :post_run_ddl] (vec deferred)))]
+                         (try
+                           (let [new (transforms.crud/create-transform! body+ddl)]
+                             {:id          (:id new)
+                              :name        (:name new)
+                              :proposal_id (:id p)
+                              :kind        (some-> (:kind p) name)
+                              :depends_on  (or (:depends_on p) [])
+                              :pending_ddl (count deferred)})
+                           (catch Exception e
+                             (log/warnf e "accept!: create-transform! failed for proposal %s" (:id p))
+                             {:proposal_id (:id p)
+                              :error       (or (ex-message e) "create failed")}))))))
+                  vec)]
+         {:created_transforms created
+          :ddl_statements     ddl-rows
+          :skipped_proposals  skipped})))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/api.clj

@@ -0,0 +1,301 @@
+(ns metabase-enterprise.transform-optimizer.api
+  "HTTP endpoints for the Transform Optimizer.
+
+  Mounted under `/api/ee/transform-optimizer` by the EE route map. All
+  three endpoints (optimize / verify / accept / indexes / drop) are plain
+  request/response — no SSE in this branch. We may revisit streaming once
+  we have incremental JSON parsing of the LLM tool call, but until then
+  the response is buffered server-side anyway and the wire complexity
+  isn't paying for itself."
+  (:require
+   [metabase-enterprise.transform-optimizer.accept :as opt.accept]
+   [metabase-enterprise.transform-optimizer.core :as opt.core]
+   [metabase-enterprise.transform-optimizer.indexes :as opt.indexes]
+   [metabase-enterprise.transform-optimizer.proposal-cache :as opt.cache]
+   [metabase-enterprise.transform-optimizer.verify :as opt.verify]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.api.util.handlers :as handlers]
+   [metabase.util.log :as log]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;; ---------------------------------------------------------------------------
+;; Optimize
+
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :post "/:id/optimize"
+  "Run the optimizer for `transform-id` and return the full result in one
+  JSON payload:
+
+    {:transform           {id, name, source_database_id, target}
+     :sql                 <compiled source SQL>
+     :summary             <one-paragraph diagnosis>
+     :proposals           [{id, name, kind, severity, body | ddl_statement, …}]
+     :optimization_degree <0..100>}
+
+  Proposals are cached server-side keyed by (user, transform, proposal id)
+  so the verify/accept endpoints can look them up by id alone."
+  [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+   _query-params
+   body :- [:maybe [:map [:analyze {:optional true} [:maybe :boolean]]]]]
+  (let [transform (api/read-check :model/Transform id)
+        analyze?  (boolean (:analyze body))]
+    (try
+      (let [result (opt.core/optimize! (:id transform) :analyze? analyze?)]
+        ;; Stash each proposal in the per-user cache so verify/accept can
+        ;; resolve a `proposal_id` back to its full payload.
+        (opt.cache/put-all! api/*current-user-id* (:id transform) (:proposals result))
+        result)
+      (catch Exception e
+        (log/errorf e "optimize failed (transform-id=%d)" (:id transform))
+        (let [data (ex-data e)]
+          {:status (or (:status-code data) 502)
+           :body   {:error     "optimize_failed"
+                    :message   (or (ex-message e) "unknown error")
+                    :retryable (boolean (:retryable data))}})))))
+
+;; ---------------------------------------------------------------------------
+;; Bulk optimize
+;;
+;; The FE sends a list of transform ids ("the slow ones, per your filter")
+;; and we kick off a single background thread that runs `optimize!`
+;; sequentially for each. Sequential — not pmap — because each call is an
+;; LLM round-trip and we don't want to pile concurrent calls against the
+;; Anthropic API on a hackathon-grade rate limit.
+;;
+;; Job state is held in-memory per-user: the FE polls `/bulk-optimize/status`
+;; while the drawer is open and shows each transform's proposals as they
+;; land. The per-user proposal cache is populated alongside so verify and
+;; accept endpoints work on bulk results without re-running the optimizer.
+
+(defonce ^:private bulk-jobs
+  ;; user-id -> {:total N
+  ;;             :pending [ids]
+  ;;             :done    {id -> {:summary :proposals :optimization_degree :transform}}
+  ;;             :failed  {id -> error-message}
+  ;;             :started_at instant}
+  (atom {}))
+
+(defn- init-bulk-job! [user-id ids]
+  (swap! bulk-jobs assoc user-id
+         {:total      (count ids)
+          :pending    (vec ids)
+          :done       {}
+          :failed     {}
+          :started_at (java.time.Instant/now)}))
+
+(defn- record-success! [user-id id result]
+  (swap! bulk-jobs update user-id
+         (fn [job]
+           (-> job
+               (update :pending (fn [ps] (vec (remove #{id} ps))))
+               (assoc-in [:done id]
+                         (select-keys result [:summary :proposals
+                                              :optimization_degree :transform]))))))
+
+(defn- record-failure! [user-id id err-msg]
+  (swap! bulk-jobs update user-id
+         (fn [job]
+           (-> job
+               (update :pending (fn [ps] (vec (remove #{id} ps))))
+               (assoc-in [:failed id] err-msg)))))
+
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :post "/bulk-optimize"
+  "Run the optimizer in the background against every transform id in
+  `transform_ids`. Returns immediately with the count accepted. The FE
+  polls `/bulk-optimize/status` for progress and the per-transform
+  results."
+  [_route-params
+   _query-params
+   body :- [:map [:transform_ids [:sequential ms/PositiveInt]]]]
+  (let [ids       (vec (distinct (:transform_ids body)))
+        ;; Permission-check each transform up front (404/403 surfaced
+        ;; synchronously); only the ids the user can read end up in the
+        ;; background queue.
+        readable  (vec (keep #(try (api/read-check :model/Transform %) %
+                                   (catch Exception _ nil))
+                             ids))
+        user-id   api/*current-user-id*]
+    (init-bulk-job! user-id readable)
+    (when (seq readable)
+      (future
+        (doseq [id readable]
+          (try
+            (let [result (opt.core/optimize! id)]
+              ;; Mirror the per-transform endpoint: cache proposals so the
+              ;; existing verify / accept endpoints can resolve them by id
+              ;; without re-running the optimizer.
+              (opt.cache/put-all! user-id id (:proposals result))
+              (record-success! user-id id result))
+            (catch Exception e
+              (log/warnf e "bulk-optimize: optimize! failed for transform-id=%s (user-id=%s)"
+                         id user-id)
+              (record-failure! user-id id (or (ex-message e) "unknown error")))))))
+    {:started       (count readable)
+     :transform_ids readable}))
+
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :get "/bulk-optimize/status"
+  "Current state of the calling user's most recent bulk-optimize run.
+  Returns an empty job shape (`{:total 0 …}`) when the user has never
+  triggered a bulk run in this process. Safe to poll on an interval."
+  [_route-params _query-params]
+  (or (get @bulk-jobs api/*current-user-id*)
+      {:total 0 :pending [] :done {} :failed {}}))
+
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :post "/:id/mark-optimized"
+  "Flip `transform.optimized = true` for `transform-id`. The bulk-drawer
+  YOLO flow calls this after accepting every proposal for a transform —
+  the user has decided this transform is done, so we persist that
+  verdict without re-running the LLM. The before-update hook on
+  Transform clears the flag again the next time the source is edited."
+  [{:keys [id]} :- [:map [:id ms/PositiveInt]]]
+  (api/write-check :model/Transform id)
+  (t2/update! :model/Transform id {:optimized true})
+  {:id id :optimized true})
+
+;; ---------------------------------------------------------------------------
+;; Verify endpoint
+
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :post "/:id/proposal/verify"
+  "Verify equivalence of the original transform vs the proposal. Materialises
+  both into scratch tables and compares via `EXCEPT ALL` in both directions.
+  See SUMMARY.md for the response shape.
+
+  Single-transform proposals only — precompute (DAG) verification is
+  deferred until DAG accept is in place.
+
+  The proposal payload is resolved server-side from the proposal cache
+  populated by `/optimize`, so the FE only sends the `proposal_id`."
+  [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+   _query-params
+   body :- [:map [:proposal_id :string]]]
+  (let [transform (api/read-check :model/Transform id)
+        pid       (:proposal_id body)
+        proposal  (opt.cache/get-one api/*current-user-id* (:id transform) pid)]
+    (cond
+      (nil? proposal)
+      {:status 404
+       :body   {:error  "proposal_not_found"
+                :detail (str "Proposal " (pr-str pid)
+                             " is not in the optimizer cache (it may have expired or this is a different "
+                             "process). Re-run /optimize to repopulate it.")}}
+
+      :else
+      (try
+        (opt.verify/verify transform proposal)
+        (catch clojure.lang.ExceptionInfo e
+          (let [data (ex-data e)]
+            (if-let [code (:status-code data)]
+              {:status code
+               :body   (-> data
+                           (dissoc :status-code)
+                           (assoc :error (or (:error data) "verify_failed")
+                                  :detail (ex-message e)))}
+              (throw e))))))))
+
+;; ---------------------------------------------------------------------------
+;; Accept endpoint
+
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :post "/:id/proposal/accept"
+  "Apply the proposal set: create new transforms for proposals that
+  carry a body, *and* run every validated `CREATE INDEX` statement
+  attached to those proposals against the source database. Single
+  rewrites pass an array of one id; precompute DAGs pass N ids in
+  dependency order (caller's responsibility — we create them in the
+  order given).
+
+  Each proposal payload is resolved server-side from the proposal cache
+  populated by `/optimize`. If any id is missing, the whole request is
+  rejected (404) — partial accept is too easy to mis-handle on the FE.
+
+  The response includes `ddl_statements` with per-statement execution
+  status (`:executed | :failed | :skipped | :pending`) plus the original
+  validation tag. Failed DDL does NOT roll back successfully-created
+  transforms — `CREATE INDEX IF NOT EXISTS` is idempotent, so the user
+  can re-run accept after fixing whatever the problem was."
+  [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+   _query-params
+   body :- [:map
+            [:proposal_ids  [:sequential :string]]
+            [:mode          {:optional true} [:maybe [:enum "new" "replace"]]]
+            [:collection_id {:optional true} [:maybe ms/PositiveInt]]]]
+  (let [transform                  (api/read-check :model/Transform id)
+        [proposals missing]        (opt.cache/get-many api/*current-user-id*
+                                                       (:id transform)
+                                                       (:proposal_ids body))
+        mode                       (keyword (or (:mode body) "new"))]
+    (cond
+      (seq missing)
+      {:status 404
+       :body   {:error              "proposal_not_found"
+                :detail             (str "These proposal ids are not in the optimizer cache "
+                                         "(expired or never seen); re-run /optimize.")
+                :missing_proposal_ids missing}}
+
+      :else
+      (try
+        (opt.accept/accept! transform proposals (:collection_id body) mode)
+        (catch clojure.lang.ExceptionInfo e
+          (let [data (ex-data e)]
+            (if-let [code (:status-code data)]
+              {:status code
+               :body   (-> data
+                           (dissoc :status-code)
+                           (assoc :detail (ex-message e)))}
+              (throw e))))))))
+
+;; ---------------------------------------------------------------------------
+;; Index management on a transform's target + source tables
+
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :get "/:id/indexes"
+  "List every index on the transform's target table *and* every source
+  table it reads from. Each row carries `:schema`/`:table` so the FE can
+  group, plus `:is_target_table` for visual separation and
+  `:managed_by_optimizer` indicating whether the optimizer registered
+  the index for post-run replay (target-table-only)."
+  [{:keys [id]} :- [:map [:id ms/PositiveInt]]]
+  (let [transform (api/read-check :model/Transform id)]
+    {:transform {:id (:id transform) :target (:target transform)}
+     :indexes   (opt.indexes/list-indexes transform)}))
+
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :post "/:id/index/drop"
+  "Drop the named index from any of the transform's referenced tables
+  (target or source). Validates that the index actually exists on one of
+  those tables — we won't drop random indices elsewhere on the DB. When
+  the dropped index was on the target *and* the optimizer was replaying
+  it via `target.post_run_ddl`, the persisted entry is also removed so
+  the next transform run doesn't recreate it."
+  [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+   _query-params
+   body :- [:map [:index_name ms/NonBlankString]]]
+  (let [transform (api/read-check :model/Transform id)
+        result    (opt.indexes/drop-index! transform (:index_name body))]
+    (case (:status result)
+      :dropped result
+      :failed  {:status 502 :body (assoc result :error "drop_failed")}
+      :skipped {:status (case (:reason result)
+                          :index-not-on-referenced-table 404
+                          :unsafe-name                   400
+                          :no-database                   404
+                          :not-postgres                  422
+                          400)
+                :body (assoc result :error "drop_skipped")})))
+
+;; ---------------------------------------------------------------------------
+;; Route bundle
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/transform-optimizer` routes."
+  (handlers/routes
+   (api.macros/ns-handler *ns* +auth)))

enterprise/backend/src/metabase_enterprise/transform_optimizer/context.clj

@@ -0,0 +1,216 @@
+(ns metabase-enterprise.transform-optimizer.context
+  "Top-level context builder for the transform optimizer.
+
+  Stitches together:
+    - `transforms-inspector.context/build-context` for sources/target/joins
+      (we re-use it rather than reimplementing source extraction).
+    - `transforms-base.util/compile-source` for the compiled native SQL we
+      feed to the LLM and to EXPLAIN.
+    - Foreign-key edges and `database_indexed` flags from the appdb
+      (`Field.fk_target_field_id`).
+    - Full index shape (composite columns, INCLUDE, partial predicate, type)
+      from the source DB via `index-introspection`.
+    - An `EXPLAIN (FORMAT JSON, VERBOSE)` plan tree.
+    - Recent run history with durations.
+
+  The returned map is what `prompt.clj` (to come) renders into the LLM
+  context block — it is not yet a string, just a data shape."
+  (:require
+   [metabase-enterprise.transform-optimizer.explain :as opt.explain]
+   [metabase-enterprise.transform-optimizer.index-introspection :as opt.indexes]
+   [metabase-enterprise.transforms-inspector.context :as inspector.context]
+   [metabase.transforms-base.util :as transforms-base.u]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;; ---------------------------------------------------------------------------
+;; SQL compilation
+
+(defn- transform-sql
+  "Compile a `:query`-type transform down to the native SQL string we feed to
+  EXPLAIN and to the LLM. Returns `nil` (with a warning) if compilation
+  fails — the optimizer should still produce a degraded context rather than
+  bailing entirely."
+  [transform]
+  (try
+    (-> (transforms-base.u/compile-source transform nil) :query)
+    (catch Exception e
+      (log/warnf e "transform-optimizer: failed to compile source SQL (transform-id=%s)"
+                 (:id transform))
+      nil)))
+
+;; ---------------------------------------------------------------------------
+;; Foreign-key resolution
+;;
+;; The inspector's `fields` only carry the basic Field shape. For optimizer
+;; context we want each field's FK target resolved to (target_schema,
+;; target_table, target_column) so the LLM can reason about join paths
+;; without us having to dump the full appdb.
+
+(defn- collect-fk-target-ids [sources]
+  (into #{}
+        (comp (mapcat :fields)
+              (keep :fk_target_field_id))
+        sources))
+
+(defn- fk-target-index ^java.util.Map [target-ids]
+  ;; Two batch queries (fields → tables) then a small in-memory join.
+  (when (seq target-ids)
+    (let [target-fields (t2/select [:model/Field :id :name :table_id]
+                                   :id [:in target-ids])
+          table-ids     (into #{} (map :table_id) target-fields)
+          id->table     (when (seq table-ids)
+                          (into {} (map (juxt :id identity))
+                                (t2/select [:model/Table :id :schema :name]
+                                           :id [:in table-ids])))]
+      (into {}
+            (keep (fn [{:keys [id name table_id]}]
+                    (when-let [t (get id->table table_id)]
+                      [id {:target_schema   (:schema t)
+                           :target_table    (:name t)
+                           :target_column   name
+                           :target_field_id id}])))
+            target-fields))))
+
+;; ---------------------------------------------------------------------------
+;; `database_indexed` lookup (cheap appdb-only signal that complements full
+;; index introspection — useful as a fallback when index-introspection fails)
+
+(defn- field-id->indexed
+  "Return `{field-id -> bool}` for every field on the given source tables. The
+  inspector's collect-field-metadata doesn't include this column, so we fetch
+  it separately."
+  [source-table-ids]
+  (when (seq source-table-ids)
+    (into {}
+          (map (juxt :id (comp boolean :database_indexed)))
+          (t2/select [:model/Field :id :database_indexed]
+                     :table_id [:in source-table-ids]
+                     :active true))))
+
+;; ---------------------------------------------------------------------------
+;; Source enrichment
+
+(defn- enrich-fields [fields fk-index id->indexed?]
+  (mapv (fn [{:keys [id fk_target_field_id] :as field}]
+          (cond-> (dissoc field :fingerprint)
+            (some? id)
+            (assoc :indexed? (boolean (get id->indexed? id)))
+
+            fk_target_field_id
+            (assoc :foreign_key (get fk-index fk_target_field_id))))
+        fields))
+
+(defn- source->table-pair [{:keys [schema table_name]}]
+  (when (and schema table_name)
+    [schema table_name]))
+
+(defn- attach-indexes [sources indexes-by-table]
+  (mapv (fn [{:keys [schema table_name] :as src}]
+          (assoc src
+                 :indexes
+                 (get indexes-by-table
+                      [(some-> schema     u/lower-case-en)
+                       (some-> table_name u/lower-case-en)]
+                      [])))
+        sources))
+
+(defn- enrich-sources [sources fk-index id->indexed?]
+  (mapv (fn [{:keys [fields] :as src}]
+          (assoc src :fields (enrich-fields fields fk-index id->indexed?)))
+        sources))
+
+;; ---------------------------------------------------------------------------
+;; Run history
+
+(defn- duration-ms [^java.time.OffsetDateTime start ^java.time.OffsetDateTime end]
+  (when (and start end)
+    (- (.toEpochMilli (.toInstant end))
+       (.toEpochMilli (.toInstant start)))))
+
+(defn- run-history [transform-id n]
+  (->> (t2/select [:model/TransformRun :id :status :start_time :end_time :message :run_method]
+                  :transform_id transform-id
+                  {:order-by [[:start_time :desc]]
+                   :limit    n})
+       (mapv (fn [run]
+               (assoc run :duration_ms (duration-ms (:start_time run) (:end_time run)))))))
+
+;; ---------------------------------------------------------------------------
+;; Entry point
+
+(defn build-context
+  "Assemble the optimizer's per-transform context map. `opts` are forwarded to
+  `explain/explain` (the only meaningful one today is `:analyze? true` —
+  *don't* pass it for transforms whose latest run took longer than a few
+  seconds; that's what we're trying to avoid).
+
+  Returns a map of:
+    :transform        {id, name, source_database_id, target}
+    :sql              compiled native SQL or nil
+    :sources          [{schema, table_name, db_id, column_count,
+                        fields [{name, base_type, indexed?, foreign_key …}],
+                        indexes [{name, definition, key_columns,
+                                  include_columns, access_method,
+                                  partial_predicate, …}]} …]
+    :target           inspector target-table info, or nil
+    :indexes_partial? true when full catalog introspection was unavailable
+                      (non-Postgres driver, transient failure) — callers
+                      should annotate the LLM prompt accordingly.
+    :explain          parsed JSON plan tree, or nil
+    :run_history      [{id, status, start_time, end_time, duration_ms,
+                        run_method, message} …] (most recent first)"
+  [transform & {:as opts}]
+  (let [;; Inspector context is best-effort. It can fail for plenty of
+        ;; reasons we can't paper over here (unparseable native SQL, missing
+        ;; FK metadata, schema drift), and when it does the optimizer should
+        ;; still produce a degraded context — the LLM gets less schema info
+        ;; but EXPLAIN, run history, and compiled SQL all still apply.
+        inspector   (try
+                      (inspector.context/build-context transform)
+                      (catch Exception e
+                        (log/warnf e "transform-optimizer: inspector context failed (transform-id=%s); proceeding without sources"
+                                   (:id transform))
+                        nil))
+        sources     (:sources inspector)
+        db-id       (or (:db-id inspector)
+                        (transforms-base.u/transform-source-database transform))
+        driver-kw   (or (:driver inspector)
+                        (when db-id
+                          (t2/select-one-fn (comp keyword :engine) :model/Database :id db-id)))
+        database    (when db-id (t2/select-one :model/Database :id db-id))
+        sql         (transform-sql transform)
+        fk-index    (fk-target-index (collect-fk-target-ids sources))
+        id->indexed (field-id->indexed (into #{} (keep :table_id) sources))
+        enriched    (enrich-sources sources fk-index id->indexed)
+        indexes     (when (and driver-kw database (seq sources))
+                      (try
+                        (opt.indexes/fetch-indexes
+                         driver-kw database
+                         (into [] (keep source->table-pair) sources))
+                        (catch Exception e
+                          (log/warnf e "index introspection failed (transform-id=%s)"
+                                     (:id transform))
+                          nil)))
+        idx-by-tbl  (opt.indexes/group-by-table (or indexes []))
+        with-idx    (attach-indexes enriched idx-by-tbl)
+        explained   (when (and database sql)
+                      (opt.explain/explain driver-kw database sql (or opts {})))]
+    {:transform        {:id                 (:id transform)
+                        :name               (:name transform)
+                        :target             (:target transform)
+                        :source_database_id (:source_database_id transform)}
+     :sql              sql
+     :sources          with-idx
+     :target           (:target inspector)
+     ;; Field IDs that the SQL actually references (FROM, WHERE, JOIN,
+     ;; GROUP BY, ORDER BY, SELECT). The prompt uses this to decide which
+     ;; fields are worth rendering :stats for — fingerprints on every
+     ;; column would blow the token budget for no win.
+     :visited_fields   (set (get-in inspector [:visited-fields :all]))
+     :indexes_partial? (and (seq sources) (nil? indexes))
+     :explain          explained
+     :run_history      (run-history (:id transform) 10)}))

enterprise/backend/src/metabase_enterprise/transform_optimizer/core.clj

@@ -0,0 +1,227 @@
+(ns metabase-enterprise.transform-optimizer.core
+  "Public entry point for the transform optimizer.
+
+  The pipeline:
+
+      transform-id ─► context.clj  ─┐
+                                    ├─► prompt.clj ─► LLM (Phase 2) ─► proposals
+                  prelude.clj  ─────┘                                        │
+                                                                             ▼
+                                                                  ddl/parse  + scoring
+                                                                             │
+                                                                             ▼
+                                                              {optimization_degree
+                                                               summary
+                                                               proposals[]}
+
+  Today this namespace stitches the deterministic halves of the pipeline
+  together — context, prompt, scoring, DDL validation. The LLM call itself
+  is a placeholder (`call-llm-stub`) we replace in Phase 2 with the actual
+  Metabot deftool wiring."
+  (:require
+   [clojure.string :as str]
+   [metabase-enterprise.transform-optimizer.context :as context]
+   [metabase-enterprise.transform-optimizer.ddl.parse :as ddl.parse]
+   [metabase-enterprise.transform-optimizer.llm :as llm]
+   [metabase-enterprise.transform-optimizer.prelude :as prelude]
+   [metabase-enterprise.transform-optimizer.prompt :as prompt]
+   [metabase-enterprise.transform-optimizer.scoring :as scoring]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;; ---------------------------------------------------------------------------
+;; Building blocks
+;;
+;; Each of these is independently exercisable from the REPL. `optimize!`
+;; below composes them; tests/UI/agent paths can re-use whichever pieces
+;; they need.
+
+(defn fetch-transform
+  "Hydrate a transform by id, with the keys the optimizer needs."
+  [transform-id]
+  (or (t2/select-one :model/Transform :id transform-id)
+      (throw (ex-info "transform not found"
+                      {:transform-id transform-id
+                       :status-code 404}))))
+
+(defn build-prompt
+  "Assemble the full LLM input for a transform: prelude (system side) + the
+  context-rendered markdown (user side). Returns a map suitable for the
+  LLM client layer:
+
+      {:system  <prelude>
+       :user    <context block>
+       :context <raw context map for downstream introspection/tests>}"
+  [transform & {:as opts}]
+  (let [ctx       (apply context/build-context transform (mapcat identity (or opts {})))
+        rendered  (prompt/render-context ctx)]
+    {:system  (prelude/prelude)
+     :user    rendered
+     :context ctx}))
+
+;; ---------------------------------------------------------------------------
+;; Proposal post-processing
+
+(defn- pair-set
+  "Set of `[schema table]` pairs the LLM is allowed to target with DDL,
+  pulled from the context's sources and (when present) target table."
+  [ctx]
+  (into #{}
+        (keep (fn [{:keys [schema table_name]}]
+                (when (and schema table_name) [schema table_name])))
+        (concat (:sources ctx)
+                (when-let [t (:target ctx)] [t]))))
+
+(defn- coerce-ddl-shape
+  "Older prelude drafts (and some LLM training data) emit
+  `ddl_statements: [{…}]` instead of `ddl_statement: {…}`. Take the first
+  entry from the plural form when the singular field is missing so we
+  don't lose the index proposal entirely. Also drop the plural field
+  afterwards so downstream code sees exactly one shape."
+  [proposal]
+  (let [{:keys [ddl_statement ddl_statements]} proposal
+        coerced (or ddl_statement (first ddl_statements))]
+    (-> proposal
+        (dissoc :ddl_statements)
+        (cond-> coerced (assoc :ddl_statement coerced)))))
+
+(defn- validate-proposal-ddl
+  "Each proposal carries at most one `:ddl_statement` (singular). Validate
+  the statement against the CREATE-INDEX allowlist and tag with
+  `:validation = :accepted | :rejected`. Rejected statements stay on the
+  proposal with their rejection reason so the UI can render the failure
+  inline instead of silently dropping the proposal."
+  [proposal allowed-tables]
+  (let [{:keys [ddl_statement] :as proposal} (coerce-ddl-shape proposal)]
+    (if-not ddl_statement
+      proposal
+      (let [{:keys [ok?] :as r} (ddl.parse/parse (:statement ddl_statement) allowed-tables)]
+        (assoc proposal
+               :ddl_statement
+               (cond-> ddl_statement
+                 ok?         (assoc :validation :accepted
+                                    :index_name (:name r))
+                 (not ok?)   (assoc :validation :rejected
+                                    :rejection  (select-keys r [:reason :detail]))))))))
+
+(defn- normalize-sql
+  "Textual normalisation for no-op detection: strip SQL comments (line +
+  block), strip trailing semicolons, lower-case, collapse all whitespace
+  runs to a single space, trim. Two SQL strings that normalise to the
+  same value differ only in formatting / punctuation / casing — i.e.
+  there's no real rewrite to ship."
+  [sql]
+  (some-> sql
+          (str/replace #"(?s)/\*.*?\*/" "")  ; block comments
+          (str/replace #"--[^\n]*" "")       ; line comments
+          (str/replace #";+\s*$" "")         ; trailing semicolon(s)
+          str/lower-case
+          (str/replace #"\s+" " ")
+          str/trim))
+
+(defn- noop-rewrite?
+  "True when the proposal carries a `body` that, after normalisation, is
+  textually identical to the original transform's SQL. This is the
+  semicolon / whitespace / comment-only \"rewrite\" anti-pattern — the
+  LLM sometimes emits these as wrapper proposals despite the prelude."
+  [original-norm proposal]
+  (when-let [body-norm (normalize-sql (:body proposal))]
+    (= body-norm original-norm)))
+
+(defn- transitively-drop-dependents
+  "Given a set of proposal ids being dropped, expand the set to include any
+  proposal in `proposals` that `depends_on` a dropped id. Iterates to
+  fixed point so chains (p3 → p2 → p1) collapse together.
+
+  Returns `[final-dropped-set kept-proposals dropped-proposals]`."
+  [proposals initial-dropped]
+  (loop [dropped initial-dropped]
+    (let [next-dropped (into dropped
+                             (keep (fn [p]
+                                     (when (and (not (contains? dropped (:id p)))
+                                                (some dropped (:depends_on p)))
+                                       (:id p))))
+                             proposals)]
+      (if (= next-dropped dropped)
+        [dropped
+         (remove #(contains? dropped (:id %)) proposals)
+         (filterv #(contains? dropped (:id %)) proposals)]
+        (recur next-dropped)))))
+
+(defn finalise-proposals
+  "Server-side cleanup applied to a raw proposal set from the LLM:
+
+   1. Drop proposals whose body is textually equivalent to the original
+      (cosmetic-only rewrites — strip comments / semicolons / whitespace
+      and compare).
+   2. Transitively drop any proposal that `depends_on` a dropped one —
+      a dependent rewrite would FROM a precompute table no one creates.
+   3. Validate the single DDL statement on each `:index` proposal against
+      the CREATE-INDEX allowlist and tag with
+      `:validation = :accepted | :rejected`.
+   4. Compute the deterministic optimization_degree from what's left.
+
+   Returns the response payload exactly as the UI consumes it."
+  [proposals ctx]
+  (let [allowed         (pair-set ctx)
+        original-norm   (normalize-sql (:sql ctx))
+        noop?           (partial noop-rewrite? original-norm)
+        initial-dropped (into #{} (keep (fn [p] (when (noop? p) (:id p)))) proposals)
+        [_dropped-ids kept dropped] (transitively-drop-dependents proposals initial-dropped)
+        cleaned         (mapv #(validate-proposal-ddl % allowed) kept)]
+    (when (seq dropped)
+      (log/infof "transform-optimizer: dropped %d proposal(s) (cosmetic + orphaned dependents): %s"
+                 (count dropped)
+                 (pr-str (mapv :id dropped))))
+    {:optimization_degree (scoring/optimization-degree cleaned)
+     :proposals           cleaned}))
+
+;; ---------------------------------------------------------------------------
+;; Public entry
+
+(defn optimize!
+  "End-to-end: load the transform, build the prompt, call Claude with our
+  structured-output schema, validate any emitted DDL, score, and return the
+  UI-shaped payload.
+
+  This is the synchronous form — useful from the REPL and from the HTTP
+  endpoint as a fallback path. The streaming endpoint emits the same payload
+  as a sequence of SSE events.
+
+  Options:
+    :analyze?     — forwarded to EXPLAIN. Defaults false. Don't pass true for
+                    transforms whose latest run took more than a few seconds.
+    :llm-opts     — forwarded to `llm/propose-optimizations` (`:model`,
+                    `:temperature`, `:max-tokens`).
+
+  Returns:
+    {:transform           {…}
+     :sql                 <compiled source SQL>
+     :summary             <LLM diagnosis>
+     :proposals           [{… :ddl_statements [{… :validation …}]} …]
+     :optimization_degree <0..100>}"
+  [transform-id & {:keys [llm-opts] :as opts}]
+  (try
+    (let [transform   (fetch-transform transform-id)
+          ctx-opts    (dissoc opts :llm-opts)
+          prompt-map  (apply build-prompt transform (mapcat identity ctx-opts))
+          llm-out     (apply llm/propose-optimizations prompt-map
+                             (mapcat identity (or llm-opts {})))
+          finalised   (finalise-proposals (:proposals llm-out) (:context prompt-map))
+          optimized?  (= 100 (:optimization_degree finalised))]
+      ;; Persist the "fully optimized" verdict so the UI can render its
+      ;; celebratory state without re-running the LLM on every page view.
+      ;; We always write — flipping back to false on a regression is what
+      ;; lets the UI hide the gif if the optimizer later finds proposals.
+      (when (not= optimized? (boolean (:optimized transform)))
+        (t2/update! :model/Transform (:id transform) {:optimized optimized?}))
+      (merge {:transform (-> (select-keys transform [:id :name :source_database_id])
+                             (assoc :optimized optimized?))
+              :sql       (-> prompt-map :context :sql)
+              :summary   (:summary llm-out)}
+             finalised))
+    (catch Exception e
+      (log/errorf e "transform-optimizer: optimize! failed for transform-id=%s" transform-id)
+      (throw e))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/ddl/execute.clj

@@ -0,0 +1,62 @@
+(ns metabase-enterprise.transform-optimizer.ddl.execute
+  "Run a validated `CREATE INDEX` statement against a Database.
+
+  Only `ddl.parse`-validated statements should reach this module — by the
+  time we get here, the statement has already passed the allowlist check
+  (single `CREATE INDEX` on a schema-qualified table from the optimizer's
+  referenced-tables set, no forbidden keywords, etc.). The executor adds
+  the runtime concerns the validator can't enforce: autocommit handling
+  for `CONCURRENTLY`, capturing per-statement success/failure, and not
+  letting one DDL failure cascade into the rest of an accept batch.
+
+  Postgres-only. Non-Postgres drivers are reported as `:skipped` so the
+  caller can present a clean per-statement status to the user."
+  (:require
+   [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+   [metabase.util.log :as log])
+  (:import
+   (java.sql Connection)))
+
+(set! *warn-on-reflection* true)
+
+(defn- run-statement!
+  "Open a fresh connection with autocommit ON, run the statement, return.
+  Autocommit is critical: `CREATE INDEX CONCURRENTLY` cannot run inside an
+  explicit transaction, and the default JDBC connection setup wraps each
+  call in one."
+  [driver-kw database ^String statement]
+  (sql-jdbc.execute/do-with-connection-with-options
+   driver-kw database nil
+   (fn [^Connection conn]
+     (.setAutoCommit conn true)
+     (with-open [stmt (.createStatement conn)]
+       (.execute stmt statement)))))
+
+(defn execute!
+  "Run one validated DDL statement against `database` (a Database record or
+  id). Returns a status map:
+
+      {:status :executed}                            -- success
+      {:status :failed :error-message <pg msg>}      -- statement errored
+      {:status :skipped :error-message <reason>}     -- non-Postgres or no DB
+
+  Idempotent in practice because the optimizer's prelude requires
+  `IF NOT EXISTS`; the validator will already have rejected anything else."
+  [driver-kw database ^String statement]
+  (cond
+    (nil? database)
+    {:status :skipped :error-message "source database not resolved"}
+
+    (not (isa? driver/hierarchy driver-kw :postgres))
+    {:status        :skipped
+     :error-message (str "DDL execution is Postgres-only in this branch (got " driver-kw ")")}
+
+    :else
+    (try
+      (run-statement! driver-kw database statement)
+      {:status :executed}
+      (catch Exception e
+        (log/warnf e "DDL execute! failed for: %s" statement)
+        {:status        :failed
+         :error-message (or (ex-message e) "unknown error")}))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/ddl/parse.clj

@@ -0,0 +1,171 @@
+(ns metabase-enterprise.transform-optimizer.ddl.parse
+  "Allowlist validator for proposed DDL statements.
+
+  Per PLAN.md → Phase 5: anything the LLM emits in `ddl_statements[].statement`
+  must be exactly one `CREATE [UNIQUE] INDEX [CONCURRENTLY] [IF NOT EXISTS]`
+  on a schema-qualified table that appeared in the optimizer's context. This
+  is the primary defence against prompt-injection coercing the LLM into
+  emitting destructive statements: we reject *anything* that doesn't match
+  the canonical CREATE-INDEX shape, regardless of how it got into the payload.
+
+  Intentionally strict. False negatives (we reject a benign statement) are
+  recoverable — the user can copy the statement manually. False positives
+  (we accept a destructive statement) are not."
+  (:require
+   [clojure.string :as str]
+   [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+;; ---------------------------------------------------------------------------
+;; Sanitisation
+;;
+;; Replace single-quoted strings and `-- …` line comments with spaces so a
+;; semicolon hidden inside them can't be used to smuggle a second statement.
+;; We intentionally do NOT handle block comments or dollar-quoted strings:
+;; if the LLM uses them, the safer outcome is to reject the statement.
+
+(defn- strip-strings-and-line-comments ^String [^String sql]
+  (let [sb  (StringBuilder.)
+        n   (.length sql)]
+    (loop [i 0
+           mode :code]
+      (if (>= i n)
+        (.toString sb)
+        (let [c (.charAt sql i)
+              c2 (when (< (inc i) n) (.charAt sql (inc i)))]
+          (case mode
+            :code
+            (cond
+              (and (= c \-) (= c2 \-))
+              (recur (+ i 2) :line-comment)
+
+              (and (= c \/) (= c2 \*))
+              ;; Block comment encountered — bail by leaving the rest opaque.
+              ;; The forbidden-keyword check + regex match against the
+              ;; remaining text will almost certainly fail; that's fine.
+              (do (.append sb \space) (recur (inc i) :code))
+
+              (= c \$)
+              ;; Dollar-quoted string — same treatment: don't try to skip,
+              ;; just refuse to parse beyond it.
+              (do (.append sb \space) (recur (inc i) :code))
+
+              (= c \')
+              (do (.append sb \space) (recur (inc i) :sstring))
+
+              :else
+              (do (.append sb c) (recur (inc i) :code)))
+
+            :sstring
+            (cond
+              ;; '' escape inside a single-quoted string
+              (and (= c \') (= c2 \'))
+              (do (.append sb \space) (.append sb \space) (recur (+ i 2) :sstring))
+
+              (= c \')
+              (do (.append sb \space) (recur (inc i) :code))
+
+              :else
+              (do (.append sb \space) (recur (inc i) :sstring)))
+
+            :line-comment
+            (if (= c \newline)
+              (do (.append sb \newline) (recur (inc i) :code))
+              (do (.append sb \space) (recur (inc i) :line-comment)))))))))
+
+;; ---------------------------------------------------------------------------
+;; Forbidden tokens
+;;
+;; After sanitisation, scan for whole-word tokens that would never appear in
+;; a benign CREATE INDEX. The presence of any one fails the validator.
+
+(def ^:private forbidden-keywords
+  ["DROP" "ALTER" "GRANT" "REVOKE" "TRUNCATE" "DELETE" "UPDATE" "INSERT"
+   "COPY" "VACUUM" "REINDEX" "CLUSTER" "REFRESH" "CALL" "DO" "PERFORM"
+   "EXECUTE" "SET" "RESET" "LISTEN" "NOTIFY" "LOCK"])
+
+(def ^:private forbidden-keywords-re
+  (re-pattern (str "(?i)\\b(?:" (str/join "|" forbidden-keywords) ")\\b")))
+
+(defn- forbidden-keyword [^String sanitised]
+  (some-> (re-find forbidden-keywords-re sanitised) u/upper-case-en))
+
+;; ---------------------------------------------------------------------------
+;; CREATE INDEX shape
+;;
+;; Less ambitious than a real SQL parse: recover the index name, schema, and
+;; table, and assert the statement *starts* with the canonical CREATE INDEX
+;; shape. Anything after the WHERE clause is treated as opaque; the planner
+;; rejects malformed predicates at execution time.
+
+;; An identifier is either bare (\w+) or double-quoted Postgres-style
+;; ("…", with "" as the escape for an inner "). We accept either form for
+;; the index name and for each side of the qualified table.
+(def ^:private ident-pattern "(?:\"(?:[^\"]|\"\")+\"|\\w+)")
+
+(def ^:private create-index-re
+  (re-pattern
+   (str "(?is)\\A\\s*CREATE\\s+(?:UNIQUE\\s+)?INDEX(?:\\s+CONCURRENTLY)?"
+        "(?:\\s+IF\\s+NOT\\s+EXISTS)?\\s+"
+        "(" ident-pattern ")\\s+ON\\s+(?:ONLY\\s+)?"
+        "(" ident-pattern ")\\.(" ident-pattern ")\\s*.*\\z")))
+
+(defn- unquote-ident
+  "Strip a wrapping pair of double-quotes (and collapse the doubled-quote
+  escape) from a Postgres identifier. No-op for bare identifiers."
+  [s]
+  (let [s (str s)]
+    (if (and (> (count s) 1)
+             (= \" (.charAt s 0))
+             (= \" (.charAt s (dec (count s)))))
+      (-> (subs s 1 (dec (count s)))
+          (str/replace "\"\"" "\""))
+      s)))
+
+;; ---------------------------------------------------------------------------
+;; Public API
+
+(defn parse
+  "Validate `statement` against the CREATE-INDEX allowlist.
+
+  `allowed-tables` is a set/seq of `[schema table]` pairs (case-insensitive)
+  drawn from the optimizer's context — only indexes on these tables are
+  accepted.
+
+  On success: `{:ok? true, :name <idx>, :schema <s>, :table <t>}`.
+  On failure: `{:ok? false, :reason <kw>, :detail <string>}`."
+  [statement allowed-tables]
+  (let [sanitised  (strip-strings-and-line-comments (str statement))
+        normalised (str/replace sanitised #"\s+" " ")
+        parts      (->> (str/split normalised #";")
+                        (map str/trim)
+                        (remove str/blank?))
+        bad-kw     (forbidden-keyword sanitised)]
+    (cond
+      bad-kw
+      {:ok? false :reason :forbidden-keyword
+       :detail (str "Statement contains forbidden keyword: " bad-kw)}
+
+      (not= 1 (count parts))
+      {:ok? false :reason :multi-statement
+       :detail (format "%d statements found; only one CREATE INDEX is allowed" (count parts))}
+
+      :else
+      (if-let [[_ idx-name schema table] (re-matches create-index-re (first parts))]
+        (let [idx-name*  (unquote-ident idx-name)
+              schema*    (unquote-ident schema)
+              table*     (unquote-ident table)
+              allowed-lc (into #{}
+                               (map (fn [[s t]]
+                                      [(some-> s u/lower-case-en)
+                                       (some-> t u/lower-case-en)]))
+                               allowed-tables)]
+          (if (contains? allowed-lc [(u/lower-case-en schema*) (u/lower-case-en table*)])
+            {:ok? true :name idx-name* :schema schema* :table table*}
+            {:ok?    false
+             :reason :unknown-table
+             :detail (str schema* "." table* " is not in the referenced-tables set")}))
+        {:ok?    false
+         :reason :not-create-index
+         :detail "Statement does not match: CREATE [UNIQUE] INDEX [CONCURRENTLY] [IF NOT EXISTS] <name> ON <schema>.<table> …"}))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/events.clj

@@ -0,0 +1,64 @@
+(ns metabase-enterprise.transform-optimizer.events
+  "Event-driven post-materialization DDL replay.
+
+  When a transform runs, the source-DB-targeting indices the optimizer
+  proposed are durable — they're on tables we don't touch. But indices on
+  the transform's *target* table get dropped every run because the
+  transform pipeline recreates the table. To make optimizer-proposed
+  indices survive subsequent runs, we persist them on the transform's
+  `:target.post_run_ddl` JSON field at accept time and re-execute them
+  whenever the transform completes.
+
+  This module subscribes to `:event/transform-run-complete` (published by
+  `transforms-base.util/complete-execution!`), looks up the transform's
+  `:target.post_run_ddl`, and runs each statement on a fresh autocommit
+  connection. Failures are logged but never fail the transform run
+  itself — `CREATE INDEX IF NOT EXISTS` is idempotent so the user can
+  re-trigger by running the transform again."
+  (:require
+   [metabase-enterprise.transform-optimizer.ddl.execute :as ddl.execute]
+   [metabase.events.core :as events]
+   [metabase.util.log :as log]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(derive ::transform-optimizer-post-run :metabase/event)
+(derive :event/transform-run-complete ::transform-optimizer-post-run)
+
+(defn- run-post-run-ddl!
+  "Look up the transform's persisted DDL list and execute each statement
+  against the target DB. Returns a vector of `{statement, status, …}`
+  maps suitable for logging."
+  [transform-id db-id]
+  (let [transform (when transform-id
+                    (t2/select-one [:model/Transform :id :target] :id transform-id))
+        ddls      (some-> transform :target :post_run_ddl seq)
+        database  (when db-id (t2/select-one :model/Database :id db-id))
+        driver-kw (some-> database :engine keyword)]
+    (when ddls
+      (log/infof "transform-optimizer: replaying %d post-run DDL statements for transform %s"
+                 (count ddls) transform-id)
+      (mapv (fn [{:keys [statement] :as d}]
+              (let [result (cond
+                             (not (string? statement))
+                             {:status :skipped :error-message "missing statement string"}
+
+                             :else
+                             (ddl.execute/execute! driver-kw database statement))]
+                (when (= :failed (:status result))
+                  (log/warnf "transform-optimizer: post-run DDL failed for transform %s: %s — %s"
+                             transform-id statement (:error-message result)))
+                (merge d result)))
+            ddls))))
+
+(methodical/defmethod events/publish-event! ::transform-optimizer-post-run
+  [_topic {{:keys [transform-id db-id]} :object}]
+  (try
+    (run-post-run-ddl! transform-id db-id)
+    (catch Exception e
+      ;; The transform run already succeeded; failing the event handler
+      ;; would falsely mark it as failed. Log loudly and move on.
+      (log/error e "transform-optimizer: post-run DDL handler errored — transform run unaffected"
+                 {:transform-id transform-id :db-id db-id}))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/explain.clj

@@ -0,0 +1,76 @@
+(ns metabase-enterprise.transform-optimizer.explain
+  "Runs `EXPLAIN (FORMAT JSON, VERBOSE)` against a transform's source DB and
+  parses the result.
+
+  No `ANALYZE` by default: actually executing the slow query is exactly what
+  we are trying to avoid. Callers can opt in (`{:analyze? true}`) when the
+  cost of one more run is acceptable — e.g. when the user requested a
+  full diagnosis from the UI."
+  (:require
+   [clojure.string :as str]
+   [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log])
+  (:import
+   (java.sql Connection)))
+
+(set! *warn-on-reflection* true)
+
+(defn- format-flags [{:keys [analyze? buffers? verbose? settings?]
+                      :or   {verbose? true}}]
+  (cond-> ["FORMAT JSON"]
+    verbose?  (conj "VERBOSE")
+    analyze?  (conj "ANALYZE")
+    buffers?  (conj "BUFFERS")
+    settings? (conj "SETTINGS")))
+
+(defn explain
+  "Run `EXPLAIN (FORMAT JSON, …) <sql>` against `database`. Returns the parsed
+  JSON plan tree, or `nil` if the planner refused. Postgres only — other
+  drivers return `nil` and log a warning.
+
+  Options:
+    :analyze?  - run the query and include real timings. **Defaults false.**
+    :verbose?  - include extra detail (column lists, qualifiers). Defaults true.
+    :buffers?  - include buffer statistics (requires :analyze?). Defaults false.
+    :settings? - include non-default planner settings. Defaults false."
+  ([driver database sql]
+   (explain driver database sql {}))
+  ([driver database sql opts]
+   (cond
+     (not (isa? driver/hierarchy driver :postgres))
+     (do (log/warnf "explain only supports :postgres (got %s)" driver)
+         nil)
+
+     (str/blank? sql)
+     nil
+
+     :else
+     (let [flags  (format-flags opts)
+           prefix (str "EXPLAIN (" (str/join ", " flags) ") ")
+           ;; Strip a trailing semicolon — EXPLAIN <query>; would parse fine
+           ;; but we'd rather feed a clean composable string.
+           sql*   (-> sql str/trim (str/replace #";\s*$" ""))
+           explain-sql (str prefix sql*)]
+       (try
+         (sql-jdbc.execute/do-with-connection-with-options
+          driver database nil
+          (fn [^Connection conn]
+            (with-open [stmt (.createStatement conn)
+                        rs   (.executeQuery stmt explain-sql)]
+              (when (.next rs)
+                ;; FORMAT JSON returns a single column containing a one-element
+                ;; JSON array (Postgres wraps the plan in a top-level array).
+                (let [raw (.getString rs 1)]
+                  (json/decode+kw raw))))))
+         (catch Exception e
+           (log/warnf e "EXPLAIN failed for SQL: %s" (subs sql* 0 (min 200 (count sql*))))
+           nil))))))
+
+(defn plan-cost
+  "Convenience: pull the top-level total-cost out of an EXPLAIN result. Returns
+  a double, or `nil` if the result doesn't have the expected shape."
+  [explain-result]
+  ;; EXPLAIN (FORMAT JSON) returns [{:Plan {:Total Cost <n> :Startup Cost …}}].
+  (some-> explain-result first :Plan (get (keyword "Total Cost"))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/index_introspection.clj

@@ -0,0 +1,151 @@
+(ns metabase-enterprise.transform-optimizer.index-introspection
+  "Postgres-catalog introspection for index detail beyond what
+  `Field.database_indexed` and `driver/describe-table-indexes` give us.
+
+  For every index on the requested set of tables we return its key columns
+  (ordered), INCLUDE columns, partial predicate, index access method
+  (btree / gin / brin / hash / …), uniqueness, and the raw
+  `pg_get_indexdef` definition string. The latter is the most LLM-friendly
+  representation and is the field we expect the prompt template to render
+  by default."
+  (:require
+   [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+   [metabase.util :as u]
+   [metabase.util.log :as log])
+  (:import
+   (java.sql Array Connection ResultSet)))
+
+(set! *warn-on-reflection* true)
+
+;; ---------------------------------------------------------------------------
+;; Postgres catalog query
+;;
+;; We resolve one row per index by joining pg_index → pg_class (table) →
+;; pg_namespace (schema) → pg_class (the index itself) → pg_am (access
+;; method). Key and INCLUDE columns are resolved via two correlated
+;; ARRAY-subqueries against pg_attribute so we preserve column order and
+;; keep the two groups distinct (indnkeyatts splits them).
+;;
+;; The table-set filter uses `unnest(?::text[], ?::text[])` so we can match
+;; on (schema, name) pairs in a single round trip.
+
+(def ^:private indexes-sql
+  "WITH refs(schema_name, table_name) AS (
+     SELECT s, t FROM unnest(?::text[], ?::text[]) AS u(s, t)
+   )
+   SELECT
+     n.nspname                                                              AS schema_name,
+     c.relname                                                              AS table_name,
+     i.relname                                                              AS index_name,
+     am.amname                                                              AS access_method,
+     ix.indisunique                                                         AS is_unique,
+     ix.indisprimary                                                        AS is_primary,
+     ix.indisvalid                                                          AS is_valid,
+     pg_get_expr(ix.indpred,  ix.indrelid)                                  AS partial_predicate,
+     pg_get_expr(ix.indexprs, ix.indrelid)                                  AS index_expressions,
+     pg_get_indexdef(ix.indexrelid)                                         AS index_def,
+     ix.indnatts                                                            AS n_total_atts,
+     ix.indnkeyatts                                                         AS n_key_atts,
+     ARRAY(
+       SELECT a.attname
+       FROM unnest(ix.indkey[0:ix.indnkeyatts - 1]) WITH ORDINALITY AS k(attnum, ord)
+       LEFT JOIN pg_attribute a ON a.attrelid = ix.indrelid AND a.attnum = k.attnum
+       ORDER BY k.ord
+     )                                                                      AS key_columns,
+     ARRAY(
+       SELECT a.attname
+       FROM unnest(ix.indkey[ix.indnkeyatts : ix.indnatts - 1]) WITH ORDINALITY AS k(attnum, ord)
+       LEFT JOIN pg_attribute a ON a.attrelid = ix.indrelid AND a.attnum = k.attnum
+       ORDER BY k.ord
+     )                                                                      AS include_columns
+   FROM pg_index ix
+   JOIN pg_class      c  ON c.oid  = ix.indrelid
+   JOIN pg_class      i  ON i.oid  = ix.indexrelid
+   JOIN pg_namespace  n  ON n.oid  = c.relnamespace
+   JOIN pg_am        am  ON am.oid = i.relam
+   JOIN refs          r  ON r.schema_name = n.nspname AND r.table_name = c.relname
+   ORDER BY c.relname, i.relname;")
+
+(defn- array->vec [^Array a]
+  (when a (vec (.getArray a))))
+
+(defn- row->index-info [^ResultSet rs]
+  {:schema             (.getString rs "schema_name")
+   :table              (.getString rs "table_name")
+   :name               (.getString rs "index_name")
+   :access_method      (.getString rs "access_method")
+   :is_unique          (.getBoolean rs "is_unique")
+   :is_primary         (.getBoolean rs "is_primary")
+   :is_valid           (.getBoolean rs "is_valid")
+   :partial_predicate  (.getString rs "partial_predicate")
+   :index_expressions  (.getString rs "index_expressions")
+   :definition         (.getString rs "index_def")
+   :key_columns        (array->vec (.getArray rs "key_columns"))
+   :include_columns    (array->vec (.getArray rs "include_columns"))})
+
+(defn- query-indexes [^Connection conn pairs]
+  (let [schemas (into-array String (map (comp str first)  pairs))
+        tables  (into-array String (map (comp str second) pairs))]
+    (with-open [stmt (.prepareStatement conn indexes-sql)]
+      (.setArray stmt 1 (.createArrayOf conn "text" schemas))
+      (.setArray stmt 2 (.createArrayOf conn "text" tables))
+      (with-open [rs (.executeQuery stmt)]
+        (loop [out []]
+          (if (.next rs)
+            (recur (conj out (row->index-info rs)))
+            out))))))
+
+;; ---------------------------------------------------------------------------
+;; Public API
+
+(defn- normalise-pairs
+  "Accept either a single `[schema table]` pair or a sequence of them. Returns
+  a sequence of pairs. This lets REPL callers write
+  `(fetch-indexes drv db [\"shop\" \"orders\"])` without ceremony while still
+  supporting the bulk shape `[[\"shop\" \"orders\"] [\"shop\" \"events\"]]`."
+  [table-pairs]
+  (cond
+    (and (sequential? table-pairs)
+         (= 2 (count table-pairs))
+         (every? string? table-pairs))
+    [table-pairs]
+
+    :else
+    table-pairs))
+
+(defn fetch-indexes
+  "Look up full index detail for the given `[[schema table] …]` pairs against
+  `database`. A single `[schema table]` pair is also accepted as a
+  convenience. Returns a vector of index info maps; each carries the
+  `:definition` (raw `CREATE INDEX …` text from `pg_get_indexdef`) plus
+  parsed-out structured fields (`:key_columns`, `:include_columns`,
+  `:access_method`, `:partial_predicate`, …).
+
+  Postgres only. Other drivers return `nil` and emit a warning — callers
+  should fall back to `driver/describe-table-indexes` and tag the resulting
+  context as partial."
+  [driver database table-pairs]
+  (let [pairs (normalise-pairs table-pairs)]
+    (cond
+      (not (isa? driver/hierarchy driver :postgres))
+      (do (log/warnf "index-introspection only supports :postgres (got %s); falling back" driver)
+          nil)
+
+      (empty? pairs)
+      []
+
+      :else
+      (sql-jdbc.execute/do-with-connection-with-options
+       driver database nil
+       (fn [^Connection conn] (query-indexes conn pairs))))))
+
+(defn group-by-table
+  "Turn the flat seq returned by `fetch-indexes` into a `{[schema table] [indexes…]}`
+  map, normalising schema and table to lowercase (Postgres unquoted-identifier
+  semantics) so callers can look up entries with values pulled from
+  `pg_namespace.nspname` / `pg_class.relname` regardless of source casing."
+  [indexes]
+  (->> indexes
+       (group-by (fn [{:keys [schema table]}]
+                   [(some-> schema u/lower-case-en) (some-> table u/lower-case-en)]))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/indexes.clj

@@ -0,0 +1,239 @@
+(ns metabase-enterprise.transform-optimizer.indexes
+  "List + drop indices on the tables a transform touches.
+
+  \"Touches\" = the **target** table (the one the transform materialises)
+  plus every **source** table the transform reads from. Both matter to
+  the optimizer:
+    - Indices on source tables back the rewrite proposals (e.g.
+      `idx_orders_customer_id` for a GROUP BY rollup).
+    - Indices on the target table speed up downstream consumers and are
+      what the optimizer's post-run replay machinery keeps alive.
+
+  Each result distinguishes between:
+    :managed_by_optimizer true  — the index appears in
+                                  `transform.target.post_run_ddl` so the
+                                  optimizer's replay will recreate it on
+                                  every transform run. Only target-table
+                                  indices can be managed this way today —
+                                  source-table indices live in
+                                  user-controlled tables.
+    :managed_by_optimizer false — pre-existing, hand-rolled, or a
+                                  source-DB index the optimizer ran at
+                                  accept time but doesn't replay."
+  (:require
+   [metabase-enterprise.transform-optimizer.ddl.execute :as ddl.execute]
+   [metabase-enterprise.transform-optimizer.ddl.parse :as ddl.parse]
+   [metabase-enterprise.transform-optimizer.index-introspection :as iix]
+   [metabase.driver :as driver]
+   [metabase.query-processor.preprocess :as qp.preprocess]
+   [metabase.transforms-base.util :as transforms-base.u]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;; ---------------------------------------------------------------------------
+;; Resolving which tables to list
+
+(defn- target-pair
+  "[schema name] of the transform's target table, or nil if not set yet."
+  [transform]
+  (let [{:keys [schema name]} (:target transform)]
+    (when (and schema name) [schema name])))
+
+(defn- source-pairs
+  "[[schema table] …] for every source table the transform reads from.
+  Resolved via `driver/native-query-deps` so we cover native queries
+  uniformly (MBQL transforms wouldn't be in the optimizer's scope anyway
+  in this branch)."
+  [transform]
+  (try
+    (let [query    (-> transform :source :query)
+          db-id    (:source_database_id transform)
+          driver-kw (when db-id
+                      (t2/select-one-fn (comp keyword :engine)
+                                        :model/Database :id db-id))]
+      (when (and driver-kw query)
+        (let [preprocessed (-> query
+                               transforms-base.u/massage-sql-query
+                               qp.preprocess/preprocess)
+              deps         (driver/native-query-deps driver-kw preprocessed)
+              table-ids    (into [] (keep :table) deps)
+              tables       (when (seq table-ids)
+                             (t2/select [:model/Table :id :schema :name]
+                                        :id [:in table-ids]))]
+          (vec (for [t tables
+                     :when (and (:schema t) (:name t))]
+                 [(:schema t) (:name t)])))))
+    (catch Exception e
+      (log/warnf e "list-indexes: failed to extract source tables for transform %s"
+                 (:id transform))
+      [])))
+
+(defn- referenced-table-pairs
+  "Union of target + source tables. Preserves order (target first, sources
+  in resolution order) and deduplicates."
+  [transform]
+  (let [target  (target-pair transform)
+        sources (source-pairs transform)
+        seen    (volatile! #{})
+        keep!   (fn [pair]
+                  (when pair
+                    (let [key [(some-> (first pair) u/lower-case-en)
+                               (some-> (second pair) u/lower-case-en)]]
+                      (when-not (@seen key)
+                        (vswap! seen conj key)
+                        pair))))]
+    (into [] (keep keep!) (cons target sources))))
+
+;; ---------------------------------------------------------------------------
+;; Listing
+
+(defn- ddl-statement->index-name
+  "Extract the index name from a `CREATE INDEX … <name> ON …` statement
+  by reusing the validator (which already parses the name out)."
+  [statement allowed-tables]
+  (let [r (ddl.parse/parse statement allowed-tables)]
+    (when (:ok? r) (:name r))))
+
+(defn- managed-index-names
+  "Names of indices the optimizer registered for replay on this transform's
+  target. Filtered to those that parse cleanly as CREATE INDEX (a sanity
+  guard — anything stored in `post_run_ddl` should already have passed
+  the validator at accept time)."
+  [transform allowed-tables]
+  (->> (get-in transform [:target :post_run_ddl])
+       (keep #(ddl-statement->index-name (:statement %) allowed-tables))
+       set))
+
+(defn list-indexes
+  "List every index on the union of `transform`'s target table and source
+  tables. Returns a vector shaped:
+
+    {:schema :table :name :access_method :is_unique :is_primary
+     :is_valid :key_columns :include_columns :partial_predicate
+     :definition :managed_by_optimizer :is_target_table}
+
+  Each row carries `:schema`/`:table` so callers can group results per
+  table. `:is_target_table` lets the UI visually distinguish the
+  output-side table from input tables. `:managed_by_optimizer` is only
+  ever `true` for indices on the target table (source-table indices
+  aren't replayed)."
+  [transform]
+  (let [pairs     (referenced-table-pairs transform)
+        target    (target-pair transform)
+        db-id     (:source_database_id transform)
+        database  (when db-id (t2/select-one :model/Database :id db-id))
+        driver-kw (some-> database :engine keyword)]
+    (cond
+      (empty? pairs)
+      []
+
+      (nil? database)
+      (do (log/warnf "list-indexes: no source database for transform %s" (:id transform))
+          [])
+
+      :else
+      (let [rows          (or (iix/fetch-indexes driver-kw database pairs) [])
+            managed-names (when target (managed-index-names transform #{target}))
+            target-schema (some-> target first  u/lower-case-en)
+            target-table  (some-> target second u/lower-case-en)
+            on-target? (fn [row]
+                         (and (some-> (:schema row) u/lower-case-en (= target-schema))
+                              (some-> (:table  row) u/lower-case-en (= target-table))))]
+        (mapv (fn [row]
+                (let [target-side? (on-target? row)]
+                  (-> row
+                      (assoc :is_target_table      target-side?
+                             :managed_by_optimizer (boolean (and target-side?
+                                                                 managed-names
+                                                                 (managed-names (:name row))))))))
+              rows)))))
+
+;; ---------------------------------------------------------------------------
+;; Dropping
+
+(defn- safe-identifier?
+  "Pre-flight check: index / schema / table names that round-trip cleanly
+  through bare unquoted Postgres identifier syntax. We refuse to drop
+  anything with shell-y characters — the names we list come from
+  `pg_class.relname` so legitimate ones always pass."
+  [s]
+  (and (string? s)
+       (boolean (re-matches #"[A-Za-z_][A-Za-z0-9_]*" s))))
+
+(defn- remove-from-post-run-ddl!
+  "Atomically strip any post_run_ddl entry that produces `index-name` so
+  the next transform run doesn't re-create what we just dropped."
+  [transform-id index-name allowed-tables]
+  (t2/with-transaction [_]
+    (when-let [transform (t2/select-one [:model/Transform :id :target] :id transform-id)]
+      (let [ddls   (get-in transform [:target :post_run_ddl])
+            kept   (vec (remove (fn [{:keys [statement]}]
+                                  (= index-name (ddl-statement->index-name statement allowed-tables)))
+                                ddls))]
+        (when-not (= (count ddls) (count kept))
+          (let [target' (if (seq kept)
+                          (assoc (:target transform) :post_run_ddl kept)
+                          (dissoc (:target transform) :post_run_ddl))]
+            (t2/update! :model/Transform transform-id {:target target'})))))))
+
+(defn drop-index!
+  "Drop the named index on any of `transform`'s referenced tables
+  (target + sources). Validates that the index actually exists on one of
+  those tables before issuing the DROP — we won't drop arbitrary indices
+  elsewhere in the source DB. Returns:
+    {:status :dropped}
+    {:status :failed   :error_message <msg>}
+    {:status :skipped  :reason :index-not-on-referenced-table
+                              | :unsafe-name | :not-postgres | :no-database}"
+  [transform index-name]
+  (let [pairs     (referenced-table-pairs transform)
+        target    (target-pair transform)
+        db-id     (:source_database_id transform)
+        database  (when db-id (t2/select-one :model/Database :id db-id))
+        driver-kw (some-> database :engine keyword)]
+    (cond
+      (not (safe-identifier? index-name))
+      {:status :skipped :reason :unsafe-name}
+
+      (nil? database)
+      {:status :skipped :reason :no-database}
+
+      (empty? pairs)
+      {:status :skipped :reason :index-not-on-referenced-table}
+
+      :else
+      (let [;; Pre-flight: confirm the index actually exists on one of the
+            ;; referenced tables (target or source). Capture which one so we
+            ;; can issue `DROP INDEX schema.name` against the right schema.
+            existing-row (some (fn [row]
+                                 (when (= index-name (:name row)) row))
+                               (or (iix/fetch-indexes driver-kw database pairs) []))]
+        (if-not existing-row
+          {:status :skipped :reason :index-not-on-referenced-table}
+          (let [schema      (:schema existing-row)
+                stmt        (format "DROP INDEX CONCURRENTLY IF EXISTS %s.%s"
+                                    (str schema) (str index-name))
+                exec-result (ddl.execute/execute! driver-kw database stmt)
+                result      (case (:status exec-result)
+                              :executed {:status :dropped}
+                              :failed   {:status :failed
+                                         :error_message (:error-message exec-result)}
+                              :skipped  {:status :skipped :reason :not-postgres})]
+            (when (and (= :dropped (:status result))
+                       target
+                       (= (some-> (:schema existing-row) u/lower-case-en)
+                          (some-> (first target)         u/lower-case-en))
+                       (= (some-> (:table existing-row)  u/lower-case-en)
+                          (some-> (second target)        u/lower-case-en)))
+              ;; Only scrub `post_run_ddl` when we dropped an index *on the
+              ;; target table*. Source-table indices were never tracked
+              ;; there in the first place.
+              (try
+                (remove-from-post-run-ddl! (:id transform) index-name #{target})
+                (catch Exception e
+                  (log/warnf e "drop-index!: failed to scrub post_run_ddl for %s on transform %s"
+                             index-name (:id transform)))))
+            result))))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/init.clj

@@ -0,0 +1,5 @@
+(ns metabase-enterprise.transform-optimizer.init
+  "Side-effecting requires for the transform-optimizer module. Loaded at
+  system launch via `metabase-enterprise.core.init`."
+  (:require
+   [metabase-enterprise.transform-optimizer.events]))

enterprise/backend/src/metabase_enterprise/transform_optimizer/llm.clj

@@ -0,0 +1,169 @@
+(ns metabase-enterprise.transform-optimizer.llm
+  "Calls the LLM with our prelude + per-transform context and a
+  structured-output JSON schema, returning the parsed proposal payload.
+
+  We use `metabot.self/call-llm-structured` rather than calling the Claude
+  client directly — it gives us retry-with-exponential-backoff, token-usage
+  telemetry, and provider abstraction (we can swap to OpenAI / OpenRouter
+  via the provider-and-model string).
+
+  Trade-off: `call-llm-structured` doesn't support a separate `system` field,
+  so we concatenate prelude + context into one user message. Cache utility
+  is slightly worse than a dedicated system prompt would give us, but this
+  keeps the dependency surface narrow.
+
+  Today the call is buffered (we consume the whole stream before returning).
+  The HTTP streaming endpoint then re-emits the parts as SSE events. The
+  client-facing contract stays the same when we upgrade to incremental
+  streaming later."
+  (:require
+   [metabase.metabot.self :as metabot.self]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+;; ---------------------------------------------------------------------------
+;; JSON Schema for the structured output
+;;
+;; Fed to the provider as the `schema` arg to `call-llm-structured`. The
+;; shape must match what `core/finalise-proposals` expects.
+
+(def ^:private ddl-target-schema
+  {:type "string"
+   :enum ["source-db" "transform-target"]
+   :description (str "source-db: run on the original transform's source database. "
+                     "transform-target: index a table created by a depends_on proposal; "
+                     "use depends_on to point at the proposal whose target the index belongs to.")})
+
+(def ^:private ddl-statement-schema
+  ;; We deliberately leave `additionalProperties` open and `id` optional —
+  ;; the LLM has muscle memory from earlier prelude drafts and sometimes
+  ;; still emits `id`. Anthropic's structured-output rejects the entire
+  ;; tool call when even one field violates a strict schema, so being
+  ;; permissive here keeps the round-trip alive.
+  {:type "object"
+   :properties {:id        {:type "string"}
+                :target    ddl-target-schema
+                :statement {:type "string"
+                            :description "Single CREATE INDEX [CONCURRENTLY] [IF NOT EXISTS] …"}
+                :rationale {:type "string"}}
+   :required ["target" "statement" "rationale"]})
+
+(def ^:private proposal-schema
+  ;; One proposal = one change. Either a rewrite/precompute (carries `body`,
+  ;; no `ddl_statement`) or a single index (carries `ddl_statement`, no
+  ;; `body`). We declare both fields as optional and accept either
+  ;; `ddl_statement` (new shape) or `ddl_statements` (older shape) — the BE
+  ;; normalises on parse. `additionalProperties` is intentionally open;
+  ;; with it strict, a single muscle-memory extra field aborts the whole
+  ;; structured-output call and we lose every proposal.
+  {:type "object"
+   :properties {:id               {:type "string"}
+                :name             {:type "string"}
+                :kind             {:type "string"
+                                   :enum ["rewrite" "index" "precompute"]}
+                :severity         {:type "string"
+                                   :enum ["high" "medium" "low"]}
+                :rationale        {:type "string"}
+                :expected_speedup {:type "string"}
+                :body             {:type ["string" "null"]
+                                   :description "SQL body for kind = rewrite | precompute; null for kind = index"}
+                :target_table     {:type ["string" "null"]
+                                   :description (str "Literal bare table name (no schema, snake_case) the BE should "
+                                                     "materialize this precompute into. Required when kind = "
+                                                     "precompute. Any dependent proposal MUST reference this "
+                                                     "table via `<original transform's schema>.<target_table>` "
+                                                     "verbatim — names cannot drift between proposals.")}
+                :ddl_statement    (merge ddl-statement-schema
+                                         {:description "Single CREATE INDEX for kind = index; omit for kind = rewrite | precompute"})
+                :ddl_statements   {:type "array"
+                                   :items ddl-statement-schema
+                                   :description "Legacy plural form (older prelude). The BE coerces a single-element array to ddl_statement."}
+                :depends_on       {:type "array" :items {:type "string"}}}
+   :required ["id" "name" "kind" "severity" "rationale" "expected_speedup"
+              "depends_on"]})
+
+(def output-schema
+  "JSON Schema for the LLM's structured response — `{summary, proposals[]}`."
+  {:type "object"
+   :properties {:summary   {:type "string"}
+                :proposals {:type "array" :items proposal-schema}}
+   :required ["summary" "proposals"]})
+
+;; ---------------------------------------------------------------------------
+;; Defaults
+
+(def ^:private default-provider-and-model
+  "Optimizer reasoning over SQL benefits from a stronger model; Sonnet is a
+  good balance of latency / quality / cost for the hackathon. Override per
+  call via the `:provider-and-model` option."
+  "anthropic/claude-sonnet-4-6")
+
+(def ^:private default-temperature 0.2)
+(def ^:private default-max-tokens  4096)
+
+;; ---------------------------------------------------------------------------
+;; Public API
+
+(defn- compose-user-message
+  "Combine the prelude and the per-transform context into a single string.
+  The `call-llm-structured` API doesn't expose a separate system slot, so
+  we use a clear delimiter and let the model treat the prelude as
+  authoritative instructions."
+  [{:keys [system user]}]
+  (str (or system "")
+       "\n\n---\n\n"
+       "# Transform to optimise\n\n"
+       (or user "")))
+
+(defn propose-optimizations
+  "Call the LLM with the optimizer prelude + rendered context and return the
+  parsed `{summary, proposals[]}` map.
+
+  `prompt-map` is the output of `core/build-prompt`:
+    {:system  <prelude markdown>
+     :user    <rendered context>
+     :context <raw context, unused here>}
+
+  Options:
+    :provider-and-model — defaults to `anthropic/claude-sonnet-4-6`
+    :temperature         — defaults to 0.2
+    :max-tokens          — defaults to 4096
+    :tracking            — extra map merged into the telemetry tracking-opts
+                           (see metabase.metabot.self/call-llm)"
+  [prompt-map & {:keys [provider-and-model temperature max-tokens tracking]
+                 :or   {provider-and-model default-provider-and-model
+                        temperature        default-temperature
+                        max-tokens         default-max-tokens}}]
+  (let [messages [{:role "user" :content (compose-user-message prompt-map)}]
+        track    (merge {:source "transform-optimizer"
+                         :tag    "optimize-proposals"} tracking)]
+    (try
+      (let [result (metabot.self/call-llm-structured
+                    provider-and-model
+                    messages
+                    output-schema
+                    temperature
+                    max-tokens
+                    track)
+            proposals (:proposals result)
+            kind-counts (frequencies (map :kind proposals))]
+        (log/infof "transform-optimizer: LLM returned %d proposal(s) — by kind: %s"
+                   (count proposals)
+                   (pr-str (or (not-empty kind-counts) "none")))
+        (when (and (seq proposals)
+                   (every? #(and (nil? (:ddl_statement %))
+                                 (nil? (not-empty (:ddl_statements %)))
+                                 (nil? (:body %)))
+                           proposals))
+          (log/warnf (str "transform-optimizer: every proposal is empty (no body, no DDL). "
+                          "This usually means the LLM hit a structured-output schema rejection; "
+                          "check the prelude for shape mismatches.")))
+        result)
+      (catch Exception e
+        (log/error e "transform-optimizer: LLM call failed")
+        (throw (ex-info "LLM call failed"
+                        {:status-code 502
+                         :cause       (ex-message e)
+                         :retryable   (boolean (-> e ex-data :status (#{408 409 429 502 503 504})))}
+                        e))))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/prelude.clj

@@ -0,0 +1,35 @@
+(ns metabase-enterprise.transform-optimizer.prelude
+  "Loads the static prelude that anchors the LLM's system prompt.
+
+  The prelude is a single curated markdown file under
+  `resources/transform_optimizer/prelude.md`. It encodes:
+
+    - what \"equivalent\" means (no semantic drift)
+    - the JSON output schema the LLM must emit
+    - the severity rubric (so server-side scoring matches what the model
+      was told to do)
+    - 4 worked pre→post examples covering the four proposal kinds
+    - constraints on emitted CREATE INDEX statements
+
+  Loaded once and cached. Edit the .md file and restart to pick up changes;
+  reloading on each call would burn a classpath read per request."
+  (:require
+   [clojure.java.io :as io]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private resource-path "transform_optimizer/prelude.md")
+
+(def ^:private cached
+  (delay
+    (if-let [r (io/resource resource-path)]
+      (slurp r)
+      (do (log/errorf "transform-optimizer: prelude resource not found at %s" resource-path)
+          ""))))
+
+(defn prelude
+  "Return the prelude markdown as a single string. Cached for the life of the
+  JVM."
+  []
+  @cached)

enterprise/backend/src/metabase_enterprise/transform_optimizer/prompt.clj

@@ -0,0 +1,127 @@
+(ns metabase-enterprise.transform-optimizer.prompt
+  "Render the optimizer context map into the markdown block the LLM consumes,
+  per PLAN.md → Phase 2."
+  (:require
+   [clojure.string :as str]
+   [metabase.util.json :as json]))
+
+(set! *warn-on-reflection* true)
+
+;; ---------------------------------------------------------------------------
+;; Helpers
+
+(defn- format-fk [{:keys [target_schema target_table target_column]}]
+  (when (and target_schema target_table target_column)
+    (format "FK → %s.%s.%s" target_schema target_table target_column)))
+
+(defn- format-num [n]
+  ;; Trim insignificant trailing zeros for double stats so we don't waste
+  ;; tokens on "1.0000000". Integers pass through as-is.
+  (cond
+    (nil? n)         nil
+    (integer? n)     (str n)
+    (double? n)      (let [s (format "%.4g" (double n))]
+                       (-> s
+                           (str/replace #"0+$" "")
+                           (str/replace #"\.$" "")))
+    :else            (str n)))
+
+(defn- format-stats
+  "Compact one-line summary of fingerprint stats. Each clause is emitted
+  only when the underlying datum is present, so partial fingerprints
+  degrade gracefully. Returns nil when there's nothing to say."
+  [{:keys [distinct_count nil_percent min max avg earliest latest]}]
+  (let [parts (cond-> []
+                distinct_count        (conj (format "ndv≈%s" (format-num distinct_count)))
+                nil_percent           (conj (format "null=%s%%" (format-num (* 100.0 (double nil_percent)))))
+                (and min max)         (conj (format "range=[%s,%s]" (format-num min) (format-num max)))
+                avg                   (conj (format "avg=%s" (format-num avg)))
+                (and earliest latest) (conj (format "time=[%s..%s]" earliest latest)))]
+    (when (seq parts)
+      (str "{" (str/join " " parts) "}"))))
+
+(defn- format-field [visited? {:keys [id name base_type semantic_type indexed? foreign_key stats]}]
+  (let [show-stats? (and stats (visited? id))
+        parts (cond-> [(format "  - %s : %s" name (or base_type "?"))]
+                semantic_type    (conj (format "/ %s" (clojure.core/name semantic_type)))
+                indexed?         (conj "[indexed]")
+                foreign_key      (conj (str "[" (format-fk foreign_key) "]"))
+                show-stats?      (conj (format-stats stats)))]
+    (str/join " " (remove nil? parts))))
+
+(defn- format-index [{:keys [name access_method is_unique is_primary
+                             key_columns include_columns partial_predicate
+                             definition]}]
+  (let [tag (cond
+              is_primary "PK"
+              is_unique  "UNIQUE"
+              :else      "INDEX")
+        cols (str/join ", " key_columns)
+        inc  (when (seq include_columns)
+               (str " INCLUDE (" (str/join ", " include_columns) ")"))
+        pred (when (seq partial_predicate)
+               (str " WHERE " partial_predicate))]
+    (format "  - %s %s [%s] (%s)%s%s\n      %s"
+            tag name access_method cols
+            (or inc "") (or pred "")
+            (or definition ""))))
+
+(defn- format-source [visited? {:keys [schema table_name column_count fields indexes]}]
+  (let [fields-block  (when (seq fields)
+                        (str/join "\n" (map (partial format-field visited?) fields)))
+        indexes-block (when (seq indexes)
+                        (str "\n  indexes:\n" (str/join "\n" (map format-index indexes))))
+        no-index-note (when (empty? indexes)
+                        "\n  indexes: (none beyond PK)")]
+    (str (format "### %s.%s (%d columns)\n" schema table_name (or column_count 0))
+         (or fields-block "")
+         (or indexes-block no-index-note))))
+
+(defn- format-run [{:keys [start_time status run_method duration_ms]}]
+  (format "- %s : %s [%s] in %sms"
+          start_time
+          (clojure.core/name (or status :unknown))
+          (clojure.core/name (or run_method :manual))
+          (or duration_ms "?")))
+
+;; ---------------------------------------------------------------------------
+;; Public API
+
+(defn render-context
+  "Render the context map from `context/build-context` into the markdown
+  payload that goes into the user-side of the LLM prompt. The system-side
+  comes from `prelude/prelude`."
+  [{:keys [transform sql sources target explain run_history indexes_partial? visited_fields]}]
+  (let [target-name (when target
+                      (format "%s.%s" (:schema target) (:table_name target)))
+        ;; visited-fields gates fingerprint stats per field — without the
+        ;; gate we'd spend tokens rendering ndv/min/max for every column
+        ;; on every source table. When the inspector couldn't resolve any
+        ;; visited fields (it failed, or returned an empty set), we skip
+        ;; stats entirely; that matches the pre-stats baseline behaviour.
+        visited?    (if (seq visited_fields)
+                      (set visited_fields)
+                      (constantly false))
+        runs-block  (if (seq run_history)
+                      (str/join "\n" (map format-run run_history))
+                      "- (no run history)")
+        sources-block (if (seq sources)
+                        (str/join "\n\n" (map (partial format-source visited?) sources))
+                        "_no referenced tables resolved_")
+        explain-block (if explain
+                        (str "```json\n" (json/encode explain) "\n```")
+                        "_EXPLAIN unavailable_")
+        partial-note  (when indexes_partial?
+                        "\n\n> ⚠ Index detail was unavailable for some tables; treat the indexes list as partial.")]
+    (str/join "\n\n"
+              ["## Transform"
+               (format "- id: %s\n- name: %s\n- target: %s"
+                       (:id transform) (:name transform) (or target-name "(none / unrun)"))
+               "## SQL"
+               (str "```sql\n" (or sql "-- (compilation failed)") "\n```")
+               "## Referenced tables"
+               (str sources-block (or partial-note ""))
+               "## EXPLAIN (FORMAT JSON, VERBOSE)"
+               explain-block
+               "## Run history (most recent first)"
+               runs-block])))

enterprise/backend/src/metabase_enterprise/transform_optimizer/proposal_cache.clj

@@ -0,0 +1,95 @@
+(ns metabase-enterprise.transform-optimizer.proposal-cache
+  "Short-lived in-memory store of proposals emitted by the optimizer
+  streaming endpoint, keyed by `(user-id, transform-id, proposal-id)`.
+
+  Lets `/verify` and `/accept` accept just the proposal id from the FE
+  (matching the wire shape we documented in SUMMARY.md) without
+  requiring the user to re-send the full proposal payload. The proper
+  persistent home is `:model/TransformOptimizerProposal` (BE-6); this
+  module is a stand-in until that lands so the API contract doesn't
+  change when persistence is added.
+
+  Properties:
+    - User-isolated: a proposal cached for user A can't be retrieved
+      by user B.
+    - TTL: entries expire after `ttl-ms` (default 1 hour) so a stale
+      proposal can't be accepted after the underlying transform / data
+      have moved on.
+    - Process-local: a Metabase restart drops the cache. The FE should
+      handle `404 proposal_not_found` by re-running `/optimize`."
+  (:require
+   [clojure.string :as str]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private ttl-ms
+  "Cache TTL: how long after `put-all!` a proposal is retrievable. One
+  hour balances 'user has time to read & verify before clicking accept'
+  against 'don't accept stale advice'."
+  (* 60 60 1000))
+
+(def ^:private store
+  "{[user-id transform-id proposal-id] {:proposal {…} :stored-at <epoch-ms>}}"
+  (atom {}))
+
+(defn- expired? [^long stored-at ^long now]
+  (> (- now stored-at) ^long ttl-ms))
+
+(defn evict-expired!
+  "Drop entries older than `ttl-ms`. Cheap and idempotent. Called
+  opportunistically on each `put-all!` so we don't need a separate
+  scheduled job."
+  []
+  (let [now (System/currentTimeMillis)]
+    (swap! store (fn [s]
+                   (into {} (remove (fn [[_ {:keys [stored-at]}]]
+                                      (expired? stored-at now)))
+                         s)))))
+
+(defn put-all!
+  "Cache every proposal in `proposals` under `(user-id, transform-id,
+  proposal :id)`. Proposals with a nil or blank `:id` are skipped."
+  [user-id transform-id proposals]
+  (when (seq proposals)
+    (evict-expired!)
+    (let [now (System/currentTimeMillis)]
+      (swap! store
+             (fn [s]
+               (reduce (fn [s {pid :id :as p}]
+                         (if (and pid (not (str/blank? (str pid))))
+                           (assoc s [user-id transform-id pid]
+                                  {:proposal p :stored-at now})
+                           s))
+                       s
+                       proposals))))
+    (log/debugf "transform-optimizer proposal cache: stored %d proposals (user=%s transform=%s)"
+                (count proposals) user-id transform-id)))
+
+(defn get-one
+  "Return the cached proposal, or `nil` if missing or expired."
+  [user-id transform-id proposal-id]
+  (let [{:keys [proposal stored-at]} (get @store [user-id transform-id proposal-id])]
+    (when (and proposal
+               (not (expired? stored-at (System/currentTimeMillis))))
+      proposal)))
+
+(defn get-many
+  "Return `[proposals missing-ids]` for the requested ids. `proposals`
+  preserves input order; `missing-ids` is the subset that wasn't found
+  in the cache. Either may be empty."
+  [user-id transform-id proposal-ids]
+  (reduce (fn [[ps missing] pid]
+            (if-let [p (get-one user-id transform-id pid)]
+              [(conj ps p) missing]
+              [ps (conj missing pid)]))
+          [[] []]
+          proposal-ids))
+
+;; -- test hook ---------------------------------------------------------------
+
+(defn clear-all!
+  "Reset the cache. Test-only — used by `proposal-cache-test`."
+  {:test-only? true}
+  []
+  (reset! store {}))

enterprise/backend/src/metabase_enterprise/transform_optimizer/scoring.clj

@@ -0,0 +1,42 @@
+(ns metabase-enterprise.transform-optimizer.scoring
+  "Deterministic mapping from a proposal set to an `optimization_degree`
+  in [0, 100]. Server-side computed so the LLM doesn't have to self-calibrate.
+
+  Rubric (from PLAN.md → Phase 2 → Optimization degree):
+
+      optimization_degree = 100 - clamp(Σ weight(severity_i), 0, 100)
+
+      weight(:high)   = 30
+      weight(:medium) = 15
+      weight(:low)    =  5
+
+  Empty proposals ⇒ 100. The LLM is told to *not* invent low-impact rewrites
+  just to look productive — that would deflate the score artificially. The
+  validator here only computes the rubric; it does not police severity."
+  (:require
+   [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private weights
+  "Severity → score weight. Edit here, in one place, if the rubric is tuned."
+  {:high   30
+   :medium 15
+   :low     5})
+
+(defn- proposal-weight
+  "Weight of a single proposal. Unknown severities count as `:low` — the
+  scoring function is forgiving so a slightly-off LLM response still produces
+  a sensible score rather than throwing."
+  [{:keys [severity]}]
+  (get weights (keyword severity) (:low weights)))
+
+(mu/defn optimization-degree :- [:int {:min 0 :max 100}]
+  "Compute the optimization degree of the original transform given the set of
+  proposals the LLM produced. See namespace docstring for the rubric.
+
+  `proposals` is a sequence of maps each containing at least
+  `:severity ∈ #{:high :medium :low}`."
+  [proposals :- [:maybe [:sequential map?]]]
+  (let [penalty (reduce + 0 (map proposal-weight proposals))]
+    (max 0 (- 100 (min 100 penalty)))))

enterprise/backend/src/metabase_enterprise/transform_optimizer/verify.clj

@@ -0,0 +1,311 @@
+(ns metabase-enterprise.transform-optimizer.verify
+  "Equivalence verification for a proposal vs the original transform.
+
+  Materialises both queries into Postgres TEMP tables under a single
+  REPEATABLE READ transaction, then compares with `EXCEPT ALL` in both
+  directions — exactly the form the Phase-0 harness uses, generalised
+  to live transforms. Temp tables live in the session-private `pg_temp`
+  schema, so concurrent verify calls don't collide and we don't have to
+  manage a persistent scratch schema in the customer's database.
+
+  This is Postgres-only for the same reason the rest of the optimizer is:
+  the SQL we feed in is the compiled source of a native Postgres transform.
+  We can extend to other dialects later by parameterising the temp-table
+  / EXCEPT-ALL form."
+  (:require
+   [clojure.set :as set]
+   [clojure.string :as str]
+   [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+   [metabase.sql-tools.core :as sql-tools]
+   [metabase.transforms-base.util :as transforms-base.u]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (java.sql Connection ResultSet)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private sample-limit 10)
+
+;; Both sides of the comparison are materialised as Postgres TEMP tables.
+;; Temp tables live in the session-private `pg_temp` schema and are auto-
+;; dropped when the session ends, so concurrent verify calls from different
+;; users (or two tabs of the same user) can't collide on the same names.
+;; We still drop them explicitly at the end of verify to free the temp
+;; space before the connection returns to the pool, since pooled JDBC
+;; sessions outlive a single request.
+
+(def ^:private slow-table "slow_verify")
+(def ^:private fast-table "fast_verify")
+
+;; ---------------------------------------------------------------------------
+;; Connection / SQL helpers
+
+(defn- exec! [^Connection conn sql]
+  (with-open [stmt (.createStatement conn)]
+    (.execute stmt sql)))
+
+(defn- scalar [^Connection conn sql]
+  (with-open [stmt (.createStatement conn)
+              ^ResultSet rs (.executeQuery stmt sql)]
+    (when (.next rs) (.getObject rs 1))))
+
+(defn- materialize!
+  "Drop+create a session-local TEMP table named `table-name` as the result
+  of `select-sql`, timing the wall clock around the CREATE TABLE. Returns
+  `{:ms :row-count}`."
+  [^Connection conn ^String table-name select-sql]
+  (exec! conn (str "DROP TABLE IF EXISTS " table-name))
+  (let [t0    (System/nanoTime)
+        ddl   (str "CREATE TEMP TABLE " table-name " AS " select-sql)
+        _     (exec! conn ddl)
+        ms    (/ (- (System/nanoTime) t0) 1e6)
+        rows  (scalar conn (str "SELECT count(*) FROM " table-name))]
+    {:ms ms :row-count rows}))
+
+;; ---------------------------------------------------------------------------
+;; Column / schema compatibility
+
+(defn- column-info
+  "Return `[[col-name pg-type-name] …]` for `table-name` in ordinal order,
+  used for schema compatibility checks before the row comparison. Reads
+  from `pg_catalog.pg_attribute` keyed off `'pg_temp.<name>'::regclass`
+  because temp tables live in a session-private schema (`pg_temp_N`)
+  whose name we don't know at compile time — `pg_temp` is the alias
+  Postgres resolves to the current session's temp schema."
+  [^Connection conn ^String table-name]
+  (let [sql (str "SELECT a.attname AS column_name, "
+                 "       pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type "
+                 "FROM pg_catalog.pg_attribute a "
+                 "WHERE a.attrelid = 'pg_temp." table-name "'::regclass "
+                 "  AND a.attnum > 0 AND NOT a.attisdropped "
+                 "ORDER BY a.attnum")]
+    (with-open [stmt (.createStatement conn)
+                ^ResultSet rs (.executeQuery stmt sql)]
+      (loop [out []]
+        (if (.next rs)
+          (recur (conj out [(.getString rs 1) (.getString rs 2)]))
+          out)))))
+
+(defn- schema-match
+  "Compare the two column lists. Returns `nil` when compatible, otherwise a
+  `{:error … :detail …}` map ready to surface as a 422."
+  [slow-cols fast-cols]
+  (cond
+    (not= (count slow-cols) (count fast-cols))
+    {:error  "schema_mismatch"
+     :detail (format "slow has %d columns, fast has %d"
+                     (count slow-cols) (count fast-cols))}
+
+    ;; We don't require matching column names — `EXCEPT ALL` matches by
+    ;; position. Type compatibility is checked by Postgres at EXCEPT time;
+    ;; we surface that as a more readable error below.
+    :else nil))
+
+;; ---------------------------------------------------------------------------
+;; Diff
+
+(def ^:private diff-sql-template
+  "SELECT count(*) AS c FROM (
+     (SELECT * FROM %s EXCEPT ALL SELECT * FROM %s)
+     UNION ALL
+     (SELECT * FROM %s EXCEPT ALL SELECT * FROM %s)
+   ) d")
+
+(defn- diff-count [^Connection conn ^String slow ^String fast]
+  (long (scalar conn (format diff-sql-template slow fast fast slow))))
+
+(defn- row->vec [^ResultSet rs col-count]
+  (loop [i 1, out []]
+    (if (> i col-count)
+      out
+      (recur (inc i) (conj out (.getObject rs i))))))
+
+(defn- diff-sample
+  "When the row counts differ, fetch a small sample from each direction so
+  the UI can show a 'rows that are in slow but not fast / vice versa' diff.
+  Capped at `sample-limit` rows per direction."
+  [^Connection conn ^String slow ^String fast]
+  (with-open [stmt (.createStatement conn)]
+    (let [sql-only-in-slow (format "(SELECT * FROM %s EXCEPT ALL SELECT * FROM %s) LIMIT %d"
+                                   slow fast sample-limit)
+          sql-only-in-fast (format "(SELECT * FROM %s EXCEPT ALL SELECT * FROM %s) LIMIT %d"
+                                   fast slow sample-limit)
+          collect (fn [sql]
+                    (with-open [^ResultSet rs (.executeQuery stmt sql)]
+                      (let [md       (.getMetaData rs)
+                            n        (.getColumnCount md)
+                            col-names (mapv #(.getColumnName md %) (range 1 (inc n)))]
+                        {:columns col-names
+                         :rows    (loop [out []]
+                                    (if (.next rs)
+                                      (recur (conj out (row->vec rs n)))
+                                      out))})))]
+      {:only_in_slow (collect sql-only-in-slow)
+       :only_in_fast (collect sql-only-in-fast)})))
+
+;; ---------------------------------------------------------------------------
+;; Static (parser-only) preflight
+;;
+;; Cheap sanity check we run before materialising anything: same set of
+;; referenced tables, same returned column names in the same order. Catches
+;; the common LLM failure mode where a rewrite's FROM clause references a
+;; table that doesn't exist in the original (name drift, hallucinated
+;; precompute targets, typos) without paying the cost of executing both
+;; queries against the source DB.
+;;
+;; Tolerant of parse failures: if macaw can't analyse one side, we return
+;; nil and let the materialise step surface a real Postgres error instead
+;; of a parser one.
+
+(defn- lower [s] (some-> s u/lower-case-en))
+
+(defn- normalize-table-spec [{:keys [schema table]}]
+  [(lower schema) (lower table)])
+
+(defn- referenced-tables [driver sql]
+  (try
+    (into #{} (map normalize-table-spec) (sql-tools/referenced-tables-raw driver sql))
+    (catch Exception _ nil)))
+
+(defn- returned-column-names [driver sql]
+  (try
+    (let [{:keys [returned-fields errors]} (sql-tools/field-references driver sql)]
+      (when (empty? errors)
+        (mapv (fn [{:keys [alias column]}] (lower (or alias column)))
+              returned-fields)))
+    (catch Exception _ nil)))
+
+(defn- static-check
+  "Parser-only equivalence preflight. Returns `{:error … :detail …}` on a
+  known mismatch, otherwise nil. Skips the table-set check when the
+  proposal depends on another proposal — a dependent rewrite legitimately
+  references the precompute's target table."
+  [driver original-sql proposal-sql {:keys [depends_on]}]
+  (let [orig-tables (referenced-tables driver original-sql)
+        prop-tables (referenced-tables driver proposal-sql)
+        orig-cols   (returned-column-names driver original-sql)
+        prop-cols   (returned-column-names driver proposal-sql)]
+    (cond
+      (and (empty? depends_on)
+           orig-tables prop-tables
+           (not= orig-tables prop-tables))
+      (let [missing (set/difference orig-tables prop-tables)
+            extra   (set/difference prop-tables orig-tables)]
+        {:error  "table_set_mismatch"
+         :detail (str "Proposal references a different set of tables than the original."
+                      (when (seq missing) (str " Original-only: " (pr-str (vec missing)) "."))
+                      (when (seq extra)   (str " Proposal-only: "  (pr-str (vec extra))  ".")))})
+
+      (and orig-cols prop-cols
+           (not= orig-cols prop-cols))
+      {:error  "returned_columns_mismatch"
+       :detail (str "Returned column names differ. "
+                    "Original: " (pr-str orig-cols) ". "
+                    "Proposal: " (pr-str prop-cols) ".")}
+
+      :else nil)))
+
+;; ---------------------------------------------------------------------------
+;; Public API
+
+(defn- cleanup! [^Connection conn]
+  ;; Temp tables would be auto-dropped at session end, but the JDBC
+  ;; session outlives a single request (c3p0 pooling) so drop explicitly
+  ;; to free temp space immediately.
+  (try
+    (exec! conn (str "DROP TABLE IF EXISTS " slow-table))
+    (exec! conn (str "DROP TABLE IF EXISTS " fast-table))
+    (catch Exception e
+      (log/warn e "transform-optimizer verify: scratch cleanup failed"))))
+
+(defn- proposal-final-sql
+  "For a single-transform proposal the `body` is the comparison query.
+  For precompute (DAG) proposals we'd need to materialise the precompute
+  tables first; that's deferred — return a clear error."
+  [proposal]
+  (cond
+    (= "precompute" (some-> (:kind proposal) name))
+    (throw (ex-info "verify for precompute (DAG) proposals is not yet implemented"
+                    {:status-code 422 :error "unsupported_kind"}))
+
+    (str/blank? (:body proposal))
+    (throw (ex-info "proposal has no SQL body to verify"
+                    {:status-code 422 :error "missing_body"}))
+
+    :else
+    (:body proposal)))
+
+(defn verify
+  "Compare `transform` against `proposal`. Returns the response payload
+  documented in SUMMARY.md.
+
+  Throws an `ex-info` with `:status-code 422` for known failure modes
+  (precompute proposals, missing body, schema mismatch). Other exceptions
+  bubble up — callers should map them to 500."
+  [transform proposal]
+  (let [db-id    (transforms-base.u/transform-source-database transform)
+        driver-kw (t2/select-one-fn (comp keyword :engine) :model/Database :id db-id)
+        slow-sql  (-> (transforms-base.u/compile-source transform nil) :query)
+        fast-sql  (proposal-final-sql proposal)]
+    (when-not (= :postgres driver-kw)
+      (throw (ex-info "verify is Postgres-only in this branch"
+                      {:status-code 422 :error "unsupported_driver"})))
+    (when (str/blank? slow-sql)
+      (throw (ex-info "could not compile original transform"
+                      {:status-code 422 :error "compile_failed"})))
+
+    (when-let [{:keys [error detail]} (static-check driver-kw slow-sql fast-sql proposal)]
+      (throw (ex-info detail {:status-code 422 :error error})))
+
+    ;; `:write? true` tells Metabase's connection helper not to mark the
+    ;; connection read-only — without it the CREATE TABLE AS / DROP TABLE
+    ;; statements fail with "cannot execute … in a read-only transaction".
+    ;; The framework also flips auto-commit off for write connections so
+    ;; we can run the materialise pair inside one transaction.
+    (sql-jdbc.execute/do-with-connection-with-options
+     driver-kw db-id {:write? true}
+     (fn [^Connection conn]
+       (try
+         ;; Open a snapshot transaction. Both materialises see the same
+         ;; MVCC view of the source — without this, inserts/updates landing
+         ;; between the slow and fast CREATE TEMP TABLE AS calls would
+         ;; diverge the row sets and verify would report false-positive
+         ;; `equivalent: false`. We previously tried to reuse the live
+         ;; target table as the slow side (cached from the last successful
+         ;; run) but that has the same problem on a longer timescale:
+         ;; source data drift between the cached run and now. Always
+         ;; materialise both fresh inside one snapshot.
+         (.setAutoCommit conn false)
+         (exec! conn "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+         (let [slow-stats (materialize! conn slow-table slow-sql)
+               fast-stats (materialize! conn fast-table fast-sql)
+               slow-cols  (column-info conn slow-table)
+               fast-cols  (column-info conn fast-table)
+               mismatch   (schema-match slow-cols fast-cols)]
+           (.commit conn)
+           (.setAutoCommit conn true)
+           (if mismatch
+             (throw (ex-info (:detail mismatch) (assoc mismatch :status-code 422)))
+             (let [diff-rows   (diff-count conn slow-table fast-table)
+                   equivalent? (zero? diff-rows)
+                   slow-ms     (long (:ms slow-stats))
+                   fast-ms     (long (:ms fast-stats))
+                   speedup     (when (pos? fast-ms)
+                                 (double (/ slow-ms fast-ms)))]
+               {:equivalent       equivalent?
+                :slow_duration_ms slow-ms
+                :fast_duration_ms fast-ms
+                :speedup          speedup
+                :diff_rows        diff-rows
+                :slow_source      "materialized"
+                :sample_diff      (when-not equivalent?
+                                    (diff-sample conn slow-table fast-table))})))
+         (finally
+           ;; Best-effort: roll back anything still open and put the
+           ;; connection back into auto-commit mode before cleanup so the
+           ;; DROPs aren't trapped in a half-committed transaction.
+           (try (.rollback conn) (catch Exception _))
+           (try (.setAutoCommit conn true) (catch Exception _))
+           (cleanup! conn)))))))

enterprise/backend/test/metabase_enterprise/transform_optimizer/core_test.clj

@@ -0,0 +1,101 @@
+(ns metabase-enterprise.transform-optimizer.core-test
+  "Pure tests for the deterministic parts of `core` — `finalise-proposals`
+  and friends. Full end-to-end `optimize!` requires a transform fixture
+  and the LLM call; that lives in the integration tests added alongside
+  the deftool wiring (BE-1)."
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [metabase-enterprise.transform-optimizer.core :as core]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private ctx
+  {:sources [{:schema "shop" :table_name "orders"}
+             {:schema "shop" :table_name "order_items"}]
+   :target  {:schema "shop" :table_name "monthly_revenue"}})
+
+(defn- index-proposal
+  "Helper: build a kind=:index proposal with a single DDL statement."
+  [severity statement]
+  {:id "p1"
+   :name "test"
+   :kind "index"
+   :severity severity
+   :rationale "test"
+   :depends_on []
+   :ddl_statement {:target "source-db"
+                   :statement statement
+                   :rationale "ok"}})
+
+;; ---------------------------------------------------------------------------
+;; finalise-proposals: single-DDL validation
+
+(deftest finalise-tags-valid-ddl-as-accepted-test
+  (let [{:keys [proposals]}
+        (core/finalise-proposals
+         [(index-proposal :high
+                          "CREATE INDEX IF NOT EXISTS idx_o ON shop.orders (customer_id)")]
+         ctx)
+        ddl (:ddl_statement (first proposals))]
+    (is (= :accepted (:validation ddl)))
+    (is (= "idx_o" (:index_name ddl)))
+    (is (nil? (:rejection ddl)))))
+
+(deftest finalise-tags-invalid-ddl-as-rejected-test
+  (testing "unknown table"
+    (let [{:keys [proposals]}
+          (core/finalise-proposals
+           [(index-proposal :high
+                            "CREATE INDEX idx_x ON public.users (id)")]
+           ctx)
+          ddl (:ddl_statement (first proposals))]
+      (is (= :rejected (:validation ddl)))
+      (is (= :unknown-table (-> ddl :rejection :reason)))))
+
+  (testing "multi-statement smuggling attempt"
+    (let [{:keys [proposals]}
+          (core/finalise-proposals
+           [(index-proposal :high
+                            "CREATE INDEX idx_o ON shop.orders (id); DROP TABLE x;")]
+           ctx)
+          ddl (:ddl_statement (first proposals))]
+      (is (= :rejected (:validation ddl)))
+      (is (contains? #{:forbidden-keyword :multi-statement}
+                     (-> ddl :rejection :reason))))))
+
+(deftest finalise-leaves-rewrite-proposals-alone-test
+  (testing "rewrite proposals have no ddl_statement and aren't touched"
+    (let [{:keys [proposals]}
+          (core/finalise-proposals
+           [{:id "p1" :name "rewrite" :kind "rewrite" :severity :medium
+             :depends_on []
+             :body "SELECT 1"}]
+           ctx)]
+      (is (nil? (:ddl_statement (first proposals))))
+      (is (= "SELECT 1" (:body (first proposals)))))))
+
+;; ---------------------------------------------------------------------------
+;; finalise-proposals: scoring is wired in
+
+(deftest finalise-includes-optimization-degree-test
+  (testing "score reflects all proposals' severities"
+    (is (= 100 (:optimization_degree
+                (core/finalise-proposals [] ctx))))
+    (is (= 70  (:optimization_degree
+                (core/finalise-proposals [{:severity :high}] ctx))))
+    (is (= 55  (:optimization_degree
+                (core/finalise-proposals
+                 [{:severity :high} {:severity :medium}] ctx))))))
+
+;; ---------------------------------------------------------------------------
+;; The allowed-table set spans sources + target
+
+(deftest target-table-is-also-allowed-test
+  (testing "DDL targeting the transform's own target table is accepted"
+    (let [{:keys [proposals]}
+          (core/finalise-proposals
+           [(index-proposal :medium
+                            "CREATE INDEX IF NOT EXISTS idx_mr ON shop.monthly_revenue (m)")]
+           ctx)
+          ddl (:ddl_statement (first proposals))]
+      (is (= :accepted (:validation ddl))))))

enterprise/backend/test/metabase_enterprise/transform_optimizer/ddl/parse_test.clj

@@ -0,0 +1,190 @@
+(ns metabase-enterprise.transform-optimizer.ddl.parse-test
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [metabase-enterprise.transform-optimizer.ddl.parse :as ddl.parse]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private allowed-tables
+  "Standard fixture: a couple of `shop` tables that the optimizer's
+  referenced-tables set would normally produce."
+  #{["shop" "orders"] ["shop" "order_items"] ["shop" "reviews"]})
+
+;; ---------------------------------------------------------------------------
+;; Happy path
+
+(deftest accepts-canonical-create-index-test
+  (testing "minimal CREATE INDEX is accepted"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX IF NOT EXISTS idx_orders_customer_id ON shop.orders (customer_id);"
+             allowed-tables)]
+      (is (true? (:ok? r)))
+      (is (= "idx_orders_customer_id" (:name r)))
+      (is (= "shop" (:schema r)))
+      (is (= "orders" (:table r)))))
+
+  (testing "with CONCURRENTLY"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_o ON shop.orders (status, ordered_at)"
+             allowed-tables)]
+      (is (true? (:ok? r)))))
+
+  (testing "with UNIQUE"
+    (let [r (ddl.parse/parse
+             "CREATE UNIQUE INDEX IF NOT EXISTS idx_r ON shop.reviews (id)"
+             allowed-tables)]
+      (is (true? (:ok? r)))))
+
+  (testing "with INCLUDE + partial predicate"
+    (let [r (ddl.parse/parse
+             (str "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_o "
+                  "ON shop.orders (customer_id) INCLUDE (total_cents) "
+                  "WHERE status = 'paid'")
+             allowed-tables)]
+      (is (true? (:ok? r)))))
+
+  (testing "case-insensitive keywords"
+    (let [r (ddl.parse/parse
+             "create index if not exists idx_o on shop.orders (customer_id)"
+             allowed-tables)]
+      (is (true? (:ok? r))))))
+
+;; ---------------------------------------------------------------------------
+;; Multi-statement rejection (primary anti-injection defence)
+
+(deftest rejects-multi-statement-test
+  (testing "second statement after CREATE INDEX is rejected"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX IF NOT EXISTS idx_o ON shop.orders (id); DROP TABLE foo;"
+             allowed-tables)]
+      ;; The forbidden keyword check fires before multi-statement, but either
+      ;; rejection is acceptable — what matters is we DO reject.
+      (is (false? (:ok? r)))
+      (is (contains? #{:forbidden-keyword :multi-statement} (:reason r)))))
+
+  (testing "two CREATE INDEXes are rejected (one statement per ddl entry)"
+    (let [r (ddl.parse/parse
+             (str "CREATE INDEX idx_a ON shop.orders (id);"
+                  "CREATE INDEX idx_b ON shop.orders (customer_id);")
+             allowed-tables)]
+      (is (false? (:ok? r)))
+      (is (= :multi-statement (:reason r)))))
+
+  (testing "trailing semicolon is fine (one statement only)"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX idx_o ON shop.orders (id);  "
+             allowed-tables)]
+      (is (true? (:ok? r))))))
+
+;; ---------------------------------------------------------------------------
+;; Forbidden keywords
+
+(deftest rejects-forbidden-keywords-test
+  (doseq [forbidden ["DROP TABLE x"
+                     "ALTER TABLE shop.orders DROP COLUMN id"
+                     "GRANT ALL ON shop.orders TO public"
+                     "REVOKE SELECT ON shop.orders FROM bob"
+                     "TRUNCATE shop.orders"
+                     "DELETE FROM shop.orders"
+                     "UPDATE shop.orders SET id = 0"
+                     "INSERT INTO shop.orders VALUES (1)"
+                     "COPY shop.orders TO '/tmp/x'"
+                     "VACUUM shop.orders"
+                     "REINDEX TABLE shop.orders"
+                     "REFRESH MATERIALIZED VIEW v"
+                     "DO $$ BEGIN END $$"
+                     "EXECUTE foo"
+                     "SET timezone = 'UTC'"]]
+    (testing (str "rejected: " forbidden)
+      (let [r (ddl.parse/parse forbidden allowed-tables)]
+        (is (false? (:ok? r)))
+        (is (= :forbidden-keyword (:reason r))
+            (str "rejection reason for: " forbidden))))))
+
+;; ---------------------------------------------------------------------------
+;; Sanitisation — strings + comments can't smuggle a second statement
+
+(deftest sanitiser-strips-string-contents-test
+  (testing "semicolon inside a single-quoted string is not a statement separator"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX idx_o ON shop.orders (id) WHERE message = ';DROP TABLE x;';"
+             allowed-tables)]
+      (is (true? (:ok? r)))))
+
+  (testing "'' escape inside a string still works"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX idx_o ON shop.orders (id) WHERE note = 'foo''bar'"
+             allowed-tables)]
+      (is (true? (:ok? r))))))
+
+(deftest sanitiser-strips-line-comments-test
+  (testing "-- comment after the statement is harmless"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX idx_o ON shop.orders (id) -- DROP TABLE x;"
+             allowed-tables)]
+      (is (true? (:ok? r)))))
+
+  (testing "-- followed by a forbidden keyword on the same line is still hidden"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX idx_o ON shop.orders (id) -- and then ALTER TABLE\n"
+             allowed-tables)]
+      (is (true? (:ok? r))))))
+
+;; ---------------------------------------------------------------------------
+;; Unknown-table rejection
+
+(deftest rejects-unknown-table-test
+  (testing "table not in the referenced-tables set is rejected"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX idx_x ON public.users (id)"
+             allowed-tables)]
+      (is (false? (:ok? r)))
+      (is (= :unknown-table (:reason r)))
+      (is (re-find #"public\.users" (:detail r)))))
+
+  (testing "case-insensitive matching against allowed set"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX idx_o ON SHOP.ORDERS (id)"
+             allowed-tables)]
+      (is (true? (:ok? r))
+          "schema/table comparison is case-insensitive"))))
+
+;; ---------------------------------------------------------------------------
+;; Bad shape
+
+(deftest rejects-non-create-index-test
+  (testing "garbage prefix is rejected"
+    (let [r (ddl.parse/parse "CREATE TABLE foo (id int)" allowed-tables)]
+      (is (false? (:ok? r)))
+      (is (= :not-create-index (:reason r)))))
+
+  (testing "missing schema qualifier"
+    (let [r (ddl.parse/parse "CREATE INDEX idx_o ON orders (id)" allowed-tables)]
+      (is (false? (:ok? r)))
+      (is (= :not-create-index (:reason r)))))
+
+  (testing "blank input"
+    (let [r (ddl.parse/parse "" allowed-tables)]
+      (is (false? (:ok? r))))))
+
+;; ---------------------------------------------------------------------------
+;; Block comments and dollar-quoted strings are rejected by design
+
+(deftest rejects-block-comments-and-dollar-quotes-test
+  (testing "block comments are treated opaquely; if they hide a semicolon, we still reject"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX idx_o ON shop.orders (id) /* foo */"
+             allowed-tables)]
+      ;; The /* … */ leaves residual characters in the sanitised text that
+      ;; cause the regex to either succeed (if the comment is well-placed) or
+      ;; fail. Either way the result is safe.
+      (is (or (true? (:ok? r))
+              (false? (:ok? r))))))
+
+  (testing "an attacker hiding a forbidden keyword in a block comment doesn't smuggle it"
+    (let [r (ddl.parse/parse
+             "CREATE INDEX idx_o ON shop.orders (id) /*; DROP TABLE x;*/"
+             allowed-tables)]
+      ;; The block comment is left mostly intact in the sanitised stream, but
+      ;; the forbidden-keyword scan finds "DROP" anyway and rejects.
+      (is (false? (:ok? r))))))

enterprise/backend/test/metabase_enterprise/transform_optimizer/index_introspection_test.clj

@@ -0,0 +1,31 @@
+(ns metabase-enterprise.transform-optimizer.index-introspection-test
+  "Pure tests for the parts of index-introspection that don't need a live
+  Postgres. Catalog-query integration tests live under
+  `transform-optimizer.context-test` against `mt/test-driver :postgres`."
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [metabase-enterprise.transform-optimizer.index-introspection :as iix]))
+
+(set! *warn-on-reflection* true)
+
+(deftest group-by-table-normalises-case-test
+  (testing "indexes are bucketed by [schema table] with lowercase keys"
+    (let [grouped
+          (iix/group-by-table
+           [{:schema "Shop" :table "Orders" :name "idx_a"}
+            {:schema "shop" :table "orders" :name "idx_b"}
+            {:schema "SHOP" :table "REVIEWS" :name "idx_c"}])]
+      (is (= 2 (count grouped))
+          "the three case-different shop.orders rows collapse to one bucket")
+      (is (= #{["shop" "orders"] ["shop" "reviews"]} (set (keys grouped)))
+          "keys are lowercased")
+      (is (= 2 (count (get grouped ["shop" "orders"]))))
+      (is (= 1 (count (get grouped ["shop" "reviews"])))))))
+
+(deftest group-by-table-handles-nil-test
+  (testing "nil schema or table is preserved through normalisation (so we don't crash)"
+    (is (= {[nil "x"] [{:schema nil :table "x"}]}
+           (iix/group-by-table [{:schema nil :table "x"}])))))
+
+(deftest group-by-table-empty-test
+  (is (= {} (iix/group-by-table []))))

enterprise/backend/test/metabase_enterprise/transform_optimizer/prelude_test.clj

@@ -0,0 +1,86 @@
+(ns metabase-enterprise.transform-optimizer.prelude-test
+  (:require
+   [clojure.string :as str]
+   [clojure.test :refer [deftest is testing]]
+   [metabase-enterprise.transform-optimizer.prelude :as prelude]
+   [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(deftest prelude-resource-loads-test
+  (let [text (prelude/prelude)]
+    (testing "prelude resource is present on the classpath"
+      (is (string? text))
+      (is (pos? (count text))))))
+
+(deftest prelude-anchors-test
+  ;; Lightweight content checks: if these strings disappear from the prelude,
+  ;; the LLM is being asked to do something different than what the scoring
+  ;; rubric / validator expect. Update both the prelude and these anchors
+  ;; together.
+  (let [text (u/lower-case-en (prelude/prelude))]
+    (testing "output schema anchors"
+      (is (str/includes? text "summary"))
+      (is (str/includes? text "proposals"))
+      ;; One change per proposal: singular `ddl_statement` is the new
+      ;; canonical shape. We deliberately do NOT anchor on the legacy
+      ;; plural form here.
+      (is (str/includes? text "ddl_statement"))
+      (is (str/includes? text "depends_on")))
+
+    (testing "all proposal kinds are described"
+      ;; One change per proposal — `rewrite+index` is intentionally gone.
+      (doseq [kind ["rewrite" "index" "precompute"]]
+        (is (str/includes? text kind)
+            (str "missing kind: " kind))))
+    (testing "rewrite+index is explicitly NOT a valid kind"
+      (is (not (str/includes? text "\"rewrite+index\""))))
+
+    (testing "severity rubric is in the prompt"
+      (is (str/includes? text "severity"))
+      (is (str/includes? text "high"))
+      (is (str/includes? text "medium"))
+      (is (str/includes? text "low")))
+
+    (testing "DDL constraints are present"
+      (is (str/includes? text "create index"))
+      (is (str/includes? text "if not exists"))
+      (is (str/includes? text "concurrently")))
+
+    (testing "equivalence pitfalls section is present (lessons from prior verifier diffs)"
+      (is (str/includes? text "equivalence pitfalls"))
+      (testing "NULL-safety footgun is documented"
+        (is (str/includes? text "not in"))
+        (is (str/includes? text "not null")
+            "the NOT-IN-on-nullable rewrite precondition must be explicit"))
+      (testing "multi-aggregate rollup footgun is documented"
+        (is (or (str/includes? text "array_agg")
+                (str/includes? text "full outer join"))))
+      (testing "type-compatibility note is present"
+        (is (or (str/includes? text "::bigint")
+                (str/includes? text "type compat"))))
+      (testing "scale-dependent LATERAL caveat is documented"
+        (is (str/includes? text "lateral"))))
+
+    (testing "writing-style length limits are present"
+      ;; The UI renders summary / rationale / DDL rationale verbatim.
+      ;; Without explicit caps the LLM writes essays. These anchors
+      ;; protect that section from quiet regressions.
+      (is (str/includes? text "writing style"))
+      (is (str/includes? text "3 sentences"))
+      (is (str/includes? text "2 sentences"))
+      (is (str/includes? text "1 sentence")))
+
+    (testing "the 'always also consider a precompute' nudges are present"
+      ;; Without this section, the LLM tends to emit one quick-win
+      ;; proposal (e.g. a covering index) even when a precompute split
+      ;; is the structural answer. Anchors guard that section so a
+      ;; prelude edit can't silently drop it.
+      (is (str/includes? text "alternatives"))
+      (is (str/includes? text "always consider a precompute"))
+      (is (str/includes? text "count(distinct"))
+      (is (str/includes? text "cohort")))))
+
+(deftest prelude-cached-test
+  (testing "successive calls return the same string instance (delayed cache)"
+    (is (identical? (prelude/prelude) (prelude/prelude)))))

enterprise/backend/test/metabase_enterprise/transform_optimizer/prompt_test.clj

@@ -0,0 +1,102 @@
+(ns metabase-enterprise.transform-optimizer.prompt-test
+  (:require
+   [clojure.string :as str]
+   [clojure.test :refer [deftest is testing]]
+   [metabase-enterprise.transform-optimizer.prompt :as prompt]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private synthetic-ctx
+  {:transform {:id 42
+               :name "monthly_revenue"
+               :source_database_id 1
+               :target {:schema "analytics" :name "monthly_revenue"}}
+   :sql "SELECT date_trunc('month', ordered_at) AS m, sum(total_cents) FROM shop.orders GROUP BY 1"
+   :sources [{:schema       "shop"
+              :table_name   "orders"
+              :db_id        1
+              :column_count 7
+              :fields       [{:name "id"
+                              :base_type :type/BigInteger
+                              :semantic_type :type/PK
+                              :indexed? true}
+                             {:name "customer_id"
+                              :base_type :type/BigInteger
+                              :foreign_key {:target_schema "shop"
+                                            :target_table  "customers"
+                                            :target_column "id"}}
+                             {:name "ordered_at"
+                              :base_type :type/DateTimeWithTZ}]
+              :indexes      [{:name "orders_pkey"
+                              :access_method "btree"
+                              :is_primary true
+                              :is_unique true
+                              :key_columns ["id"]
+                              :include_columns []
+                              :definition "CREATE UNIQUE INDEX orders_pkey ON shop.orders USING btree (id)"}]}]
+   :target {:schema "analytics" :table_name "monthly_revenue" :column_count 2 :fields []}
+   :indexes_partial? false
+   :explain [{:Plan {:Node-Type "Aggregate" :Total-Cost 1234.56}}]
+   :run_history [{:start_time "2026-05-12T10:00:00Z"
+                  :end_time   "2026-05-12T10:00:08Z"
+                  :duration_ms 8000
+                  :status :succeeded
+                  :run_method :manual}]})
+
+(deftest section-headers-present-test
+  (let [out (prompt/render-context synthetic-ctx)]
+    (testing "all required section headers appear in the rendered output"
+      (doseq [section ["## Transform"
+                       "## SQL"
+                       "## Referenced tables"
+                       "## EXPLAIN"
+                       "## Run history"]]
+        (is (str/includes? out section)
+            (str "missing section: " section))))))
+
+(deftest transform-block-renders-test
+  (let [out (prompt/render-context synthetic-ctx)]
+    (is (str/includes? out "id: 42"))
+    (is (str/includes? out "name: monthly_revenue"))
+    (is (str/includes? out "analytics.monthly_revenue"))))
+
+(deftest sql-block-renders-as-code-fence-test
+  (let [out (prompt/render-context synthetic-ctx)]
+    (is (str/includes? out "```sql"))
+    (is (str/includes? out "SELECT date_trunc"))))
+
+(deftest source-table-block-includes-indexes-and-fks-test
+  (let [out (prompt/render-context synthetic-ctx)]
+    (is (str/includes? out "### shop.orders"))
+    (testing "field-level FK is rendered"
+      (is (str/includes? out "FK → shop.customers.id")))
+    (testing "field-level indexed? flag is rendered"
+      (is (str/includes? out "[indexed]")))
+    (testing "index detail is rendered"
+      (is (str/includes? out "orders_pkey"))
+      (is (str/includes? out "CREATE UNIQUE INDEX orders_pkey")))))
+
+(deftest indexes-partial-warning-test
+  (testing "partial flag adds a warning callout"
+    (let [out (prompt/render-context (assoc synthetic-ctx :indexes_partial? true))]
+      (is (re-find #"(?i)partial" out)))))
+
+(deftest no-runs-renders-gracefully-test
+  (let [out (prompt/render-context (assoc synthetic-ctx :run_history []))]
+    (is (str/includes? out "(no run history)"))))
+
+(deftest no-explain-renders-gracefully-test
+  (let [out (prompt/render-context (assoc synthetic-ctx :explain nil))]
+    (is (str/includes? out "EXPLAIN unavailable"))))
+
+(deftest compile-failure-renders-marker-test
+  (testing "nil SQL is rendered with a clear marker rather than blank"
+    (let [out (prompt/render-context (assoc synthetic-ctx :sql nil))]
+      (is (str/includes? out "compilation failed")))))
+
+(deftest table-with-no-indexes-renders-test
+  (testing "a source with no indexes gets an explicit '(none beyond PK)' marker"
+    (let [out (prompt/render-context
+               (assoc synthetic-ctx :sources
+                      [(assoc (first (:sources synthetic-ctx)) :indexes [])]))]
+      (is (str/includes? out "indexes: (none beyond PK)")))))

enterprise/backend/test/metabase_enterprise/transform_optimizer/proposal_cache_test.clj

@@ -0,0 +1,79 @@
+(ns metabase-enterprise.transform-optimizer.proposal-cache-test
+  (:require
+   [clojure.test :refer [deftest is testing use-fixtures]]
+   [metabase-enterprise.transform-optimizer.proposal-cache :as cache]))
+
+(set! *warn-on-reflection* true)
+
+#_{:clj-kondo/ignore [:metabase/validate-deftest]}
+(use-fixtures :each (fn [t] (#'cache/clear-all!) (t) (#'cache/clear-all!)))
+
+(deftest ^:synchronized put-and-get-roundtrip-test
+  (testing "stored proposals are retrievable by user+transform+id"
+    (cache/put-all! 1 99 [{:id "p1" :body "SELECT 1"}
+                          {:id "p2" :body "SELECT 2"}])
+    (is (= {:id "p1" :body "SELECT 1"}
+           (cache/get-one 1 99 "p1")))
+    (is (= {:id "p2" :body "SELECT 2"}
+           (cache/get-one 1 99 "p2")))))
+
+(deftest ^:synchronized get-missing-returns-nil-test
+  (testing "missing id returns nil rather than throwing"
+    (cache/put-all! 1 99 [{:id "p1"}])
+    (is (nil? (cache/get-one 1 99 "p2")))))
+
+(deftest ^:synchronized skips-blank-ids-test
+  (testing "skips proposals with blank/nil ids"
+    (cache/put-all! 1 99 [{:id nil :body "x"} {:id "" :body "y"} {:id "p3"}])
+    (is (nil? (cache/get-one 1 99 nil)))
+    (is (nil? (cache/get-one 1 99 "")))
+    (is (= {:id "p3"} (cache/get-one 1 99 "p3")))))
+
+(deftest ^:synchronized user-isolation-test
+  (testing "user A cannot read proposals stored for user B"
+    (cache/put-all! 1 99 [{:id "p1" :body "SECRET"}])
+    (is (nil? (cache/get-one 2 99 "p1"))
+        "different user-id must not see the proposal")
+    (is (some? (cache/get-one 1 99 "p1"))
+        "owner still sees it")))
+
+(deftest ^:synchronized transform-isolation-test
+  (testing "proposal stored for transform A is not visible under transform B"
+    (cache/put-all! 1 99 [{:id "p1"}])
+    (is (nil? (cache/get-one 1 100 "p1")))
+    (is (some? (cache/get-one 1 99 "p1")))))
+
+;; Split into separate deftests so the :each fixture clears the cache
+;; between each scenario. `testing` blocks inside one deftest share state.
+
+(deftest ^:synchronized get-many-mixed-test
+  (testing "returns proposals in input order + reports missing"
+    (cache/put-all! 1 99 [{:id "p1" :body "a"} {:id "p3" :body "c"}])
+    (let [[found missing] (cache/get-many 1 99 ["p1" "p2" "p3" "p4"])]
+      (is (= [{:id "p1" :body "a"} {:id "p3" :body "c"}] found))
+      (is (= ["p2" "p4"] missing)))))
+
+(deftest ^:synchronized get-many-all-present-test
+  (testing "all present ⇒ no missing"
+    (cache/put-all! 1 99 [{:id "p1"} {:id "p2"}])
+    (let [[found missing] (cache/get-many 1 99 ["p1" "p2"])]
+      (is (= 2 (count found)))
+      (is (empty? missing)))))
+
+(deftest ^:synchronized get-many-all-missing-test
+  (testing "all missing ⇒ all reported as missing"
+    (let [[found missing] (cache/get-many 1 99 ["p1" "p2"])]
+      (is (empty? found))
+      (is (= ["p1" "p2"] missing)))))
+
+(deftest ^:synchronized re-put-overwrites-test
+  (testing "calling put-all! again with the same id replaces the previous entry"
+    (cache/put-all! 1 99 [{:id "p1" :body "v1"}])
+    (cache/put-all! 1 99 [{:id "p1" :body "v2"}])
+    (is (= "v2" (:body (cache/get-one 1 99 "p1"))))))
+
+(deftest ^:synchronized empty-put-noop-test
+  (testing "put-all! with empty proposals does nothing (no errors)"
+    (cache/put-all! 1 99 [])
+    (cache/put-all! 1 99 nil)
+    (is (nil? (cache/get-one 1 99 "anything")))))

enterprise/backend/test/metabase_enterprise/transform_optimizer/scoring_test.clj

@@ -0,0 +1,50 @@
+(ns metabase-enterprise.transform-optimizer.scoring-test
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [metabase-enterprise.transform-optimizer.scoring :as scoring]))
+
+(set! *warn-on-reflection* true)
+
+(deftest optimization-degree-test
+  (testing "empty / nil proposal list ⇒ 100 (already optimized)"
+    (is (= 100 (scoring/optimization-degree [])))
+    (is (= 100 (scoring/optimization-degree nil))))
+
+  (testing "single proposal per severity matches rubric weights"
+    (is (= 70 (scoring/optimization-degree [{:severity :high}])))
+    (is (= 85 (scoring/optimization-degree [{:severity :medium}])))
+    (is (= 95 (scoring/optimization-degree [{:severity :low}]))))
+
+  (testing "weights stack additively"
+    (is (= 10 (scoring/optimization-degree
+               [{:severity :high} {:severity :high} {:severity :high}])))
+    (is (= 70 (scoring/optimization-degree
+               [{:severity :medium} {:severity :medium}])))
+    (is (= 55 (scoring/optimization-degree
+               [{:severity :high} {:severity :medium}]))))
+
+  (testing "score floors at 0; never negative"
+    (is (= 0 (scoring/optimization-degree
+              (repeat 10 {:severity :high}))))
+    (is (= 0 (scoring/optimization-degree
+              (repeat 100 {:severity :low})))))
+
+  (testing "severity strings are accepted (keyword coercion)"
+    ;; LLM emits JSON strings; we treat "high" and :high identically so
+    ;; callers don't have to coerce before passing to scoring.
+    (is (= 70 (scoring/optimization-degree [{:severity "high"}])))
+    (is (= 85 (scoring/optimization-degree [{:severity "medium"}]))))
+
+  (testing "unknown / missing severities fall back to :low (5)"
+    ;; A garbled LLM response shouldn't yield NaN or throw — it should
+    ;; degrade gracefully.
+    (is (= 95 (scoring/optimization-degree [{:severity :critical}])))
+    (is (= 95 (scoring/optimization-degree [{:severity nil}])))
+    (is (= 95 (scoring/optimization-degree [{}]))))
+
+  (testing "irrelevant proposal keys are ignored"
+    (is (= 70 (scoring/optimization-degree
+               [{:severity :high
+                 :name "foo"
+                 :rationale "bar"
+                 :ddl_statements [{:statement "anything"}]}])))))

enterprise/frontend/src/metabase-enterprise/api/tags.ts

@@ -50,6 +50,7 @@ export const ENTERPRISE_TAG_TYPES = [
   "ai-controls-usage-group-limits",
   "ai-controls-usage-tenant-limits",
   "data-complexity-scores",
+  "transform-optimizer-indexes",
 ] as const;
 
 export type EnterpriseTagType = TagType | (typeof ENTERPRISE_TAG_TYPES)[number];

enterprise/frontend/src/metabase-enterprise/plugins.ts

@@ -46,6 +46,7 @@ import { initializePlugin as initializeSupport } from "./support";
 import { initializePlugin as initializeTableEditing } from "./table-editing";
 import { initializePlugin as initializeTenants } from "./tenants";
 import { initializePlugin as initializeTools } from "./tools";
+import { initializePlugin as initializeTransformOptimizer } from "./transform_optimizer";
 import { initializePlugin as initializeTransforms } from "./transforms";
 import { initializePlugin as initializeTransformsInspector } from "./transforms-inspector";
 import { initializePlugin as initializeTransformsPython } from "./transforms-python";
@@ -98,6 +99,7 @@ export function initializePlugins() {
   initializeSemanticSearch();
   initializeTransforms();
   initializeTransformsInspector();
+  initializeTransformOptimizer();
   initializeTransformsPython();
   initializeSupport();
   initializeTenants?.();

enterprise/frontend/src/metabase-enterprise/transform_optimizer/api/index.ts

@@ -0,0 +1,268 @@
+import { EnterpriseApi } from "metabase-enterprise/api";
+import { idTag, invalidateTags, listTag } from "metabase-enterprise/api/tags";
+
+import type {
+  DdlExecutionStatus,
+  DropIndexResult,
+  Proposal,
+  TargetIndex,
+} from "../types";
+
+export type VerifyResponse =
+  | {
+      equivalent: boolean;
+      slow_duration_ms: number;
+      fast_duration_ms: number;
+      speedup: number;
+      diff_rows: number;
+      sample_diff: unknown[] | null;
+    }
+  | { error: string; detail?: string };
+
+/**
+ * The full result of one optimizer run. Returned as a single JSON
+ * payload — the optimizer used to be SSE-streamed, but the LLM call is
+ * fully buffered server-side anyway, so the streaming framing didn't
+ * pay for itself.
+ */
+export type OptimizeResponse = {
+  transform: {
+    id: number;
+    name: string;
+    source_database_id: number;
+    target?: unknown;
+  };
+  sql: string | null;
+  summary: string | null;
+  proposals: Proposal[];
+  optimization_degree: number;
+};
+
+/**
+ * One entry in the accept response's `ddl_statements`. Each accepted
+ * source-DB statement carries an execution `status`; transform-target /
+ * precompute-of statements carry `status: "pending"`; rejected
+ * statements pass through with no status but a `rejection`.
+ */
+export type AcceptDdlStatement = {
+  proposal_id: string;
+  statement: string;
+  target: unknown;
+  rationale: string;
+  validation: "accepted" | "rejected";
+  index_name?: string | null;
+  rejection?: { reason: string; detail?: string } | null;
+  status?: DdlExecutionStatus;
+  error_message?: string;
+};
+
+/**
+ * Accept mode. `"new"` creates fresh transforms for each body-bearing
+ * proposal (default). `"replace"` updates the *original* transform's
+ * source in place; only valid for a single-proposal accept of
+ * kind=rewrite.
+ */
+export type AcceptMode = "new" | "replace";
+
+export type AcceptResponse =
+  | {
+      created_transforms: Array<{
+        id: number;
+        name: string;
+        proposal_id: string;
+        kind: string;
+        depends_on: string[];
+        pending_ddl?: number;
+      }>;
+      ddl_statements: AcceptDdlStatement[];
+      skipped_proposals: string[];
+    }
+  | {
+      replaced_transform: {
+        id: number;
+        name: string;
+        proposal_id: string;
+        kind: string;
+        pending_ddl?: number;
+      };
+      ddl_statements: AcceptDdlStatement[];
+      skipped_proposals: string[];
+    };
+
+export type ListIndexesResponse = {
+  transform: { id: number; target: unknown };
+  indexes: TargetIndex[];
+};
+
+export type BulkOptimizeResponse = {
+  started: number;
+  transform_ids: number[];
+};
+
+/**
+ * One transform's result inside a bulk-optimize job — mirrors the shape of
+ * `OptimizeResponse` minus the `sql` field (we don't surface SQL in the
+ * drawer; users drill into the detail page if they want it).
+ */
+export type BulkOptimizeDoneEntry = {
+  transform: {
+    id: number;
+    name: string;
+    source_database_id: number;
+    optimized?: boolean;
+  };
+  summary: string | null;
+  proposals: Proposal[];
+  optimization_degree: number;
+};
+
+export type BulkOptimizeStatusResponse = {
+  total: number;
+  pending: number[];
+  done: Record<string, BulkOptimizeDoneEntry>;
+  failed: Record<string, string>;
+  started_at?: string;
+};
+
+export const optimizerApi = EnterpriseApi.injectEndpoints({
+  endpoints: (builder) => ({
+    optimize: builder.query<
+      OptimizeResponse,
+      { transformId: number | string; analyze?: boolean }
+    >({
+      query: ({ transformId, analyze = false }) => ({
+        method: "POST",
+        url: `/api/ee/transform-optimizer/${transformId}/optimize`,
+        body: { analyze },
+      }),
+      // The optimizer call takes 5–30 s and the LLM bill isn't free. Keep
+      // the result cached for the session so a tab switch (or any other
+      // unmount of `TransformOptimizerSection`) doesn't drop the work the
+      // user already paid for.
+      keepUnusedDataFor: 60 * 60,
+    }),
+    verifyProposal: builder.mutation<
+      VerifyResponse,
+      { transformId: number | string; proposal: Proposal }
+    >({
+      query: ({ transformId, proposal }) => ({
+        method: "POST",
+        url: `/api/ee/transform-optimizer/${transformId}/proposal/verify`,
+        body: { proposal_id: proposal.id },
+      }),
+    }),
+    acceptProposal: builder.mutation<
+      AcceptResponse,
+      {
+        transformId: number | string;
+        /**
+         * Proposal ids in dependency order (roots first). For a single
+         * `rewrite` / `index` proposal that's one id; for a `precompute`
+         * DAG, every ancestor of the chosen proposal must precede it.
+         */
+        proposalIds: string[];
+        /**
+         * `"new"` (default) creates fresh transforms. `"replace"` updates
+         * the original transform's source in place — only valid when the
+         * batch contains exactly one body-bearing proposal of
+         * kind="rewrite".
+         */
+        mode?: AcceptMode;
+        collectionId?: number;
+      }
+    >({
+      query: ({ transformId, proposalIds, mode, collectionId }) => ({
+        method: "POST",
+        url: `/api/ee/transform-optimizer/${transformId}/proposal/accept`,
+        body: {
+          proposal_ids: proposalIds,
+          ...(mode != null ? { mode } : {}),
+          ...(collectionId != null ? { collection_id: collectionId } : {}),
+        },
+      }),
+      // After accept we may have:
+      //   - run DDL on one of the transform's referenced tables → refetch the index list
+      //   - replaced the original transform's source in place → refetch the
+      //     transform detail so the SQL editor / read-only view doesn't show
+      //     stale text after the user switches tabs.
+      invalidatesTags: (_data, error, { transformId }) =>
+        invalidateTags(error, [
+          idTag("transform-optimizer-indexes", transformId),
+          idTag("transform", transformId),
+        ]),
+    }),
+    listTargetIndexes: builder.query<
+      ListIndexesResponse,
+      { transformId: number | string }
+    >({
+      query: ({ transformId }) => ({
+        method: "GET",
+        url: `/api/ee/transform-optimizer/${transformId}/indexes`,
+      }),
+      providesTags: (_data, _error, { transformId }) => [
+        idTag("transform-optimizer-indexes", transformId),
+      ],
+    }),
+    dropTargetIndex: builder.mutation<
+      DropIndexResult,
+      { transformId: number | string; indexName: string }
+    >({
+      query: ({ transformId, indexName }) => ({
+        method: "POST",
+        url: `/api/ee/transform-optimizer/${transformId}/index/drop`,
+        body: { index_name: indexName },
+      }),
+      invalidatesTags: (_data, error, { transformId }) =>
+        invalidateTags(error, [
+          idTag("transform-optimizer-indexes", transformId),
+        ]),
+    }),
+    bulkOptimize: builder.mutation<
+      BulkOptimizeResponse,
+      { transformIds: number[] }
+    >({
+      query: ({ transformIds }) => ({
+        method: "POST",
+        url: "/api/ee/transform-optimizer/bulk-optimize",
+        body: { transform_ids: transformIds },
+      }),
+      // The optimizer flips transform.optimized as each analysis lands.
+      // Invalidate the transform list so the UI re-fetches and the gif
+      // appears on transforms that the bulk pass marked optimized.
+      invalidatesTags: (_data, error) =>
+        invalidateTags(error, [listTag("transform")]),
+    }),
+    bulkOptimizeStatus: builder.query<BulkOptimizeStatusResponse, void>({
+      query: () => ({
+        method: "GET",
+        url: "/api/ee/transform-optimizer/bulk-optimize/status",
+      }),
+    }),
+    markOptimized: builder.mutation<
+      { id: number; optimized: true },
+      { transformId: number }
+    >({
+      query: ({ transformId }) => ({
+        method: "POST",
+        url: `/api/ee/transform-optimizer/${transformId}/mark-optimized`,
+      }),
+      invalidatesTags: (_data, error, { transformId }) =>
+        invalidateTags(error, [
+          idTag("transform", transformId),
+          listTag("transform"),
+        ]),
+    }),
+  }),
+});
+
+export const {
+  useLazyOptimizeQuery,
+  useOptimizeQuery,
+  useVerifyProposalMutation,
+  useAcceptProposalMutation,
+  useListTargetIndexesQuery,
+  useDropTargetIndexMutation,
+  useBulkOptimizeMutation,
+  useBulkOptimizeStatusQuery,
+  useMarkOptimizedMutation,
+} = optimizerApi;

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/BulkResultsDrawer/BulkResultsDrawer.module.css

@@ -0,0 +1,114 @@
+/* YOLO marquee — rainbow gradient sweeping behind the button label, plus a
+   gentle pulse on the border so the call-to-action is impossible to miss
+   in a list of "regular" accordion rows. Mantine's Button paints its bg
+   via the `--button-bg` custom prop on the root element, so we override
+   that with our animated gradient. */
+.yoloButton {
+  --button-bg: linear-gradient(
+    90deg,
+    #ff5e5b,
+    #ffae42,
+    #fff15c,
+    #50fa7b,
+    #5eb1ff,
+    #d970ff,
+    #ff5e5b
+  );
+  background-size: 300% 100%;
+  animation:
+    yoloMarquee 2.5s linear infinite,
+    yoloPulse 1.4s ease-in-out infinite,
+    yoloBlink 3s linear infinite;
+  color: #fff;
+  font-weight: 700;
+  letter-spacing: 0.04em;
+  text-shadow: 0 0 6px rgba(0, 0, 0, 0.35);
+  border: 2px solid rgba(255, 255, 255, 0.6);
+}
+
+@keyframes yoloMarquee {
+  0% {
+    background-position: 0% 50%;
+  }
+  100% {
+    background-position: 300% 50%;
+  }
+}
+
+@keyframes yoloPulse {
+  0%,
+  100% {
+    box-shadow:
+      0 0 0 0 rgba(255, 255, 255, 0.5),
+      0 0 12px 2px rgba(255, 100, 200, 0.4);
+  }
+  50% {
+    box-shadow:
+      0 0 0 4px rgba(255, 255, 255, 0.15),
+      0 0 18px 6px rgba(100, 200, 255, 0.5);
+  }
+}
+
+/* Coin explosion — fixed-position overlay that briefly rains coins from
+   the centre of the screen outward and downward. Each coin reads its own
+   trajectory from CSS vars set inline so we can randomise per-coin without
+   needing N keyframe definitions. */
+.coinExplosion {
+  position: fixed;
+  inset: 0;
+  pointer-events: none;
+  z-index: 9999;
+  overflow: hidden;
+}
+
+.coin {
+  position: absolute;
+  top: 50%;
+  left: 50%;
+  font-size: 32px;
+  line-height: 1;
+  animation: coinFly 1.8s var(--coin-delay, 0s) cubic-bezier(0.15, 0.7, 0.4, 1)
+    forwards;
+}
+
+@keyframes coinFly {
+  0% {
+    transform: translate(-50%, -50%) translate(0, 0) rotate(0deg) scale(0.4);
+    opacity: 1;
+  }
+  20% {
+    transform: translate(-50%, -50%)
+      translate(calc(var(--coin-tx) * 0.4), calc(var(--coin-ty) * 0.4))
+      rotate(calc(var(--coin-rot) * 0.4)) scale(1.2);
+    opacity: 1;
+  }
+  100% {
+    transform: translate(-50%, -50%) translate(var(--coin-tx), var(--coin-ty))
+      rotate(var(--coin-rot)) scale(0.7);
+    opacity: 0;
+  }
+}
+
+/* Flashing rainbow — never hides the button. We rotate the hue of the
+   whole element through 360° and dip the brightness/opacity slightly in
+   sync, so the rainbow gradient continuously shifts colors while the
+   label stays readable. */
+@keyframes yoloBlink {
+  0%,
+  100% {
+    filter: hue-rotate(0deg) saturate(1.4) brightness(1.05);
+    opacity: 1;
+  }
+  25% {
+    filter: hue-rotate(90deg) saturate(1.8) brightness(1.15);
+    opacity: 0.92;
+  }
+  50% {
+    filter: hue-rotate(180deg) saturate(1.4) brightness(1);
+    opacity: 1;
+  }
+  75% {
+    filter: hue-rotate(270deg) saturate(1.8) brightness(1.15);
+    opacity: 0.92;
+  }
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/BulkResultsDrawer/BulkResultsDrawer.tsx

@@ -0,0 +1,746 @@
+import {
+  type CSSProperties,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
+import { Link } from "react-router";
+import { c, t } from "ttag";
+
+import { useListTransformsQuery } from "metabase/api";
+import { useDispatch } from "metabase/redux";
+import {
+  Accordion,
+  Anchor,
+  Badge,
+  Box,
+  Button,
+  Divider,
+  Group,
+  Icon,
+  Loader,
+  Modal,
+  Progress,
+  ScrollArea,
+  Stack,
+  Text,
+  Tooltip,
+} from "metabase/ui";
+import * as Urls from "metabase/urls";
+
+import { useMetadataToasts } from "metabase/metadata/hooks";
+
+import {
+  type AcceptMode,
+  type BulkOptimizeDoneEntry,
+  type BulkOptimizeStatusResponse,
+  optimizerApi,
+  useAcceptProposalMutation,
+  useBulkOptimizeStatusQuery,
+  useMarkOptimizedMutation,
+} from "../../api";
+
+import S from "./BulkResultsDrawer.module.css";
+import { BulkTransformCard } from "./BulkTransformCard";
+
+type RowStatus = "running" | "queued" | "done" | "failed";
+
+type Row =
+  | { id: number; name: string; status: "running" }
+  | { id: number; name: string; status: "queued" }
+  | { id: number; name: string; status: "done"; entry: BulkOptimizeDoneEntry }
+  | { id: number; name: string; status: "failed"; message: string };
+
+// Poll every 2s while a job is in flight. Once everything is settled the
+// query stays mounted (so the drawer keeps showing results) but pollingInterval
+// resolves to 0, which RTK Query treats as "stop polling".
+const POLL_MS = 2000;
+
+type BulkResultsDrawerProps = {
+  opened: boolean;
+  onClose: () => void;
+};
+
+export function BulkResultsDrawer({ opened, onClose }: BulkResultsDrawerProps) {
+  // Polling cadence is driven from a state that we adjust in an effect once
+  // the response lands — destructuring `data` and referencing it inside the
+  // same hook-options object is a temporal-dead-zone trap.
+  const [pollingInterval, setPollingInterval] = useState(0);
+  const { data, isLoading } = useBulkOptimizeStatusQuery(undefined, {
+    pollingInterval,
+    skip: !opened,
+  });
+  const dispatch = useDispatch();
+
+  useEffect(() => {
+    if (!opened) {
+      setPollingInterval(0);
+      return;
+    }
+    const stillRunning = (data?.pending ?? []).length > 0;
+    setPollingInterval(stillRunning ? POLL_MS : 0);
+  }, [opened, data]);
+
+  // The detail page's TransformOptimizerSection reads from the optimize
+  // endpoint's RTK Query cache. The bulk run populated the BE proposal
+  // cache (so verify / accept work) but didn't touch the FE cache —
+  // navigating to ANY of the bulk-analysed transforms would otherwise
+  // land on the trigger button and force a re-analysis. Seed every
+  // completed entry into the FE cache as soon as it lands, regardless
+  // of whether the user later clicks through the drawer or arrives at
+  // the transform via some other route (list page, deep link).
+  const seedOptimizeCache = useCallback(
+    (entry: BulkOptimizeDoneEntry) => {
+      dispatch(
+        optimizerApi.util.upsertQueryData(
+          "optimize",
+          { transformId: entry.transform.id, analyze: false },
+          {
+            transform: {
+              id: entry.transform.id,
+              name: entry.transform.name,
+              source_database_id: entry.transform.source_database_id,
+            },
+            // The bulk endpoint doesn't return the compiled SQL — the
+            // section doesn't read from this field, only the proposals
+            // array, so null is safe.
+            sql: null,
+            summary: entry.summary,
+            proposals: entry.proposals,
+            optimization_degree: entry.optimization_degree,
+          },
+        ),
+      );
+    },
+    [dispatch],
+  );
+
+  // Seed every entry that's appeared in `data.done` since the last poll.
+  // Tracked through a ref so re-renders triggered by polling don't keep
+  // dispatching identical upserts (RTK would no-op them, but it's wasted
+  // work and clutters DevTools).
+  const seededIdsRef = useRef<Set<number>>(new Set());
+  useEffect(() => {
+    if (!data?.done) {
+      return;
+    }
+    for (const entry of Object.values(data.done)) {
+      if (!seededIdsRef.current.has(entry.transform.id)) {
+        seededIdsRef.current.add(entry.transform.id);
+        seedOptimizeCache(entry);
+      }
+    }
+  }, [data?.done, seedOptimizeCache]);
+
+  // Reset the dedup set when a fresh bulk run kicks off — `started_at`
+  // changes per POST /bulk-optimize and is the cleanest "this is a new
+  // batch" signal we have without threading a job id everywhere.
+  useEffect(() => {
+    seededIdsRef.current = new Set();
+  }, [data?.started_at]);
+
+  const total = data?.total ?? 0;
+  const doneEntries = useMemo(
+    () => Object.values(data?.done ?? {}),
+    [data?.done],
+  );
+  const failedEntries = useMemo(
+    () => Object.entries(data?.failed ?? {}),
+    [data?.failed],
+  );
+  const pendingIds = data?.pending ?? [];
+  const settled = doneEntries.length + failedEntries.length;
+  const isRunning = pendingIds.length > 0;
+
+  // The transforms list is almost always cached by the parent
+  // TransformListPage, so this is a no-network lookup in practice. We use
+  // it to render each pending row by name rather than by bare id.
+  const { data: transforms } = useListTransformsQuery({});
+  const transformNameById = useMemo(() => {
+    const out = new Map<number, string>();
+    for (const t of transforms ?? []) {
+      out.set(t.id, t.name);
+    }
+    return out;
+  }, [transforms]);
+
+  // Cross-card dedup for identical index proposals. The optimizer often
+  // emits the same `CREATE INDEX IF NOT EXISTS idx_events_customer_id …`
+  // across multiple transforms that reference the same hot table; once any
+  // one is accepted, the rest are no-ops. We key on `ddl_statement.index_name`
+  // which the BE assigns at validation time, so the comparison is stable
+  // across both `source-db` and `transform-target` targets.
+  const [acceptedIndexNames, setAcceptedIndexNames] = useState<Set<string>>(
+    () => new Set(),
+  );
+  const handleIndexAccepted = useCallback((indexName: string) => {
+    setAcceptedIndexNames((prev) => {
+      if (prev.has(indexName)) {
+        return prev;
+      }
+      const next = new Set(prev);
+      next.add(indexName);
+      return next;
+    });
+  }, []);
+  // Reset the dedup set when a fresh bulk run starts so previously accepted
+  // indexes don't pre-suppress the next batch.
+  useEffect(() => {
+    setAcceptedIndexNames(new Set());
+  }, [data?.started_at]);
+
+  // YOLO accept: iterate every done entry and accept its proposals in one
+  // pass. Replace mode when there's a rewrite (source swap in place,
+  // precomputes become siblings, indices run inline); New mode otherwise.
+  // Sequential so per-transform failures surface as discrete toasts.
+  const [accept, { isLoading: isAccepting }] = useAcceptProposalMutation();
+  // After every successful YOLO accept on a transform we also flip its
+  // `optimized` flag. The accept itself replaces the source, which the
+  // before-update hook treats as "must re-verify" by clearing the flag —
+  // markOptimized writes it back to true because the user has made an
+  // explicit "I'm done with this one" decision via YOLO.
+  const [markOptimized] = useMarkOptimizedMutation();
+  const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
+  const [yoloAcceptedIds, setYoloAcceptedIds] = useState<Set<number>>(
+    () => new Set(),
+  );
+  useEffect(() => {
+    setYoloAcceptedIds(new Set());
+  }, [data?.started_at]);
+
+  // Coin explosion: incremented each time YOLO is fired. The mounted
+  // overlay keyed on this value runs its CSS animation once and then
+  // unmounts via a 2s timeout. Keying ensures back-to-back YOLO clicks
+  // restart the animation cleanly instead of letting React reuse the
+  // already-played overlay.
+  const [coinKey, setCoinKey] = useState<number | null>(null);
+
+  const handleYolo = useCallback(async () => {
+    setCoinKey(Date.now());
+    setTimeout(() => setCoinKey(null), 2000);
+    let successCount = 0;
+    let failCount = 0;
+    for (const entry of doneEntries) {
+      if (yoloAcceptedIds.has(entry.transform.id)) {
+        continue;
+      }
+      const visible = entry.proposals.filter((p) => {
+        const idxName = p.ddl_statement?.index_name;
+        return !(idxName && acceptedIndexNames.has(idxName));
+      });
+      if (visible.length === 0) {
+        continue;
+      }
+      const hasRewrite = visible.some((p) => p.kind === "rewrite");
+      const mode: AcceptMode = hasRewrite ? "replace" : "new";
+      const proposalIds = visible.map((p) => p.id);
+      const { data: result, error } = await accept({
+        transformId: entry.transform.id,
+        proposalIds,
+        mode,
+      });
+      if (error || !result) {
+        failCount += 1;
+        sendErrorToast(t`YOLO failed on "${entry.transform.name}"`);
+        continue;
+      }
+      successCount += 1;
+      // Mirror the per-card cross-batch dedup: tell siblings about every
+      // index we just installed so they don't re-issue duplicate CREATE
+      // INDEX statements.
+      for (const p of visible) {
+        const idxName = p.ddl_statement?.index_name;
+        if (idxName) {
+          handleIndexAccepted(idxName);
+        }
+      }
+      // Flip the persisted `optimized` flag so the transform's list-page
+      // row shows the Sonic gif right away. We swallow errors here —
+      // the accept already landed; failing to mark optimized is cosmetic.
+      try {
+        await markOptimized({ transformId: entry.transform.id });
+      } catch (_) {
+        // ignore — accept succeeded, the user can re-optimize later
+      }
+      setYoloAcceptedIds((prev) => {
+        const next = new Set(prev);
+        next.add(entry.transform.id);
+        return next;
+      });
+    }
+    if (successCount > 0) {
+      sendSuccessToast(
+        c("YOLO toast — N succeeded, M failed")
+          .t`YOLO done: ${successCount} accepted${failCount > 0 ? `, ${failCount} failed` : ""}.`,
+      );
+      // Close the modal once the batch lands so the user sees the toast
+      // confirm against the transforms list (with newly-flipped Sonic
+      // gifs) rather than the now-stale drawer. Small delay lets the
+      // coin explosion finish playing before the modal goes away.
+      setTimeout(onClose, 700);
+    }
+  }, [
+    accept,
+    doneEntries,
+    acceptedIndexNames,
+    yoloAcceptedIds,
+    handleIndexAccepted,
+    markOptimized,
+    sendErrorToast,
+    sendSuccessToast,
+    onClose,
+  ]);
+
+  // Eligible-for-YOLO count: done entries with at least one proposal still
+  // visible after cross-card dedup, that haven't already been YOLO'd.
+  const yoloableCount = useMemo(() => {
+    return doneEntries.reduce((acc, entry) => {
+      if (yoloAcceptedIds.has(entry.transform.id)) {
+        return acc;
+      }
+      const has = entry.proposals.some((p) => {
+        const idxName = p.ddl_statement?.index_name;
+        return !(idxName && acceptedIndexNames.has(idxName));
+      });
+      return has ? acc + 1 : acc;
+    }, 0);
+  }, [doneEntries, acceptedIndexNames, yoloAcceptedIds]);
+
+  // Stable ordering for the accordion. The BE status response gives us
+  // `pending` in submission order and `done`/`failed` as unordered maps
+  // keyed by insertion-order — which is *completion* order, not submission
+  // order, so a slower transform would visually jump past a faster one as
+  // it finishes. We freeze each id's position at first sight and never
+  // re-shuffle. Reset on `started_at` so a new batch starts clean.
+  const orderRef = useRef<number[]>([]);
+  useEffect(() => {
+    orderRef.current = [];
+  }, [data?.started_at]);
+  const rows = useMemo<Row[]>(() => {
+    const doneById = new Map(doneEntries.map((e) => [e.transform.id, e]));
+    const failedByIdStr = new Map(failedEntries);
+    const seen = new Set(orderRef.current);
+    // Collect any ids we haven't seen yet across all three buckets, then
+    // sort them by id ascending so the initial seed (which may happen
+    // mid-run, with some items already complete) lands in a deterministic
+    // order roughly matching submission. After that, ids only get appended
+    // here in the rare case a new one shows up mid-batch.
+    const incoming: number[] = [];
+    for (const id of pendingIds) {
+      if (!seen.has(id)) {
+        incoming.push(id);
+        seen.add(id);
+      }
+    }
+    for (const entry of doneEntries) {
+      if (!seen.has(entry.transform.id)) {
+        incoming.push(entry.transform.id);
+        seen.add(entry.transform.id);
+      }
+    }
+    for (const [idStr] of failedEntries) {
+      const id = Number(idStr);
+      if (!seen.has(id)) {
+        incoming.push(id);
+        seen.add(id);
+      }
+    }
+    incoming.sort((a, b) => a - b);
+    orderRef.current = [...orderRef.current, ...incoming];
+    // Drop ids that no longer appear anywhere — defensive against a fresh
+    // batch arriving without `started_at` changing (shouldn't happen, but
+    // cheap to handle).
+    const present = new Set<number>([
+      ...pendingIds,
+      ...doneEntries.map((e) => e.transform.id),
+      ...failedEntries.map(([id]) => Number(id)),
+    ]);
+    orderRef.current = orderRef.current.filter((id) => present.has(id));
+
+    return orderRef.current.map((id): Row => {
+      const name = transformNameById.get(id) ?? t`Transform #${id}`;
+      const done = doneById.get(id);
+      if (done) {
+        return { id, name, status: "done", entry: done };
+      }
+      const failed = failedByIdStr.get(String(id));
+      if (failed != null) {
+        return { id, name, status: "failed", message: failed };
+      }
+      // The BE processes the queue sequentially, so the first pending id
+      // is the one actively analyzing right now.
+      const pendingIdx = pendingIds.indexOf(id);
+      return {
+        id,
+        name,
+        status: pendingIdx === 0 ? "running" : "queued",
+      };
+    });
+  }, [pendingIds, doneEntries, failedEntries, transformNameById]);
+
+  return (
+    <Modal
+      opened={opened}
+      onClose={onClose}
+      centered
+      // The drawer used to be a side panel — wide enough for proposal
+      // cards but cramped for the SQL diffs. Centered modal at 80vw / 90vh
+      // gives breathing room for several proposals side by side.
+      size="80%"
+      // Outer chrome padding — Mantine applies this to the modal body
+      // including the title row, so the bolt icon / heading sit inside
+      // the same gutter as the content below. Inner sections then only
+      // need vertical spacing between siblings.
+      padding="xl"
+      title={
+        <Group gap="sm" wrap="nowrap">
+          <Icon name="bolt" />
+          <Text fw="bold">{t`Bulk optimization`}</Text>
+        </Group>
+      }
+    >
+      {isLoading && !data ? (
+        <Group justify="center" py="md">
+          <Loader size="sm" />
+        </Group>
+      ) : total === 0 ? (
+        <Box>
+          <Text c="text-secondary">
+            {t`No bulk-optimize run has been kicked off yet.`}
+          </Text>
+        </Box>
+      ) : (
+        <Stack gap="lg">
+          <BulkProgressHeader
+            data={data!}
+            settled={settled}
+            isRunning={isRunning}
+            currentName={
+              isRunning
+                ? transformNameById.get(pendingIds[0]) ??
+                  t`Transform #${pendingIds[0]}`
+                : null
+            }
+          />
+          <Divider />
+          <ScrollArea h="calc(90vh - 280px)" type="auto" offsetScrollbars>
+            <Box pr="md">
+              {rows.length === 0 ? (
+                <Text c="text-secondary">{t`No transforms in this batch yet.`}</Text>
+              ) : (
+                <Accordion
+                  multiple
+                  // Queued items have no body content yet, so chevrons on
+                  // those rows are hidden — keep the header informative
+                  // but not interactive.
+                  chevronPosition="right"
+                  variant="separated"
+                  radius="md"
+                  // Closed by default: the user opens the file they want to
+                  // act on. Empty initial value with `multiple` lets them
+                  // open several at once for comparison.
+                  defaultValue={[]}
+                >
+                  {rows.map((row) => (
+                    <BulkResultRow
+                      key={row.id}
+                      row={row}
+                      acceptedIndexNames={acceptedIndexNames}
+                      onIndexAccepted={handleIndexAccepted}
+                    />
+                  ))}
+                </Accordion>
+              )}
+            </Box>
+          </ScrollArea>
+          {!isRunning && yoloableCount > 0 && (
+            <Group justify="center" pt="md">
+              <Tooltip
+                label={t`Accept every visible proposal across all transforms in one go. Rewrites replace the source in place; indices run immediately.`}
+                multiline
+                w={280}
+              >
+                <Button
+                  className={S.yoloButton}
+                  size="xl"
+                  loading={isAccepting}
+                  onClick={handleYolo}
+                  leftSection={<Icon name="bolt" size={20} />}
+                  px="xl"
+                >
+                  {c("YOLO accept-all button label, {0} is a count")
+                    .t`YOLO — accept all (${yoloableCount})`}
+                </Button>
+              </Tooltip>
+            </Group>
+          )}
+        </Stack>
+      )}
+      {coinKey != null && <CoinExplosion key={coinKey} />}
+    </Modal>
+  );
+}
+
+function BulkProgressHeader({
+  data,
+  settled,
+  isRunning,
+  currentName,
+}: {
+  data: BulkOptimizeStatusResponse;
+  settled: number;
+  isRunning: boolean;
+  currentName: string | null;
+}) {
+  const total = data.total;
+  const value = total > 0 ? (settled / total) * 100 : 0;
+  const failedCount = Object.keys(data.failed).length;
+
+  return (
+    <Stack gap="xs">
+      <Group justify="space-between" wrap="nowrap" align="flex-start">
+        <Stack gap={2} miw={0}>
+          <Text fw="bold">
+            {isRunning
+              ? c("Bulk-optimize progress, X of Y")
+                  .t`Analyzing ${settled} / ${total}`
+              : c("Bulk-optimize complete, X of Y")
+                  .t`Done — ${settled} / ${total}`}
+          </Text>
+          {isRunning && currentName && (
+            <Text c="text-secondary" fz="sm">
+              {c("Bulk-optimize header subline naming the transform in flight")
+                .t`Currently analyzing: ${currentName}`}
+            </Text>
+          )}
+        </Stack>
+        {isRunning && <Loader size="xs" />}
+      </Group>
+      <Progress value={value} animated={isRunning} />
+      {failedCount > 0 && (
+        <Text c="error" fz="sm">
+          {c("Failed count in bulk-optimize header")
+            .t`${failedCount} failed — see below.`}
+        </Text>
+      )}
+    </Stack>
+  );
+}
+
+/**
+ * One accordion item per transform in the bulk batch. Header carries the
+ * status (icon / loader + label + suggestion count badge) and stays
+ * informative when collapsed; the panel shows the result body — proposals
+ * for `done`, an error message for `failed`, and a placeholder for the
+ * still-in-flight states. Queued rows have no body and are non-interactive.
+ */
+function BulkResultRow({
+  row,
+  acceptedIndexNames,
+  onIndexAccepted,
+}: {
+  row: Row;
+  acceptedIndexNames: Set<string>;
+  onIndexAccepted: (indexName: string) => void;
+}) {
+  const expandable = row.status === "done" || row.status === "failed";
+  return (
+    <Accordion.Item value={String(row.id)}>
+      <Accordion.Control
+        // Disable the chevron + click handling for rows with no body. The
+        // header text still conveys "Queued" / "Analyzing…", which is what
+        // matters at this stage.
+        disabled={!expandable}
+        chevron={expandable ? undefined : <Box w={0} />}
+      >
+        <RowHeader row={row} />
+      </Accordion.Control>
+      {expandable && (
+        <Accordion.Panel>
+          {row.status === "done" ? (
+            <BulkTransformCard
+              entry={row.entry}
+              acceptedIndexNames={acceptedIndexNames}
+              onIndexAccepted={onIndexAccepted}
+            />
+          ) : (
+            <FailureBody transformId={row.id} message={row.message} />
+          )}
+        </Accordion.Panel>
+      )}
+    </Accordion.Item>
+  );
+}
+
+/**
+ * True when this batch entry has at least one high-severity rewrite or
+ * precompute proposal. Index proposals are valuable but the user's bar
+ * for "strong rewrite candidate" is specifically about *structural* SQL
+ * changes the LLM is confident about — those are the ones worth a human
+ * review pass first. We don't include medium/low because the LLM already
+ * downgrades cosmetic-or-borderline rewrites per the prelude.
+ */
+function isStrongRewriteCandidate(row: Row): boolean {
+  if (row.status !== "done") {
+    return false;
+  }
+  return row.entry.proposals.some(
+    (p) =>
+      (p.kind === "rewrite" || p.kind === "precompute") &&
+      p.severity === "high",
+  );
+}
+
+function RowHeader({ row }: { row: Row }) {
+  const isStrong = isStrongRewriteCandidate(row);
+  return (
+    <Group gap="sm" wrap="nowrap" align="center" w="100%">
+      <StatusIndicator status={row.status} />
+      <Text fw="bold" miw={0} truncate style={{ flex: 1 }}>
+        {row.name}
+      </Text>
+      {isStrong && (
+        <Tooltip
+          label={t`The optimizer is confident this transform has a high-impact rewrite or precompute available.`}
+          multiline
+          w={260}
+        >
+          <Badge
+            color="brand"
+            variant="filled"
+            leftSection={<Icon name="star" size={10} />}
+          >
+            {t`Top rewrite candidate`}
+          </Badge>
+        </Tooltip>
+      )}
+      <RowStatusMeta row={row} />
+    </Group>
+  );
+}
+
+function StatusIndicator({ status }: { status: RowStatus }) {
+  switch (status) {
+    case "running":
+      return <Loader size="xs" />;
+    case "queued":
+      return <Icon name="clock" c="text-secondary" />;
+    case "done":
+      return <Icon name="check_filled" c="success" />;
+    case "failed":
+      return <Icon name="warning" c="error" />;
+  }
+}
+
+function RowStatusMeta({ row }: { row: Row }) {
+  switch (row.status) {
+    case "running":
+      return (
+        <Text c="text-secondary" fz="sm">
+          {t`Analyzing…`}
+        </Text>
+      );
+    case "queued":
+      return (
+        <Text c="text-secondary" fz="sm">
+          {t`Queued`}
+        </Text>
+      );
+    case "failed":
+      return (
+        <Badge color="error" variant="light">
+          {t`Failed`}
+        </Badge>
+      );
+    case "done": {
+      const count = row.entry.proposals.length;
+      const isOptimized = row.entry.optimization_degree === 100;
+      if (isOptimized || count === 0) {
+        return (
+          <Badge color="success" variant="light">
+            {t`Fully optimized`}
+          </Badge>
+        );
+      }
+      // Use the warning palette and a bigger size so the "this transform
+      // has suggestions to review" chip jumps out of the row at a glance
+      // — brand-filled was visually similar to the row chrome and got lost.
+      return (
+        <Badge color="warning" variant="filled" size="lg">
+          {c("Proposal count badge in bulk results")
+            .t`${count} suggestion(s)`}
+        </Badge>
+      );
+    }
+  }
+}
+
+function FailureBody({
+  transformId,
+  message,
+}: {
+  transformId: number;
+  message: string;
+}) {
+  return (
+    <Group gap="sm" wrap="nowrap" align="flex-start" p="md">
+      <Icon name="warning" c="error" />
+      <Stack gap={2} miw={0} style={{ flex: 1 }}>
+        <Anchor component={Link} to={Urls.transformRun(transformId)} fw="bold">
+          {c("Failed transform link in bulk results, {0} is the transform id")
+            .t`Open transform #${transformId}`}
+        </Anchor>
+        <Text c="error" fz="sm">
+          {message}
+        </Text>
+      </Stack>
+    </Group>
+  );
+}
+
+/**
+ * Coin shower triggered when the user fires YOLO. We pre-generate 60 coins
+ * each with a randomised polar trajectory + delay so the burst feels
+ * organic — pure CSS animation from there; React only owns mount/unmount.
+ *
+ * Mounted under the Modal which itself portals to the document root, so
+ * the position-fixed overlay covers the full viewport regardless of the
+ * surrounding layout.
+ */
+function CoinExplosion() {
+  const coins = useMemo(() => {
+    return Array.from({ length: 60 }, (_, i) => {
+      const angle = Math.random() * Math.PI * 2;
+      const dist = 220 + Math.random() * 420;
+      const tx = Math.cos(angle) * dist;
+      // Bias the spread upward — feels more like an explosion than a splash.
+      const ty = Math.sin(angle) * dist - 120;
+      const rot = (Math.random() - 0.5) * 720;
+      const delay = Math.random() * 0.25;
+      return { id: i, tx, ty, rot, delay };
+    });
+  }, []);
+
+  return (
+    <Box className={S.coinExplosion} aria-hidden="true">
+      {coins.map((c) => (
+        <span
+          key={c.id}
+          className={S.coin}
+          style={
+            {
+              "--coin-tx": `${c.tx}px`,
+              "--coin-ty": `${c.ty}px`,
+              "--coin-rot": `${c.rot}deg`,
+              "--coin-delay": `${c.delay}s`,
+            } as CSSProperties
+          }
+        >
+          🪙
+        </span>
+      ))}
+    </Box>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/BulkResultsDrawer/BulkTransformCard.tsx

@@ -0,0 +1,254 @@
+import { useMemo, useState } from "react";
+import { Link } from "react-router";
+import { c, t } from "ttag";
+
+import { useGetTransformQuery } from "metabase/api";
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import {
+  Anchor,
+  Badge,
+  Card,
+  Divider,
+  Group,
+  Stack,
+  Text,
+} from "metabase/ui";
+import * as Urls from "metabase/urls";
+
+import {
+  type AcceptMode,
+  type BulkOptimizeDoneEntry,
+  type VerifyResponse,
+  useAcceptProposalMutation,
+  useVerifyProposalMutation,
+} from "../../api";
+import type { Proposal } from "../../types";
+import { ProposalCard } from "../ProposalCard";
+import {
+  getNativeSql,
+  groupProposalsByDependency,
+  reportVerify,
+  resolveDependencyNames,
+  topoOrderForAccept,
+} from "../TransformOptimizerSection/TransformOptimizerSection";
+import { ProposalGroup } from "../TransformOptimizerSection/ProposalGroup";
+
+type Props = {
+  entry: BulkOptimizeDoneEntry;
+  /**
+   * Index names that have already been accepted somewhere in this bulk
+   * batch — index proposals matching one of these are auto-dismissed
+   * from this card so the user doesn't see the same `CREATE INDEX`
+   * twice across transforms that share a hot table.
+   */
+  acceptedIndexNames: Set<string>;
+  /**
+   * Called when this card accepts an index proposal so other cards can
+   * dedup against it. The `indexName` is `ddl_statement.index_name`,
+   * which the BE assigns at validation time.
+   */
+  onIndexAccepted: (indexName: string) => void;
+};
+
+/**
+ * One transform's slice of the bulk-optimize drawer. Reuses the exact same
+ * `<ProposalCard>` the detail page renders — verify and accept hit the
+ * existing endpoints, and the bulk-optimize BE already populated the
+ * proposal cache so they resolve without re-running the LLM.
+ *
+ * Local-only state:
+ *   - `dismissed` — proposals the user accepted or dismissed in this
+ *     drawer session. We don't mutate the bulk-status BE state; we just
+ *     hide them on the client so the card empties out naturally.
+ *   - `busyProposalId` — which proposal's button should show a spinner.
+ */
+export function BulkTransformCard({
+  entry,
+  acceptedIndexNames,
+  onIndexAccepted,
+}: Props) {
+  const { transform, summary, proposals, optimization_degree } = entry;
+  // The bulk endpoint doesn't return the compiled SQL — fetch the full
+  // Transform object so ProposalCard can render a line-diff instead of
+  // the raw proposed body. RTK Query will hit the cache populated by the
+  // transforms list query without a network round-trip in the common case.
+  const { data: fullTransform } = useGetTransformQuery(transform.id);
+  const currentSql = fullTransform ? getNativeSql(fullTransform) : null;
+  const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
+  const [verify, verifyResult] = useVerifyProposalMutation();
+  const [accept, acceptResult] = useAcceptProposalMutation();
+  const [dismissed, setDismissed] = useState<Set<string>>(() => new Set());
+  const [busyProposalId, setBusyProposalId] = useState<string | null>(null);
+
+  const visibleProposals = useMemo(
+    () =>
+      proposals.filter((p) => {
+        if (dismissed.has(p.id)) {
+          return false;
+        }
+        // Drop any index proposal whose physical index has already been
+        // accepted by another card in this batch — accepting again would
+        // just hit `IF NOT EXISTS` and clutter the UI.
+        const idxName = p.ddl_statement?.index_name;
+        if (idxName && acceptedIndexNames.has(idxName)) {
+          return false;
+        }
+        return true;
+      }),
+    [proposals, dismissed, acceptedIndexNames],
+  );
+
+  const dismissOne = (id: string) =>
+    setDismissed((prev) => {
+      const next = new Set(prev);
+      next.add(id);
+      return next;
+    });
+
+  const handleVerify = async (proposal: Proposal) => {
+    setBusyProposalId(proposal.id);
+    const { data, error } = await verify({
+      transformId: transform.id,
+      proposal,
+    });
+    setBusyProposalId(null);
+    if (error) {
+      sendErrorToast(t`Failed to verify proposal`);
+      return;
+    }
+    reportVerify(data as VerifyResponse, sendSuccessToast, sendErrorToast);
+  };
+
+  const handleAccept = async (proposal: Proposal, mode: AcceptMode = "new") => {
+    // Replace mode only touches the original transform — never pull
+    // ancestor precomputes into the accept set.
+    const proposalIds =
+      mode === "replace"
+        ? [proposal.id]
+        : topoOrderForAccept(proposal, visibleProposals);
+    setBusyProposalId(proposal.id);
+    const { data, error } = await accept({
+      transformId: transform.id,
+      proposalIds,
+      mode,
+    });
+    setBusyProposalId(null);
+    if (error) {
+      sendErrorToast(t`Failed to accept proposal`);
+      return;
+    }
+    if (data && "replaced_transform" in data) {
+      sendSuccessToast(t`Replaced source of "${data.replaced_transform.name}"`);
+    } else {
+      const created = data?.created_transforms ?? [];
+      if (created.length > 0) {
+        sendSuccessToast(
+          created.length === 1
+            ? t`New transform "${created[0].name}" created`
+            : c("Bulk drawer accept toast, {0} is a count")
+                .t`${created.length} new transforms created`,
+        );
+      } else {
+        sendSuccessToast(t`Index changes applied`);
+      }
+    }
+    for (const id of proposalIds) {
+      dismissOne(id);
+      // Tell sibling cards this index is now installed so they drop their
+      // duplicate. Non-index proposals have no `index_name` so the check
+      // is a cheap no-op for rewrites / precomputes.
+      const accepted = proposals.find((p) => p.id === id);
+      const idxName = accepted?.ddl_statement?.index_name;
+      if (idxName) {
+        onIndexAccepted(idxName);
+      }
+    }
+  };
+
+  const isOptimized = optimization_degree === 100;
+  const allDismissed =
+    visibleProposals.length === 0 && proposals.length > 0;
+
+  return (
+    <Card withBorder p="md" radius="md">
+      <Stack gap="sm">
+        <Group justify="space-between" wrap="nowrap" align="flex-start">
+          <Stack gap={2} miw={0} style={{ flex: 1 }}>
+            <Anchor
+              component={Link}
+              to={Urls.transformRun(transform.id)}
+              fw="bold"
+            >
+              {transform.name}
+            </Anchor>
+            {summary && (
+              <Text c="text-secondary" fz="sm">
+                {summary}
+              </Text>
+            )}
+          </Stack>
+          <Badge
+            color={isOptimized || allDismissed ? "success" : "brand"}
+            variant={isOptimized || allDismissed ? "light" : "filled"}
+          >
+            {isOptimized
+              ? t`Fully optimized`
+              : allDismissed
+                ? t`Triaged`
+                : c("Proposal count badge in bulk results")
+                    .t`${visibleProposals.length} suggestion(s)`}
+          </Badge>
+        </Group>
+
+        {visibleProposals.length > 0 && (
+          <>
+            <Divider />
+            <Stack gap="md">
+              {groupProposalsByDependency(visibleProposals).map((group) => (
+                <ProposalGroup
+                  key={group.key}
+                  proposals={group.proposals}
+                  renderCard={(proposal) => {
+                    const canVerify = proposal.kind !== "index";
+                    const dependencyNames = resolveDependencyNames(
+                      proposal,
+                      visibleProposals,
+                    );
+                    return (
+                      <ProposalCard
+                        key={proposal.id}
+                        proposal={proposal}
+                        currentSql={currentSql}
+                        dependencyNames={dependencyNames}
+                        actions={{
+                          accept: {
+                            kind: "accept",
+                            busy:
+                              busyProposalId === proposal.id &&
+                              acceptResult.isLoading,
+                          },
+                          verify: canVerify
+                            ? {
+                                kind: "verify",
+                                busy:
+                                  busyProposalId === proposal.id &&
+                                  verifyResult.isLoading,
+                              }
+                            : undefined,
+                          dismiss: { kind: "dismiss" },
+                        }}
+                        onAccept={(mode) => handleAccept(proposal, mode)}
+                        onVerify={() => handleVerify(proposal)}
+                        onDismiss={() => dismissOne(proposal.id)}
+                      />
+                    );
+                  }}
+                />
+              ))}
+            </Stack>
+          </>
+        )}
+      </Stack>
+    </Card>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/BulkResultsDrawer/index.ts

@@ -0,0 +1 @@
+export { BulkResultsDrawer } from "./BulkResultsDrawer";

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/DefragLoader/DefragLoader.module.css

@@ -0,0 +1,297 @@
+/* 90s-defrag aesthetic, Metabase palette. Hard pixel borders, beveled
+   chrome, chunky cells — but every colour is pulled from the brand tokens
+   so it sits inside the rest of the product rather than next to it. */
+
+.window {
+  /* Chrome neutrals — derived from Metabase's grayscale so the window
+     reads as "Metabase chrome", not "Windows 95 chrome". */
+  --chrome: var(--mb-color-background-secondary);
+  --chrome-darker: color-mix(
+    in srgb,
+    var(--mb-color-border) 60%,
+    var(--mb-color-background-tertiary)
+  );
+  --chrome-shadow: var(--mb-color-border);
+  --chrome-highlight: var(--mb-color-background-primary);
+
+  /* Title bar — brand-blue gradient. */
+  --title-from: color-mix(in srgb, var(--mb-color-brand) 70%, black);
+  --title-to: var(--mb-color-brand);
+  --title-fg: var(--mb-color-text-white, #fff);
+
+  /* Cell palette — semantic tokens. */
+  --grid-bg: var(--mb-color-background-primary);
+  --grid-line: color-mix(in srgb, var(--mb-color-border) 70%, transparent);
+  --data: var(--mb-color-brand);
+  --data-light: color-mix(in srgb, var(--mb-color-brand) 60%, white);
+  --data-dark: color-mix(in srgb, var(--mb-color-brand) 60%, black);
+  --system: var(--mb-color-warning);
+  --system-light: color-mix(in srgb, var(--mb-color-warning) 60%, white);
+  --system-dark: color-mix(in srgb, var(--mb-color-warning) 60%, black);
+  --unmoveable: var(--mb-color-error);
+  --unmoveable-light: color-mix(in srgb, var(--mb-color-error) 60%, white);
+  --unmoveable-dark: color-mix(in srgb, var(--mb-color-error) 60%, black);
+  --moving: var(--mb-color-success);
+  --moving-light: color-mix(in srgb, var(--mb-color-success) 60%, white);
+  --moving-dark: color-mix(in srgb, var(--mb-color-success) 60%, black);
+  --free: var(--mb-color-background-primary);
+  --free-edge: color-mix(in srgb, var(--mb-color-border) 60%, transparent);
+
+  display: block;
+  width: 100%;
+  max-width: 40rem;
+  margin: 0 auto;
+  background: var(--chrome);
+  padding: 2px;
+  box-sizing: border-box;
+  /* Outer bevel: bright top-left, dim bottom-right. */
+  border-top: 2px solid var(--chrome-highlight);
+  border-left: 2px solid var(--chrome-highlight);
+  border-right: 2px solid var(--chrome-darker);
+  border-bottom: 2px solid var(--chrome-darker);
+  box-shadow: 1px 1px 0 0 var(--chrome-shadow);
+  font-family:
+    "Pixelated MS Sans Serif", "MS Sans Serif", Geneva, Tahoma, sans-serif;
+  color: var(--mb-color-text-primary);
+  user-select: none;
+}
+
+/* ---------- Title bar ---------- */
+.titleBar {
+  display: flex;
+  align-items: center;
+  gap: 4px;
+  height: 18px;
+  padding: 2px 2px 2px 3px;
+  background: linear-gradient(
+    90deg,
+    var(--title-from) 0%,
+    var(--title-to) 100%
+  );
+  color: var(--title-fg);
+  font-weight: bold;
+  font-size: 11px;
+}
+
+.titleIcon {
+  width: 14px;
+  height: 14px;
+  display: grid;
+  place-items: center;
+  background: var(--chrome);
+  color: var(--mb-color-text-primary);
+  border-top: 1px solid var(--chrome-highlight);
+  border-left: 1px solid var(--chrome-highlight);
+  border-right: 1px solid var(--chrome-darker);
+  border-bottom: 1px solid var(--chrome-darker);
+}
+
+.titleIconInner {
+  font-size: 10px;
+  font-weight: 900;
+  line-height: 1;
+  color: var(--mb-color-brand);
+}
+
+.titleText {
+  flex: 1;
+  letter-spacing: 0.02em;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+}
+
+.titleControls {
+  display: flex;
+  gap: 2px;
+}
+
+.titleBtn,
+.titleBtnClose {
+  display: inline-grid;
+  place-items: center;
+  width: 14px;
+  height: 14px;
+  background: var(--chrome);
+  color: var(--mb-color-text-primary);
+  font-size: 10px;
+  font-weight: bold;
+  line-height: 1;
+  border-top: 1px solid var(--chrome-highlight);
+  border-left: 1px solid var(--chrome-highlight);
+  border-right: 1px solid var(--chrome-darker);
+  border-bottom: 1px solid var(--chrome-darker);
+}
+
+/* ---------- Body ---------- */
+.body {
+  padding: 6px;
+  display: flex;
+  flex-direction: column;
+  gap: 6px;
+}
+
+.gridFrame {
+  background: var(--grid-bg);
+  padding: 2px;
+  /* Sunken bevel — opposite of the outer window. */
+  border-top: 1px solid var(--chrome-darker);
+  border-left: 1px solid var(--chrome-darker);
+  border-right: 1px solid var(--chrome-highlight);
+  border-bottom: 1px solid var(--chrome-highlight);
+}
+
+.grid {
+  display: grid;
+  gap: 1px;
+  background: var(--grid-line);
+  width: 100%;
+  /* Lock the cell aspect ratio to the grid's column/row count so the
+     squares stay square-ish as the container resizes. */
+  aspect-ratio: 28 / 8;
+}
+
+.cell {
+  display: block;
+  width: 100%;
+  height: 100%;
+  position: relative;
+}
+
+.cell::after {
+  content: "";
+  position: absolute;
+  inset: 0;
+  pointer-events: none;
+  box-shadow:
+    inset 1px 1px 0 0 rgba(255, 255, 255, 0.22),
+    inset -1px -1px 0 0 rgba(0, 0, 0, 0.18);
+}
+
+.free {
+  background: var(--free);
+}
+
+.free::after {
+  box-shadow: inset 0 0 0 1px var(--free-edge);
+}
+
+.data {
+  background: var(--data);
+}
+
+.data::after {
+  box-shadow:
+    inset 1px 1px 0 0 var(--data-light),
+    inset -1px -1px 0 0 var(--data-dark);
+}
+
+.system {
+  background: var(--system);
+}
+
+.system::after {
+  box-shadow:
+    inset 1px 1px 0 0 var(--system-light),
+    inset -1px -1px 0 0 var(--system-dark);
+}
+
+.unmoveable {
+  background: var(--unmoveable);
+}
+
+.unmoveable::after {
+  box-shadow:
+    inset 1px 1px 0 0 var(--unmoveable-light),
+    inset -1px -1px 0 0 var(--unmoveable-dark);
+}
+
+.moving {
+  background: var(--moving);
+  animation: blink 240ms steps(2, start) infinite;
+}
+
+.moving::after {
+  box-shadow:
+    inset 1px 1px 0 0 var(--moving-light),
+    inset -1px -1px 0 0 var(--moving-dark);
+}
+
+@keyframes blink {
+  0%,
+  49% {
+    filter: brightness(1.18);
+  }
+
+  50%,
+  100% {
+    filter: brightness(0.88);
+  }
+}
+
+/* ---------- Legend ---------- */
+.legend {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px 10px;
+  font-size: 10px;
+  padding: 2px;
+  color: var(--mb-color-text-secondary);
+}
+
+.legendItem {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+}
+
+.legendSwatch {
+  display: inline-block;
+  width: 10px;
+  height: 10px;
+  position: relative;
+  border-top: 1px solid var(--chrome-darker);
+  border-left: 1px solid var(--chrome-darker);
+  border-right: 1px solid var(--chrome-highlight);
+  border-bottom: 1px solid var(--chrome-highlight);
+}
+
+.legendLabel {
+  font-family: inherit;
+  color: var(--mb-color-text-primary);
+}
+
+/* ---------- Status bar ---------- */
+.statusBar {
+  display: flex;
+  gap: 6px;
+  align-items: center;
+  background: var(--chrome);
+  padding: 2px 4px;
+  font-family: "Courier New", "Lucida Console", monospace;
+  font-size: 10px;
+  border-top: 1px solid var(--chrome-highlight);
+  border-left: 1px solid var(--chrome-highlight);
+  border-right: 1px solid var(--chrome-darker);
+  border-bottom: 1px solid var(--chrome-darker);
+  color: var(--mb-color-text-primary);
+}
+
+.statusText,
+.statusTextRight {
+  background: var(--chrome);
+  padding: 1px 4px;
+  border-top: 1px solid var(--chrome-darker);
+  border-left: 1px solid var(--chrome-darker);
+  border-right: 1px solid var(--chrome-highlight);
+  border-bottom: 1px solid var(--chrome-highlight);
+  letter-spacing: 0.04em;
+}
+
+.statusTextRight {
+  margin-left: auto;
+  min-width: 36px;
+  text-align: right;
+  color: var(--mb-color-brand);
+  font-weight: bold;
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/DefragLoader/DefragLoader.tsx

@@ -0,0 +1,220 @@
+import { useEffect, useMemo, useRef, useState } from "react";
+import { c, t } from "ttag";
+
+import { Box } from "metabase/ui";
+
+import S from "./DefragLoader.module.css";
+
+/**
+ * 90s-defrag-style loader. Renders a grid of "disk clusters" that
+ * compact themselves leftward over time, the way Windows 9x Disk
+ * Defragmenter visualised cluster movement:
+ *
+ *   white   = empty cluster
+ *   blue    = used data
+ *   yellow  = system file (immovable)
+ *   red     = unmoveable
+ *   green   = currently moving
+ *
+ * The animation loops: compact → small pause → re-fragment → compact.
+ * Used as the "Analyzing…" indicator while the optimizer stream runs.
+ */
+
+type Cell = "free" | "data" | "system" | "unmoveable" | "moving";
+
+const COLS = 28;
+const ROWS = 8;
+const TOTAL = COLS * ROWS;
+const TICK_MS = 70;
+const PAUSE_TICKS = 18;
+
+type Phase = "compacting" | "paused";
+
+type GridState = {
+  grid: Cell[];
+  phase: Phase;
+  pauseLeft: number;
+  moves: number;
+  passes: number;
+};
+
+const CELL_CLASS: Record<Cell, string> = {
+  free: S.free,
+  data: S.data,
+  system: S.system,
+  unmoveable: S.unmoveable,
+  moving: S.moving,
+};
+
+export function DefragLoader() {
+  const [state, setState] = useState<GridState>(() => initState());
+  // Stable cell array so React doesn't reconcile the whole grid every tick.
+  const cellsArr = useMemo(
+    () => Array.from({ length: TOTAL }, (_, i) => i),
+    [],
+  );
+  const tickRef = useRef<number>(0);
+
+  useEffect(() => {
+    const id = window.setInterval(() => {
+      tickRef.current += 1;
+      setState(advance);
+    }, TICK_MS);
+    return () => window.clearInterval(id);
+  }, []);
+
+  const headIdx = Math.max(
+    0,
+    state.grid.findIndex((cell) => cell === "free"),
+  );
+  const compacted = headIdx < 0 ? TOTAL : headIdx;
+  const percent = Math.min(99, Math.round((compacted / TOTAL) * 100));
+
+  return (
+    <Box className={S.window} role="img">
+      <div className={S.titleBar}>
+        <div className={S.titleIcon} aria-hidden>
+          <span className={S.titleIconInner}>T</span>
+        </div>
+        <div
+          className={S.titleText}
+        >{t`Transform Optimizer — analyzing opportunities...`}</div>
+      </div>
+
+      <div className={S.body}>
+        <div className={S.gridFrame}>
+          <div
+            className={S.grid}
+            style={{
+              gridTemplateColumns: `repeat(${COLS}, 1fr)`,
+              gridTemplateRows: `repeat(${ROWS}, 1fr)`,
+            }}
+          >
+            {cellsArr.map((i) => (
+              <span
+                key={i}
+                className={`${S.cell} ${CELL_CLASS[state.grid[i] ?? "free"]}`}
+              />
+            ))}
+          </div>
+        </div>
+
+        <div className={S.legend} aria-hidden>
+          <LegendSwatch swatchClass={S.data} label={t`Rewrite`} />
+          <LegendSwatch swatchClass={S.system} label={t`Index`} />
+          <LegendSwatch swatchClass={S.unmoveable} label={t`Risk`} />
+          <LegendSwatch swatchClass={S.moving} label={t`Precompute`} />
+        </div>
+
+        <div className={S.statusBar}>
+          <span className={S.statusText}>
+            {c("Disk-defrag-style status line")
+              .t`Cluster ${pad(compacted, 4)} / ${pad(TOTAL, 4)}`}
+          </span>
+          <span className={S.statusText}>
+            {c("Disk-defrag-style status line — moves performed")
+              .t`${pad(state.moves, 4)} moves`}
+          </span>
+          <span className={S.statusTextRight}>{`${pad(percent, 2)}%`}</span>
+        </div>
+      </div>
+    </Box>
+  );
+}
+
+function LegendSwatch({
+  swatchClass,
+  label,
+}: {
+  swatchClass: string;
+  label: string;
+}) {
+  return (
+    <span className={S.legendItem}>
+      <span className={`${S.legendSwatch} ${swatchClass}`} />
+      <span className={S.legendLabel}>{label}</span>
+    </span>
+  );
+}
+
+// ---------------------------------------------------------------------------
+// State machine
+
+function initState(): GridState {
+  return {
+    grid: makeFragmentedGrid(),
+    phase: "compacting",
+    pauseLeft: 0,
+    moves: 0,
+    passes: 0,
+  };
+}
+
+function advance(prev: GridState): GridState {
+  if (prev.phase === "paused") {
+    const left = prev.pauseLeft - 1;
+    if (left <= 0) {
+      return {
+        grid: makeFragmentedGrid(),
+        phase: "compacting",
+        pauseLeft: 0,
+        moves: 0,
+        passes: prev.passes + 1,
+      };
+    }
+    return { ...prev, pauseLeft: left };
+  }
+
+  // Phase: compacting. If a "moving" cell exists, swap it with the earliest
+  // free cell. Otherwise pick the next out-of-place data cell to flag as
+  // moving.
+  const movingIdx = prev.grid.indexOf("moving");
+  if (movingIdx >= 0) {
+    const freeIdx = prev.grid.indexOf("free");
+    if (freeIdx >= 0 && freeIdx < movingIdx) {
+      const next = prev.grid.slice();
+      next[freeIdx] = "data";
+      next[movingIdx] = "free";
+      return { ...prev, grid: next, moves: prev.moves + 1 };
+    }
+    // Defensive: turn the orphan move back into data.
+    const next = prev.grid.slice();
+    next[movingIdx] = "data";
+    return { ...prev, grid: next };
+  }
+
+  const freeIdx = prev.grid.indexOf("free");
+  if (freeIdx < 0) {
+    return { ...prev, phase: "paused", pauseLeft: PAUSE_TICKS };
+  }
+  for (let i = freeIdx + 1; i < prev.grid.length; i++) {
+    if (prev.grid[i] === "data") {
+      const next = prev.grid.slice();
+      next[i] = "moving";
+      return { ...prev, grid: next };
+    }
+  }
+  // No reachable data cell after the head → compaction complete.
+  return { ...prev, phase: "paused", pauseLeft: PAUSE_TICKS };
+}
+
+function makeFragmentedGrid(): Cell[] {
+  const out: Cell[] = new Array(TOTAL);
+  for (let i = 0; i < TOTAL; i++) {
+    const r = Math.random();
+    if (r < 0.025) {
+      out[i] = "unmoveable";
+    } else if (r < 0.075) {
+      out[i] = "system";
+    } else if (r < 0.7) {
+      out[i] = "data";
+    } else {
+      out[i] = "free";
+    }
+  }
+  return out;
+}
+
+function pad(n: number, width: number): string {
+  return n.toString().padStart(width, "0");
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/DefragLoader/index.ts

@@ -0,0 +1 @@
+export { DefragLoader } from "./DefragLoader";

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/FindSlowTool/FindSlowTool.tsx

@@ -0,0 +1,156 @@
+import { useDisclosure } from "@mantine/hooks";
+import { type FormEvent, useState } from "react";
+import { c, t } from "ttag";
+
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import {
+  Box,
+  Button,
+  Divider,
+  Group,
+  Icon,
+  NumberInput,
+  Popover,
+  Stack,
+  Switch,
+  Text,
+} from "metabase/ui";
+import type { FindSlowToolProps } from "metabase/plugins/oss/transform-optimizer";
+
+import { useBulkOptimizeMutation } from "../../api";
+import { BulkResultsDrawer } from "../BulkResultsDrawer";
+
+const POPOVER_MIN_WIDTH = 320;
+
+export function FindSlowTool({
+  thresholdSec,
+  onThresholdChange,
+  includeTimedOut,
+  onIncludeTimedOutChange,
+  matchingTransformIds,
+}: FindSlowToolProps) {
+  const [isOpened, { toggle, close }] = useDisclosure();
+  const [isDrawerOpened, { open: openDrawer, close: closeDrawer }] =
+    useDisclosure();
+  const [bulkOptimize, { isLoading }] = useBulkOptimizeMutation();
+  const { sendErrorToast } = useMetadataToasts();
+  const [draftSec, setDraftSec] = useState<number | "">(thresholdSec ?? "");
+
+  const handleApply = (event: FormEvent) => {
+    event.preventDefault();
+    const next = typeof draftSec === "number" && draftSec > 0 ? draftSec : undefined;
+    onThresholdChange(next);
+  };
+
+  const handleClear = () => {
+    setDraftSec("");
+    onThresholdChange(undefined);
+    close();
+  };
+
+  const handleHammerTime = async () => {
+    if (matchingTransformIds.length === 0) {
+      return;
+    }
+    const { data, error } = await bulkOptimize({
+      transformIds: matchingTransformIds,
+    });
+    if (error || !data) {
+      sendErrorToast(t`Failed to start bulk optimize`);
+      return;
+    }
+    close();
+    // The drawer polls /bulk-optimize/status from here — rows appear as
+    // each transform's analysis finishes; users can click through to
+    // verify / accept on the detail page (proposals are server-cached).
+    openDrawer();
+  };
+
+  const matchCount = matchingTransformIds.length;
+  const hasThreshold = thresholdSec != null;
+
+  return (
+    <>
+    <Popover opened={isOpened} position="bottom-end" onDismiss={close}>
+      <Popover.Target>
+        <Button
+          variant={hasThreshold ? "filled" : "default"}
+          leftSection={<Icon name="clock" />}
+          onClick={toggle}
+        >
+          {hasThreshold
+            ? t`Slow: ≥ ${thresholdSec}s (${matchCount})`
+            : t`Find slow`}
+        </Button>
+      </Popover.Target>
+      <Popover.Dropdown>
+        <Box component="form" miw={POPOVER_MIN_WIDTH} onSubmit={handleApply}>
+          <Stack p="md" gap="sm">
+            <Text fw="bold">{t`Find slow transforms`}</Text>
+            <Text c="text-secondary" fz="sm">
+              {t`Filter the list to transforms whose last successful run took at least this long.`}
+            </Text>
+            <NumberInput
+              label={t`Threshold (seconds)`}
+              placeholder="60"
+              value={draftSec}
+              min={1}
+              step={1}
+              allowDecimal={false}
+              allowNegative={false}
+              autoFocus
+              onChange={(v) => setDraftSec(typeof v === "number" ? v : "")}
+            />
+            <Switch
+              label={t`Include timed-out runs`}
+              description={t`Show transforms whose last run hit the timeout, regardless of threshold.`}
+              checked={includeTimedOut}
+              onChange={(event) =>
+                onIncludeTimedOutChange(event.currentTarget.checked)
+              }
+            />
+            {hasThreshold && (
+              <Text c="text-secondary" fz="sm">
+                {matchCount === 0
+                  ? t`No transforms match the current threshold.`
+                  : c("Match count under the slow-transforms filter")
+                      .t`${matchCount} matching transform(s)`}
+              </Text>
+            )}
+          </Stack>
+          <Divider />
+          <Group p="md" justify="space-between">
+            <Button
+              variant="subtle"
+              disabled={!hasThreshold && draftSec === ""}
+              onClick={handleClear}
+            >
+              {t`Clear`}
+            </Button>
+            <Group gap="sm">
+              <Button
+                type="submit"
+                variant={hasThreshold ? "default" : "filled"}
+                disabled={draftSec === "" || draftSec === thresholdSec}
+              >
+                {hasThreshold ? t`Update filter` : t`Apply filter`}
+              </Button>
+              <Button
+                color="warning"
+                variant="filled"
+                leftSection={<Icon name="bolt" />}
+                disabled={!hasThreshold || matchCount === 0 || isLoading}
+                loading={isLoading}
+                onClick={handleHammerTime}
+              >
+                {t`Hammer time`}
+              </Button>
+            </Group>
+          </Group>
+        </Box>
+      </Popover.Dropdown>
+    </Popover>
+    <BulkResultsDrawer opened={isDrawerOpened} onClose={closeDrawer} />
+    </>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/FindSlowTool/index.ts

@@ -0,0 +1 @@
+export { FindSlowTool } from "./FindSlowTool";

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/OptimizationDegreeDial/OptimizationDegreeDial.module.css

@@ -0,0 +1,49 @@
+.dialWrap {
+  position: relative;
+  flex-shrink: 0;
+}
+
+.score {
+  position: absolute;
+  inset: 0;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  pointer-events: none;
+}
+
+.track {
+  stroke: var(--mb-color-border);
+}
+
+.green {
+  stroke: var(--mb-color-success);
+  transition: stroke-dashoffset 400ms ease-out;
+}
+
+.greenYellow {
+  stroke: color-mix(
+    in srgb,
+    var(--mb-color-success) 60%,
+    var(--mb-color-warning)
+  );
+  transition: stroke-dashoffset 400ms ease-out;
+}
+
+.orange {
+  stroke: var(--mb-color-warning);
+  transition: stroke-dashoffset 400ms ease-out;
+}
+
+.red {
+  stroke: var(--mb-color-error);
+  transition: stroke-dashoffset 400ms ease-out;
+}
+
+.errorChip {
+  width: 12px;
+  height: 12px;
+  border-radius: 50%;
+  background: var(--mb-color-error);
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/OptimizationDegreeDial/OptimizationDegreeDial.tsx

@@ -0,0 +1,147 @@
+import { c, t } from "ttag";
+
+import { Box, Group, Stack, Text } from "metabase/ui";
+
+import { DefragLoader } from "../DefragLoader";
+
+import S from "./OptimizationDegreeDial.module.css";
+
+export type DialStatus = "streaming" | "done" | "error" | "idle";
+
+type Props = {
+  status: DialStatus;
+  score: number | null;
+  errorMessage?: string | null;
+};
+
+const SIZE = 96;
+const STROKE = 10;
+const RADIUS = (SIZE - STROKE) / 2;
+const CIRCUMFERENCE = 2 * Math.PI * RADIUS;
+
+export function OptimizationDegreeDial({ status, score, errorMessage }: Props) {
+  if (status === "streaming" && score == null) {
+    return <DefragLoader />;
+  }
+
+  if (status === "error") {
+    return (
+      <Group gap="md" align="center">
+        <Box className={S.errorChip} aria-hidden />
+        <Stack gap={2}>
+          <Text fw="bold" c="error">{t`Couldn't analyze transform`}</Text>
+          {errorMessage && <Text c="text-secondary">{errorMessage}</Text>}
+        </Stack>
+      </Group>
+    );
+  }
+
+  const value = clampScore(score);
+  const bucket = bucketFor(value);
+  // The dial fills proportionally to how *optimized* the query is, so a
+  // green sliver matches a healthy score.
+  const dashOffset = CIRCUMFERENCE * (1 - value / 100);
+
+  return (
+    <Group
+      gap="md"
+      align="center"
+      role="img"
+      aria-label={c("Optimization-degree dial label")
+        .t`Optimization score ${value} out of 100`}
+    >
+      <Box className={S.dialWrap} style={{ width: SIZE, height: SIZE }}>
+        <svg
+          width={SIZE}
+          height={SIZE}
+          viewBox={`0 0 ${SIZE} ${SIZE}`}
+          aria-hidden
+        >
+          <circle
+            cx={SIZE / 2}
+            cy={SIZE / 2}
+            r={RADIUS}
+            fill="none"
+            className={S.track}
+            strokeWidth={STROKE}
+          />
+          <circle
+            cx={SIZE / 2}
+            cy={SIZE / 2}
+            r={RADIUS}
+            fill="none"
+            strokeWidth={STROKE}
+            strokeLinecap="round"
+            strokeDasharray={CIRCUMFERENCE}
+            strokeDashoffset={dashOffset}
+            transform={`rotate(-90 ${SIZE / 2} ${SIZE / 2})`}
+            className={S[bucket]}
+          />
+        </svg>
+        <Box className={S.score}>
+          <Text fz="xl" fw="bold" lh={1}>
+            {value}
+          </Text>
+          <Text fz="xs" c="text-secondary">
+            {c("Denominator for the optimization-degree score").t`/ 100`}
+          </Text>
+        </Box>
+      </Box>
+      <Stack gap={2}>
+        <Text fw="bold">{bucketLabel(bucket)}</Text>
+        <Text c="text-secondary">{bucketHint(bucket, status)}</Text>
+      </Stack>
+    </Group>
+  );
+}
+
+function clampScore(score: number | null): number {
+  if (score == null || !Number.isFinite(score)) {
+    return 0;
+  }
+  return Math.max(0, Math.min(100, Math.round(score)));
+}
+
+type Bucket = "green" | "greenYellow" | "orange" | "red";
+
+function bucketFor(score: number): Bucket {
+  if (score >= 100) {
+    return "green";
+  }
+  if (score >= 70) {
+    return "greenYellow";
+  }
+  if (score >= 40) {
+    return "orange";
+  }
+  return "red";
+}
+
+function bucketLabel(bucket: Bucket): string {
+  switch (bucket) {
+    case "green":
+      return t`Already optimized`;
+    case "greenYellow":
+      return t`Mostly optimized`;
+    case "orange":
+      return t`Optimization wins available`;
+    case "red":
+      return t`Multiple high-severity issues`;
+  }
+}
+
+function bucketHint(bucket: Bucket, status: DialStatus): string {
+  if (status === "streaming") {
+    return t`Streaming proposals…`;
+  }
+  switch (bucket) {
+    case "green":
+      return t`No further changes to suggest.`;
+    case "greenYellow":
+      return t`Minor rewrites or indexes may help.`;
+    case "orange":
+      return t`Real speedups available — review the proposals below.`;
+    case "red":
+      return t`Multiple high-severity issues — review the proposals below.`;
+  }
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/OptimizationDegreeDial/index.ts

@@ -0,0 +1,4 @@
+export {
+  OptimizationDegreeDial,
+  type DialStatus,
+} from "./OptimizationDegreeDial";

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/ProposalCard/IndexChangesSection.module.css

@@ -0,0 +1,36 @@
+.row {
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-sm);
+  padding: var(--mantine-spacing-sm);
+  background: var(--mb-color-background-secondary);
+}
+
+.codeBlock {
+  display: block;
+  width: 100%;
+  min-width: 0;
+  max-width: 100%;
+  font-size: 12px;
+  white-space: pre;
+  overflow: auto;
+}
+
+.status_pending {
+  color: var(--mb-color-text-secondary);
+}
+
+.status_running {
+  color: var(--mb-color-brand);
+}
+
+.status_executed {
+  color: var(--mb-color-success);
+}
+
+.status_failed {
+  color: var(--mb-color-error);
+}
+
+.status_skipped {
+  color: var(--mb-color-text-secondary);
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/ProposalCard/IndexChangesSection.tsx

@@ -0,0 +1,60 @@
+import { c, t } from "ttag";
+
+import { Box, Code, Stack, Text } from "metabase/ui";
+
+import type { DdlStatement, DdlTarget } from "../../types";
+
+import S from "./IndexChangesSection.module.css";
+
+type Props = {
+  /** One DDL per `:index` proposal in this branch. */
+  statements: DdlStatement[];
+};
+
+/**
+ * Render the DDL block on a proposal card. With one-change-per-proposal
+ * the proposal's `rationale` already explains *why* the index helps, so
+ * we deliberately do NOT also render `ddl_statement.rationale` — that
+ * duplication is the recent complaint. We show only what's not on the
+ * parent card already: the target hint and the SQL statement itself.
+ * Rejected DDL keeps a one-line failure note.
+ */
+export function IndexChangesSection({ statements }: Props) {
+  return (
+    <Stack gap="sm">
+      {statements.map((statement, i) => (
+        <DdlRow
+          key={`${statement.index_name ?? "ddl"}-${i}`}
+          statement={statement}
+        />
+      ))}
+    </Stack>
+  );
+}
+
+function DdlRow({ statement }: { statement: DdlStatement }) {
+  return (
+    <Box className={S.row}>
+      <Text fz="xs" c="text-secondary" mb={4}>
+        {targetLabel(statement.target)}
+      </Text>
+      <Code block className={S.codeBlock}>
+        {statement.statement}
+      </Code>
+      {statement.validation === "rejected" && statement.rejection && (
+        <Text fz="xs" c="error" mt={4}>
+          {c("Reason an LLM-emitted DDL was rejected by the server validator")
+            .t`Rejected: ${statement.rejection.reason}`}
+          {statement.rejection.detail ? ` — ${statement.rejection.detail}` : ""}
+        </Text>
+      )}
+    </Box>
+  );
+}
+
+function targetLabel(target: DdlTarget): string {
+  if (target === "source-db") {
+    return t`On source database`;
+  }
+  return t`On a new transform target (created by a linked proposal)`;
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/ProposalCard/ProposalCard.module.css

@@ -0,0 +1,45 @@
+.card {
+  width: 100%;
+  /* Let the card shrink inside a flex/grid parent so its child SQL
+     blocks (white-space: pre) can clip+scroll instead of dragging the
+     whole card past the container edge. */
+  min-width: 0;
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-md);
+  background: var(--mb-color-background-primary);
+}
+
+.codeBlock {
+  display: block;
+  width: 100%;
+  min-width: 0;
+  max-height: 320px;
+  overflow: auto;
+  font-size: 12px;
+  white-space: pre;
+}
+
+.kindTag {
+  text-transform: uppercase;
+  font-size: 10px;
+  letter-spacing: 0.04em;
+}
+
+.severity_high {
+  background: color-mix(in srgb, var(--mb-color-error) 12%, transparent);
+  color: var(--mb-color-error);
+}
+
+.severity_medium {
+  background: color-mix(in srgb, var(--mb-color-warning) 14%, transparent);
+  color: var(--mb-color-warning);
+}
+
+.severity_low {
+  background: color-mix(
+    in srgb,
+    var(--mb-color-text-secondary) 12%,
+    transparent
+  );
+  color: var(--mb-color-text-secondary);
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/ProposalCard/ProposalCard.tsx

@@ -0,0 +1,301 @@
+import { c, t } from "ttag";
+
+import {
+  Badge,
+  Box,
+  Button,
+  Code,
+  Divider,
+  Group,
+  Icon,
+  Menu,
+  Stack,
+  Text,
+  Tooltip,
+} from "metabase/ui";
+
+import type { AcceptMode } from "../../api";
+import type { DdlStatement, Proposal, ProposalSeverity } from "../../types";
+
+import { IndexChangesSection } from "./IndexChangesSection";
+import S from "./ProposalCard.module.css";
+import { SqlDiff } from "./SqlDiff";
+
+type Action =
+  | {
+      kind: "accept";
+      busy?: boolean;
+      disabled?: boolean;
+      disabledReason?: string;
+    }
+  | {
+      kind: "verify";
+      busy?: boolean;
+      disabled?: boolean;
+      disabledReason?: string;
+    }
+  | { kind: "dismiss" };
+
+type Props = {
+  proposal: Proposal;
+  /**
+   * Current SQL of the transform being optimised. When present and the
+   * proposal has a `body`, the card renders a line-diff instead of the
+   * raw proposed SQL.
+   */
+  currentSql?: string | null;
+  /**
+   * Resolved display names for each id in `proposal.depends_on`. Rendered
+   * as a small "Depends on" line so the user can see the materialisation
+   * order linkage between proposals. The parent passes this in because
+   * the card itself doesn't have access to the full proposal set.
+   */
+  dependencyNames?: Array<{ id: string; name: string }>;
+  actions: {
+    accept?: Action & { kind: "accept" };
+    verify?: Action & { kind: "verify" };
+    dismiss?: Action & { kind: "dismiss" };
+  };
+  /**
+   * Invoked when the user accepts the proposal. The `mode` argument
+   * controls how the BE materialises it:
+   *   "new"      — create a sibling transform (the default)
+   *   "replace"  — update the original transform's source in place
+   *                (only offered for `kind: "rewrite"`)
+   */
+  onAccept?: (mode: AcceptMode) => void;
+  onVerify?: () => void;
+  onDismiss?: () => void;
+};
+
+export function ProposalCard({
+  proposal,
+  currentSql,
+  dependencyNames,
+  actions,
+  onAccept,
+  onVerify,
+  onDismiss,
+}: Props) {
+  return (
+    <Box className={S.card} aria-label={proposal.name}>
+      <Stack gap="sm" p="md">
+        <Group justify="space-between" align="flex-start" wrap="nowrap">
+          <Stack gap={4} miw={0}>
+            <Group gap="xs">
+              <SeverityBadge severity={proposal.severity} />
+              <KindTag kind={proposal.kind} />
+              <Text c="text-secondary" fz="sm">
+                {c("Expected speedup, e.g. ≥100×")
+                  .t`Expected speedup ${proposal.expected_speedup}`}
+              </Text>
+            </Group>
+            <Text fw="bold" fz="md">
+              {proposal.name}
+            </Text>
+            <Text c="text-secondary">{proposal.rationale}</Text>
+            {dependencyNames && dependencyNames.length > 0 && (
+              <Group gap={4} wrap="wrap" mt={4}>
+                <Text c="text-secondary" fz="xs">
+                  {t`Depends on:`}
+                </Text>
+                {dependencyNames.map((dep) => (
+                  <Badge key={dep.id} variant="default" size="sm" radius="sm">
+                    {dep.name}
+                  </Badge>
+                ))}
+              </Group>
+            )}
+          </Stack>
+        </Group>
+
+        {proposal.body && (
+          <Box>
+            <Text fz="sm" c="text-secondary" mb={4}>
+              {currentSql ? t`Proposed changes` : t`Proposed transform`}
+            </Text>
+            {currentSql ? (
+              <SqlDiff before={currentSql} after={proposal.body} />
+            ) : (
+              <Code block className={S.codeBlock}>
+                {proposal.body}
+              </Code>
+            )}
+          </Box>
+        )}
+      </Stack>
+
+      {proposal.ddl_statement && (
+        <>
+          <Divider />
+          <Box p="md">
+            <IndexChangesSection statements={[proposal.ddl_statement]} />
+          </Box>
+        </>
+      )}
+
+      <Divider />
+      <Group justify="flex-end" gap="sm" p="md">
+        <DismissButton action={actions.dismiss} onClick={onDismiss} />
+        <VerifyButton action={actions.verify} onClick={onVerify} />
+        <AcceptButton
+          action={actions.accept}
+          // Only `:rewrite` supports replace-in-place — precompute DAGs
+          // can't be folded back into a single transform, and indices
+          // don't change the source body.
+          allowReplace={proposal.kind === "rewrite"}
+          onClick={onAccept}
+        />
+      </Group>
+    </Box>
+  );
+}
+
+function SeverityBadge({ severity }: { severity: ProposalSeverity }) {
+  return (
+    <Badge className={S[`severity_${severity}`]} variant="light" radius="sm">
+      {severityLabel(severity)}
+    </Badge>
+  );
+}
+
+function severityLabel(severity: ProposalSeverity): string {
+  switch (severity) {
+    case "high":
+      return t`High impact`;
+    case "medium":
+      return t`Medium impact`;
+    case "low":
+      return t`Low impact`;
+  }
+}
+
+function KindTag({ kind }: { kind: Proposal["kind"] }) {
+  return (
+    <Badge variant="outline" radius="sm" className={S.kindTag}>
+      {kindLabel(kind)}
+    </Badge>
+  );
+}
+
+function kindLabel(kind: Proposal["kind"]): string {
+  switch (kind) {
+    case "rewrite":
+      return t`Rewrite`;
+    case "index":
+      return t`Index`;
+    case "precompute":
+      return t`Precompute`;
+  }
+}
+
+function AcceptButton({
+  action,
+  allowReplace,
+  onClick,
+}: {
+  action?: Action & { kind: "accept" };
+  /** When true, render a split menu offering replace-in-place too. */
+  allowReplace?: boolean;
+  onClick?: (mode: AcceptMode) => void;
+}) {
+  // Show the control iff the parent supplied an action entry. The
+  // callback is always non-null because TransformOptimizerSection wires
+  // them up unconditionally, so don't gate on it.
+  if (!action || !onClick) {
+    return null;
+  }
+  if (!allowReplace) {
+    const button = (
+      <Button
+        variant="filled"
+        loading={action.busy}
+        disabled={action.disabled}
+        onClick={() => onClick("new")}
+      >
+        {t`Accept`}
+      </Button>
+    );
+    return action.disabled && action.disabledReason ? (
+      <Tooltip label={action.disabledReason}>{button}</Tooltip>
+    ) : (
+      button
+    );
+  }
+  // Rewrite proposals get a menu so the user picks `new` vs `replace`.
+  const trigger = (
+    <Button
+      variant="filled"
+      loading={action.busy}
+      disabled={action.disabled}
+      rightSection={<Icon name="chevrondown" />}
+    >
+      {t`Accept`}
+    </Button>
+  );
+  return (
+    <Menu shadow="md" disabled={action.disabled}>
+      <Menu.Target>
+        {action.disabled && action.disabledReason ? (
+          <Tooltip label={action.disabledReason}>{trigger}</Tooltip>
+        ) : (
+          trigger
+        )}
+      </Menu.Target>
+      <Menu.Dropdown>
+        <Menu.Item onClick={() => onClick("new")}>
+          {t`Create as a new transform`}
+        </Menu.Item>
+        <Menu.Item onClick={() => onClick("replace")}>
+          {t`Replace this transform's source`}
+        </Menu.Item>
+      </Menu.Dropdown>
+    </Menu>
+  );
+}
+
+function VerifyButton({
+  action,
+  onClick,
+}: {
+  action?: Action & { kind: "verify" };
+  onClick?: () => void;
+}) {
+  if (!action || !onClick) {
+    return null;
+  }
+  const button = (
+    <Button
+      variant="default"
+      loading={action.busy}
+      disabled={action.disabled}
+      onClick={onClick}
+    >
+      {t`Verify`}
+    </Button>
+  );
+  if (action.disabled && action.disabledReason) {
+    return <Tooltip label={action.disabledReason}>{button}</Tooltip>;
+  }
+  return button;
+}
+
+function DismissButton({
+  action,
+  onClick,
+}: {
+  action?: Action & { kind: "dismiss" };
+  onClick?: () => void;
+}) {
+  if (!action || !onClick) {
+    return null;
+  }
+  return (
+    <Button variant="subtle" onClick={onClick}>
+      {t`Dismiss`}
+    </Button>
+  );
+}
+
+// Re-exports so consumers can pass typed action info from the parent.
+export type { DdlStatement };

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/ProposalCard/SqlDiff.module.css

@@ -0,0 +1,46 @@
+.wrap {
+  /* The diff content is white-space: pre — without these the long lines
+     push the card past its container instead of producing a horizontal
+     scrollbar inside the diff. */
+  width: 100%;
+  min-width: 0;
+  max-width: 100%;
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-sm);
+  background: var(--mb-color-background-secondary);
+  max-height: 360px;
+  overflow: auto;
+}
+
+.pre {
+  margin: 0;
+  padding: var(--mantine-spacing-sm);
+  font-family: var(--mb-default-monospace-font-family, monospace);
+  font-size: 12px;
+  line-height: 1.4;
+  white-space: pre;
+  tab-size: 2;
+}
+
+.context {
+  color: var(--mb-color-text-primary);
+  display: inline;
+}
+
+.added {
+  display: inline;
+  background: color-mix(in srgb, var(--mb-color-success) 18%, transparent);
+  color: var(--mb-color-success);
+}
+
+.removed {
+  display: inline;
+  background: color-mix(in srgb, var(--mb-color-error) 14%, transparent);
+  color: var(--mb-color-error);
+  text-decoration: line-through;
+  text-decoration-color: color-mix(
+    in srgb,
+    var(--mb-color-error) 60%,
+    transparent
+  );
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/ProposalCard/SqlDiff.tsx

@@ -0,0 +1,71 @@
+import { diffLines } from "diff";
+import { useMemo } from "react";
+
+import { Box } from "metabase/ui";
+
+import S from "./SqlDiff.module.css";
+
+type Props = {
+  /** SQL of the existing transform (current state). */
+  before: string;
+  /** SQL of the proposed transform (after applying the suggestion). */
+  after: string;
+};
+
+type Segment = {
+  kind: "context" | "added" | "removed";
+  text: string;
+};
+
+/**
+ * Unified line-by-line diff of two SQL bodies. We deliberately use
+ * `diffLines` rather than word/char diff: SQL is line-oriented and
+ * mixed-granularity diffs render as noise.
+ */
+export function SqlDiff({ before, after }: Props) {
+  const segments = useMemo(() => buildSegments(before, after), [before, after]);
+
+  return (
+    <Box className={S.wrap} role="figure" aria-label="SQL diff">
+      <pre className={S.pre}>
+        {segments.map((seg, idx) => (
+          <span key={idx} className={S[seg.kind]}>
+            {prefixLines(seg.text, seg.kind)}
+          </span>
+        ))}
+      </pre>
+    </Box>
+  );
+}
+
+function buildSegments(before: string, after: string): Segment[] {
+  // Normalise trailing whitespace per line so cosmetic-only diffs don't render.
+  const left = normalise(before);
+  const right = normalise(after);
+  return diffLines(left, right).map(
+    (part): Segment => ({
+      kind: part.added ? "added" : part.removed ? "removed" : "context",
+      text: part.value,
+    }),
+  );
+}
+
+function normalise(sql: string): string {
+  return sql
+    .split(/\r?\n/)
+    .map((line) => line.replace(/\s+$/u, ""))
+    .join("\n");
+}
+
+function prefixLines(text: string, kind: Segment["kind"]): string {
+  const marker = kind === "added" ? "+ " : kind === "removed" ? "- " : "  ";
+  // Each chunk from diffLines ends in a newline; keep that newline and
+  // prefix every line *inside* the chunk.
+  const lines = text.split("\n");
+  // The last element after split is "" when text ends with "\n"; preserve it.
+  return lines
+    .map((line, i) =>
+      i === lines.length - 1 && line === "" ? "" : marker + line,
+    )
+    .join("\n");
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/ProposalCard/index.ts

@@ -0,0 +1,2 @@
+export { ProposalCard } from "./ProposalCard";
+export { IndexChangesSection } from "./IndexChangesSection";

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/TargetIndexesSection/TargetIndexesSection.module.css

@@ -0,0 +1,20 @@
+.row {
+  display: grid;
+  grid-template-columns: 1fr auto;
+  gap: 0.75rem;
+  align-items: start;
+  padding: 0.75rem;
+  border: 1px solid var(--mb-color-border);
+  border-radius: 8px;
+  background: var(--mb-color-background-primary);
+}
+
+.definition {
+  display: block;
+  font-family: var(--mb-default-monospace-font-family);
+  font-size: 0.85rem;
+  white-space: pre-wrap;
+  word-break: break-word;
+  color: var(--mb-color-text-secondary);
+  margin-top: 0.25rem;
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/TargetIndexesSection/TargetIndexesSection.tsx

@@ -0,0 +1,269 @@
+import { useCallback } from "react";
+import { t } from "ttag";
+
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import {
+  Alert,
+  Badge,
+  Box,
+  Button,
+  Group,
+  Icon,
+  Loader,
+  Stack,
+  Text,
+  Tooltip,
+} from "metabase/ui";
+import type { Transform } from "metabase-types/api";
+
+import {
+  useDropTargetIndexMutation,
+  useListTargetIndexesQuery,
+} from "../../api";
+import type { DropIndexResult, TargetIndex } from "../../types";
+
+import styles from "./TargetIndexesSection.module.css";
+
+type Props = {
+  transform: Transform;
+  /**
+   * When the user has no write permission on the source DB the drop
+   * action is hidden — the BE refuses too, but we don't even surface the
+   * button.
+   */
+  readOnly?: boolean;
+};
+
+export function TargetIndexesSection({ transform, readOnly }: Props) {
+  const { data, isLoading, error, refetch, isFetching } =
+    useListTargetIndexesQuery({ transformId: transform.id });
+  const [dropIndex, dropResult] = useDropTargetIndexMutation();
+  const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
+
+  const handleDrop = useCallback(
+    async (index: TargetIndex) => {
+      const { data: result, error: dropError } = await dropIndex({
+        transformId: transform.id,
+        indexName: index.name,
+      });
+      if (dropError) {
+        sendErrorToast(t`Failed to drop ${index.name}`);
+        return;
+      }
+      const r = result as DropIndexResult | undefined;
+      if (!r || r.status !== "dropped") {
+        sendErrorToast(formatDropFailure(index.name, r));
+        return;
+      }
+      sendSuccessToast(t`Dropped ${index.name}`);
+      refetch();
+    },
+    [dropIndex, transform.id, refetch, sendSuccessToast, sendErrorToast],
+  );
+
+  // Group indices by [schema, table]. Target table first (so the
+  // optimizer-managed indices land at the top), then source tables in
+  // resolution order — the BE returns them in that order already.
+  const groups = groupByTable(data?.indexes ?? []);
+
+  return (
+    <Stack gap="sm">
+      <Group justify="space-between" align="center">
+        <Box>
+          <Text fw="bold">{t`Indexes on referenced tables`}</Text>
+          <Text c="text-secondary" size="sm">
+            {t`Includes indexes on this transform's target table and the source tables it reads from.`}
+          </Text>
+        </Box>
+        <Button
+          variant="subtle"
+          size="xs"
+          leftSection={<Icon name="refresh" />}
+          onClick={() => refetch()}
+          disabled={isFetching}
+        >
+          {t`Refresh`}
+        </Button>
+      </Group>
+
+      {isLoading && (
+        <Group gap="xs">
+          <Loader size="xs" />
+          <Text c="text-secondary" size="sm">{t`Loading indexes…`}</Text>
+        </Group>
+      )}
+
+      {Boolean(error) && !isLoading && (
+        <Alert color="error" icon={<Icon name="warning" />}>
+          {t`Couldn't load indexes.`}
+        </Alert>
+      )}
+
+      {data && data.indexes.length === 0 && !isLoading && (
+        <Text c="text-secondary" size="sm">
+          {t`No indexes found on any referenced table yet.`}
+        </Text>
+      )}
+
+      {groups.length > 0 && (
+        <Stack gap="md">
+          {groups.map(({ schema, table, isTarget, rows }) => (
+            <Stack key={`${schema}.${table}`} gap="xs">
+              <Group gap="xs">
+                <Text fw="bold" size="sm" ff="monospace">
+                  {schema}.{table}
+                </Text>
+                {isTarget ? (
+                  <Badge color="brand" variant="light" size="sm">
+                    {t`Target`}
+                  </Badge>
+                ) : (
+                  <Badge variant="default" size="sm">
+                    {t`Source`}
+                  </Badge>
+                )}
+              </Group>
+              {rows.map((idx) => (
+                <IndexRow
+                  key={idx.name}
+                  index={idx}
+                  onDrop={handleDrop}
+                  busy={dropResult.isLoading}
+                  readOnly={readOnly}
+                />
+              ))}
+            </Stack>
+          ))}
+        </Stack>
+      )}
+    </Stack>
+  );
+}
+
+function groupByTable(indexes: TargetIndex[]) {
+  const groups: Array<{
+    schema: string;
+    table: string;
+    isTarget: boolean;
+    rows: TargetIndex[];
+  }> = [];
+  const byKey = new Map<string, (typeof groups)[number]>();
+  for (const idx of indexes) {
+    const key = `${idx.schema}.${idx.table}`;
+    let group = byKey.get(key);
+    if (!group) {
+      group = {
+        schema: idx.schema,
+        table: idx.table,
+        isTarget: idx.is_target_table,
+        rows: [],
+      };
+      byKey.set(key, group);
+      groups.push(group);
+    }
+    group.rows.push(idx);
+  }
+  return groups;
+}
+
+function IndexRow({
+  index,
+  onDrop,
+  busy,
+  readOnly,
+}: {
+  index: TargetIndex;
+  onDrop: (index: TargetIndex) => void;
+  busy: boolean;
+  readOnly?: boolean;
+}) {
+  // Primary keys are foundational — we don't even offer the option to
+  // drop them (would orphan FKs and uniqueness guarantees). For everything
+  // else, the button is enabled unless the user lacks DB write perms,
+  // in which case we render a disabled+tooltip variant.
+  const isPk = index.is_primary;
+  const canDrop = !isPk && !readOnly;
+  return (
+    <Box className={styles.row}>
+      <Box miw={0}>
+        <Group gap="xs" wrap="wrap">
+          <Text fw="bold" ff="monospace">
+            {index.name}
+          </Text>
+          <Badge variant="default" size="sm">
+            {index.access_method}
+          </Badge>
+          {index.is_primary && (
+            <Badge color="info" variant="light" size="sm">
+              {t`Primary key`}
+            </Badge>
+          )}
+          {!index.is_primary && index.is_unique && (
+            <Badge color="brand" variant="light" size="sm">
+              {t`Unique`}
+            </Badge>
+          )}
+          {index.managed_by_optimizer && (
+            <Badge color="success" variant="light" size="sm">
+              {t`Optimizer-managed`}
+            </Badge>
+          )}
+          {!index.is_valid && (
+            <Badge color="error" variant="light" size="sm">
+              {t`Invalid`}
+            </Badge>
+          )}
+        </Group>
+        <code className={styles.definition}>{index.definition}</code>
+      </Box>
+      <Group justify="flex-end" gap="xs" wrap="nowrap">
+        {isPk ? null : canDrop ? (
+          <Button
+            color="error"
+            variant="subtle"
+            size="xs"
+            disabled={busy}
+            onClick={() => onDrop(index)}
+          >
+            {t`Drop`}
+          </Button>
+        ) : (
+          <Tooltip label={t`You don't have permission to drop indexes here.`}>
+            <span>
+              <Button color="error" variant="subtle" size="xs" disabled>
+                {t`Drop`}
+              </Button>
+            </span>
+          </Tooltip>
+        )}
+      </Group>
+    </Box>
+  );
+}
+
+function formatDropFailure(
+  name: string,
+  result: DropIndexResult | undefined,
+): string {
+  if (!result) {
+    return t`Couldn't drop ${name}`;
+  }
+  if (result.status === "failed") {
+    return t`Couldn't drop ${name}: ${result.error_message}`;
+  }
+  if (result.status === "skipped") {
+    switch (result.reason) {
+      case "index-not-on-referenced-table":
+        return t`Index ${name} isn't on this transform's target or source tables.`;
+      case "unsafe-name":
+        return t`Refused to drop ${name}: name contains unsafe characters.`;
+      case "no-database":
+        return t`Couldn't drop ${name}: no database connection.`;
+      case "not-postgres":
+        return t`Dropping indexes is only supported on Postgres in this branch.`;
+      default:
+        return t`Couldn't drop ${name}`;
+    }
+  }
+  return t`Couldn't drop ${name}`;
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/TargetIndexesSection/index.ts

@@ -0,0 +1 @@
+export { TargetIndexesSection } from "./TargetIndexesSection";

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/TransformOptimizerSection/ProposalGroup.module.css

@@ -0,0 +1,57 @@
+.group {
+  /* Without these, a step's white-space: pre SQL drives the grid track
+     (and therefore .group) to min-content width, dragging the whole
+     pipeline past the page. */
+  min-width: 0;
+  max-width: 100%;
+  border: 1px solid var(--mb-color-border);
+  border-radius: 12px;
+  background: var(--mb-color-bg-light, var(--mb-color-bg-white));
+  padding: 0.75rem;
+}
+
+.header {
+  display: flex;
+  align-items: center;
+  gap: 0.5rem;
+  margin-bottom: 0.5rem;
+}
+
+.pipeline {
+  /* Each card in a multi-step pipeline gets a subtle vertical connector
+   * via this nested grid so the user reads top-to-bottom as one flow.
+   *
+   * `minmax(0, 1fr)` is the key bit: a default grid auto-track sizes to
+   * its widest child's min-content, which for white-space: pre SQL is
+   * the longest unbroken line. The 0 lower bound lets the track shrink
+   * below that and forces the inner code blocks to use their own
+   * overflow: auto instead. */
+  display: grid;
+  grid-template-columns: minmax(0, 1fr);
+  gap: 0.5rem;
+}
+
+.step {
+  position: relative;
+  /* Grid items default to min-width: auto (= content). Forcing 0 keeps
+     a wide card from blowing out the grid track even if the track were
+     auto-sized. Defense in depth alongside the minmax(0, 1fr) above. */
+  min-width: 0;
+}
+
+.step::before {
+  /* A 2px vertical line connecting consecutive cards, drawn only
+   * between steps (skipped on the last child). Sits in the gutter so it
+   * doesn't overlap any card content. */
+  content: "";
+  position: absolute;
+  left: -0.75rem;
+  top: 100%;
+  height: 0.5rem;
+  width: 2px;
+  background: var(--mb-color-border);
+}
+
+.step:last-child::before {
+  display: none;
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/TransformOptimizerSection/ProposalGroup.tsx

@@ -0,0 +1,73 @@
+import type { ReactNode } from "react";
+import { ngettext, msgid } from "ttag";
+
+import { Badge, Box, Group, Icon, Text } from "metabase/ui";
+
+import type { Proposal } from "../../types";
+
+import S from "./ProposalGroup.module.css";
+
+type Props = {
+  /** Members of one dependency component, root-first. */
+  proposals: Proposal[];
+  /**
+   * Card renderer. The group wraps each card in a positional `<Box>`
+   * so a vertical connector line can be drawn between consecutive
+   * steps. The parent supplies the actual `ProposalCard` so this
+   * component stays UI-only.
+   */
+  renderCard: (proposal: Proposal) => ReactNode;
+};
+
+/**
+ * One pipeline of dependent proposals — a precompute and the rewrite
+ * that reads from it, a rewrite and a supporting transform-target
+ * index, etc. Standalone proposals (1-member groups) render flat with
+ * no wrapper chrome so we don't add visual noise.
+ */
+export function ProposalGroup({ proposals, renderCard }: Props) {
+  if (proposals.length === 1) {
+    // Single-proposal "group": render the card directly. The user
+    // shouldn't pay any visual cost for a non-pipeline proposal.
+    return <>{renderCard(proposals[0])}</>;
+  }
+
+  const stepCount = proposals.length;
+  return (
+    <Box className={S.group} aria-label="Proposal pipeline">
+      <Group className={S.header} gap="xs">
+        <Icon name="link" />
+        <Text fw="bold" fz="sm">
+          {ngettext(
+            msgid`Pipeline (${stepCount} step)`,
+            `Pipeline (${stepCount} steps)`,
+            stepCount,
+          )}
+        </Text>
+        <Badge variant="default" size="sm" radius="sm">
+          {labelForLeaf(proposals[proposals.length - 1])}
+        </Badge>
+      </Group>
+      <Box className={S.pipeline}>
+        {proposals.map((proposal) => (
+          <Box key={proposal.id} className={S.step}>
+            {renderCard(proposal)}
+          </Box>
+        ))}
+      </Box>
+    </Box>
+  );
+}
+
+function labelForLeaf(leaf: Proposal): string {
+  // The leaf's kind tells the user what the *whole* pipeline ultimately
+  // does — a precompute-and-rewrite pipeline ends in a rewrite, etc.
+  switch (leaf.kind) {
+    case "rewrite":
+      return "ends in rewrite";
+    case "index":
+      return "ends in index";
+    case "precompute":
+      return "ends in precompute";
+  }
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/TransformOptimizerSection/SonicOptimizedBanner.tsx

@@ -0,0 +1,45 @@
+import { t } from "ttag";
+
+import { Box, Button, Group, Stack, Text } from "metabase/ui";
+
+// Drop the gif file at this path: resources/frontend_client/app/assets/img/sonic-gotta-go-fast.gif
+const SONIC_GIF_URL = "app/assets/img/sonic-gotta-go-fast.gif";
+
+type SonicOptimizedBannerProps = {
+  summary?: string;
+  onReanalyze: () => void;
+};
+
+export function SonicOptimizedBanner({
+  summary,
+  onReanalyze,
+}: SonicOptimizedBannerProps) {
+  return (
+    <Stack
+      align="center"
+      gap="sm"
+      p="lg"
+      bg="bg-light"
+      style={{ borderRadius: "var(--mantine-radius-md)" }}
+    >
+      <Box
+        component="img"
+        src={SONIC_GIF_URL}
+        alt={t`Sonic running fast — this transform is optimized`}
+        h={160}
+        style={{ objectFit: "contain" }}
+      />
+      <Text fw="bold" fz="lg">{t`This transform is fully optimized!`}</Text>
+      {summary && (
+        <Text c="text-secondary" ta="center">
+          {summary}
+        </Text>
+      )}
+      <Group justify="center" mt="sm">
+        <Button variant="subtle" onClick={onReanalyze}>
+          {t`Re-analyze`}
+        </Button>
+      </Group>
+    </Stack>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/TransformOptimizerSection/TransformOptimizerSection.tsx

@@ -0,0 +1,556 @@
+import { useCallback, useState } from "react";
+import { t } from "ttag";
+
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import {
+  Alert,
+  Box,
+  Button,
+  Divider,
+  Group,
+  Icon,
+  Stack,
+  Text,
+} from "metabase/ui";
+import * as Urls from "metabase/urls";
+import type { Transform } from "metabase-types/api";
+
+import {
+  type AcceptMode,
+  type VerifyResponse,
+  useAcceptProposalMutation,
+  useVerifyProposalMutation,
+} from "../../api";
+import { useOptimizerStream } from "../../hooks/use-optimizer-stream";
+import type { Proposal } from "../../types";
+import { OptimizationDegreeDial } from "../OptimizationDegreeDial";
+import { ProposalCard } from "../ProposalCard";
+import { TargetIndexesSection } from "../TargetIndexesSection";
+
+import { ProposalGroup } from "./ProposalGroup";
+import { SonicOptimizedBanner } from "./SonicOptimizedBanner";
+
+type Props = {
+  transform: Transform;
+  readOnly?: boolean;
+};
+
+export function TransformOptimizerSection({ transform, readOnly }: Props) {
+  const currentSql = getNativeSql(transform);
+  const { state, start, abort, dismissProposal } = useOptimizerStream({
+    transformId: transform.id,
+  });
+  const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
+  const [verify, verifyResult] = useVerifyProposalMutation();
+  const [accept, acceptResult] = useAcceptProposalMutation();
+  const [busyProposalId, setBusyProposalId] = useState<string | null>(null);
+  // Tracks whether the user has explicitly clicked Re-analyze in this
+  // mount. Once true, we trust the in-session optimize state over the
+  // persisted `transform.optimized` flag. Without this, navigating to a
+  // YOLO'd transform would see `transform.optimized=true` AND a stale
+  // optimize-endpoint cache populated by the bulk run; we want the
+  // banner (persisted verdict) to win until the user asks for a fresh
+  // analysis.
+  const [hasUserReAnalyzed, setHasUserReAnalyzed] = useState(false);
+
+  const handleStart = useCallback(() => {
+    setHasUserReAnalyzed(true);
+    start();
+  }, [start]);
+
+  const handleVerify = useCallback(
+    async (proposal: Proposal) => {
+      setBusyProposalId(proposal.id);
+      const { data, error } = await verify({
+        transformId: transform.id,
+        proposal,
+      });
+      setBusyProposalId(null);
+      if (error) {
+        sendErrorToast(t`Failed to verify proposal`);
+        return;
+      }
+      reportVerify(data as VerifyResponse, sendSuccessToast, sendErrorToast);
+    },
+    [verify, transform.id, sendErrorToast, sendSuccessToast],
+  );
+
+  const handleAccept = useCallback(
+    async (proposal: Proposal, mode: AcceptMode = "new") => {
+      // Replace mode is single-proposal: never drag ancestors in,
+      // since `update-transform!` only touches the original transform.
+      const proposalIds =
+        mode === "replace"
+          ? [proposal.id]
+          : topoOrderForAccept(proposal, state.proposals);
+      setBusyProposalId(proposal.id);
+      const { data, error } = await accept({
+        transformId: transform.id,
+        proposalIds,
+        mode,
+      });
+      setBusyProposalId(null);
+      if (error) {
+        sendErrorToast(t`Failed to accept proposal`);
+        return;
+      }
+      if (data && "replaced_transform" in data) {
+        // Replace mode: the user is already on this transform's page —
+        // navigating to it would be a no-op. Just a confirmation toast.
+        sendSuccessToast(
+          t`Replaced source of "${data.replaced_transform.name}"`,
+        );
+      } else {
+        const created = data?.created_transforms ?? [];
+        if (created.length > 0) {
+          // Open the *leaf* of the created set — for a single rewrite
+          // that's the only one; for a precompute DAG it's the final
+          // proposal the user clicked, which is the one whose result
+          // they care about.
+          const leaf = created[created.length - 1];
+          sendSuccessToast(
+            created.length === 1
+              ? t`New transform "${leaf.name}" created`
+              : t`${created.length} new transforms created — leaf: "${leaf.name}"`,
+            // Open in a new tab so the user keeps the optimizer context
+            // (the proposals list, the source transform) visible while
+            // inspecting the generated transform.
+            () =>
+              window.open(Urls.transform(leaf.id), "_blank", "noopener,noreferrer"),
+            t`Open`,
+          );
+        } else {
+          sendSuccessToast(t`Index changes applied`);
+        }
+      }
+      // Remove every proposal we just accepted from the list.
+      for (const id of proposalIds) {
+        dismissProposal(id);
+      }
+    },
+    [
+      accept,
+      transform.id,
+      state.proposals,
+      sendErrorToast,
+      sendSuccessToast,
+      dismissProposal,
+    ],
+  );
+
+  const isLoading = state.status === "loading";
+  const isDone = state.status === "done";
+  const isErrored = state.status === "error";
+  const isInSessionOptimized =
+    isDone && state.optimizationDegree === 100 && state.proposals.length === 0;
+  // Render the Sonic banner whenever the optimizer's verdict — either
+  // persisted on the transform from a prior run/YOLO, or from this
+  // session — is "fully optimized." When the persisted flag says we're
+  // done, the banner *overrides* any stale proposals sitting in the
+  // optimize endpoint's cache (e.g. seeded by a bulk run that the user
+  // then YOLO'd). The user has to click Re-analyze to opt into a fresh
+  // run; that flips `hasUserReAnalyzed` and lets the in-session state
+  // take over.
+  const showOptimizedBanner =
+    !isLoading &&
+    (isInSessionOptimized ||
+      (transform.optimized === true && !hasUserReAnalyzed));
+  const canTrigger = !isLoading;
+
+  return (
+    <Stack p="lg" gap="md">
+      {showOptimizedBanner ? (
+        <SonicOptimizedBanner
+          summary={state.summary}
+          onReanalyze={handleStart}
+        />
+      ) : (
+        <>
+          <Group
+            justify="space-between"
+            align="flex-start"
+            gap="lg"
+            wrap="nowrap"
+          >
+            <Stack gap={4} miw={0}>
+              <Text fw="bold">{t`Optimize this transform`}</Text>
+              <Text c="text-secondary">
+                {t`Analyze the current native SQL and propose equivalent transforms with better performance.`}
+              </Text>
+            </Stack>
+            <TriggerButton
+              status={state.status}
+              canTrigger={canTrigger}
+              onStart={handleStart}
+              onAbort={abort}
+            />
+          </Group>
+
+          {(isLoading || isDone || isErrored) && (
+            <Stack gap="md">
+              {!isErrored && (
+                <OptimizationDegreeDial
+                  status={isLoading ? "streaming" : "done"}
+                  score={state.optimizationDegree}
+                />
+              )}
+
+              {state.summary && !isErrored && (
+                <Box>
+                  <Text c="text-secondary">{state.summary}</Text>
+                </Box>
+              )}
+
+              {isErrored && state.error && (
+                <Alert color="error" icon={<Icon name="warning" />}>
+                  <Text fw="bold">{t`Couldn't analyze transform`}</Text>
+                  <Text c="text-secondary" mt={4}>
+                    {state.error.message}
+                  </Text>
+                  {state.error.retryable && (
+                    <Group justify="flex-start" mt="sm">
+                      <Button size="xs" variant="default" onClick={handleStart}>
+                        {t`Retry`}
+                      </Button>
+                    </Group>
+                  )}
+                </Alert>
+              )}
+
+              <Stack gap="md">
+                {groupProposalsByDependency(state.proposals).map((group) => (
+                  <ProposalGroup
+                    key={group.key}
+                    proposals={group.proposals}
+                    renderCard={(proposal) => {
+                      const canVerify = proposal.kind !== "index";
+                      const dependencyNames = resolveDependencyNames(
+                        proposal,
+                        state.proposals,
+                      );
+                      return (
+                        <ProposalCard
+                          key={proposal.id}
+                          proposal={proposal}
+                          currentSql={currentSql}
+                          dependencyNames={dependencyNames}
+                          actions={{
+                            accept: {
+                              kind: "accept",
+                              busy:
+                                busyProposalId === proposal.id &&
+                                acceptResult.isLoading,
+                              disabled: readOnly,
+                              disabledReason: readOnly
+                                ? t`You don't have permission to create transforms here.`
+                                : undefined,
+                            },
+                            verify: canVerify
+                              ? {
+                                  kind: "verify",
+                                  busy:
+                                    busyProposalId === proposal.id &&
+                                    verifyResult.isLoading,
+                                }
+                              : undefined,
+                            dismiss: { kind: "dismiss" },
+                          }}
+                          onAccept={(mode) => handleAccept(proposal, mode)}
+                          onVerify={() => handleVerify(proposal)}
+                          onDismiss={() => dismissProposal(proposal.id)}
+                        />
+                      );
+                    }}
+                  />
+                ))}
+              </Stack>
+            </Stack>
+          )}
+        </>
+      )}
+
+      <Divider />
+      <TargetIndexesSection transform={transform} readOnly={readOnly} />
+    </Stack>
+  );
+}
+
+function TriggerButton({
+  status,
+  canTrigger,
+  onStart,
+  onAbort,
+}: {
+  status: ReturnType<typeof useOptimizerStream>["state"]["status"];
+  canTrigger: boolean;
+  onStart: () => void;
+  onAbort: () => void;
+}) {
+  if (status === "loading") {
+    return (
+      <Button variant="default" onClick={onAbort}>
+        {t`Stop`}
+      </Button>
+    );
+  }
+  return (
+    <Button
+      variant="filled"
+      onClick={onStart}
+      disabled={!canTrigger}
+      leftSection={<Icon name="sparkles" />}
+    >
+      {status === "idle" ? t`Suggest optimizations` : t`Re-analyze`}
+    </Button>
+  );
+}
+
+/**
+ * Pull the current native SQL off a transform so the proposal card can diff
+ * against it. Handles two on-the-wire shapes for the same thing:
+ *   - legacy `DatasetQuery`:  source.query.native.query  (string)
+ *   - new MBQL stages:        source.query.stages[i].native  where
+ *                             the stage's `lib/type` is "mbql.stage/native"
+ *
+ * We don't rely on a `type` discriminator because `DatasetQuery` is
+ * declared opaque to TS — those fields exist on the wire but are hidden
+ * from the type system.
+ */
+export function getNativeSql(transform: Transform): string | null {
+  const source = transform.source as unknown as {
+    type?: string;
+    query?: {
+      native?: { query?: unknown };
+      stages?: Array<{ "lib/type"?: string; native?: unknown }>;
+    };
+  };
+  if (source?.type === "python") {
+    return null;
+  }
+  const query = source?.query;
+  if (!query) {
+    return null;
+  }
+
+  // Legacy: { native: { query: "..." } }
+  const legacy = query.native?.query;
+  if (typeof legacy === "string" && legacy.trim().length > 0) {
+    return legacy;
+  }
+
+  // New MBQL stages: pick the first native stage.
+  if (Array.isArray(query.stages)) {
+    const nativeStage = query.stages.find(
+      (s) => s?.["lib/type"] === "mbql.stage/native",
+    );
+    const stageSql = nativeStage?.native;
+    if (typeof stageSql === "string" && stageSql.trim().length > 0) {
+      return stageSql;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Walk `depends_on` from the chosen proposal back through every ancestor and
+ * return ids in roots-first order (matching the accept-endpoint contract).
+ *
+ * The server rejects the whole request if any referenced id is missing
+ * from the cache, so a missing ancestor in the client state is dropped
+ * here — the user will see a 404 with the offending ids if that ever
+ * happens.
+ *
+ * Exception: a `:index` proposal whose DDL targets `source-db` doesn't
+ * actually need its `depends_on` ancestors to exist at accept time —
+ * the index runs against tables that already exist in the source
+ * database. The LLM sometimes tags those indices with `depends_on`
+ * pointing at the rewrite they support (a *logical* link, not a
+ * *materialisation* link). Topo-including the rewrite there would
+ * silently accept the rewrite alongside, which is not what the user
+ * asked for.
+ */
+export function topoOrderForAccept(
+  target: Proposal,
+  available: Proposal[],
+): string[] {
+  if (target.kind === "index" && target.ddl_statement?.target === "source-db") {
+    return [target.id];
+  }
+
+  const byId = new Map(available.map((p) => [p.id, p]));
+  const visited = new Set<string>();
+  const order: string[] = [];
+
+  const visit = (id: string) => {
+    if (visited.has(id)) {
+      return;
+    }
+    visited.add(id);
+    const node = byId.get(id);
+    if (!node) {
+      return;
+    }
+    for (const depId of node.depends_on) {
+      visit(depId);
+    }
+    order.push(id);
+  };
+
+  visit(target.id);
+  return order;
+}
+
+type ProposalGroupShape = {
+  /** Stable React key — concatenated, topo-ordered proposal ids. */
+  key: string;
+  /** Members of the group in topological (root-first) order. */
+  proposals: Proposal[];
+};
+
+/**
+ * Walk the `depends_on` edges (as undirected) and split proposals into
+ * connected components. Each component is one "pipeline" — a precompute
+ * + its rewrite leaf, a rewrite + its supporting index, etc. — that
+ * the user accepts as a whole or breaks apart by `Dismiss`-ing parts.
+ *
+ * Within a component, proposals are returned in topological (root-first)
+ * order so the UI naturally reads top-to-bottom along the
+ * "build precompute → consume in rewrite" flow.
+ *
+ * Components are returned in input order of their first-seen proposal,
+ * which preserves the LLM's emit order across the panel.
+ */
+export function groupProposalsByDependency(
+  proposals: Proposal[],
+): ProposalGroupShape[] {
+  // Union-find over the undirected depends_on graph.
+  const parent = new Map<string, string>();
+  for (const p of proposals) {
+    parent.set(p.id, p.id);
+  }
+  const find = (id: string): string => {
+    let cur = id;
+    while (parent.get(cur) !== cur) {
+      const p = parent.get(cur)!;
+      parent.set(cur, parent.get(p)!);
+      cur = parent.get(cur)!;
+    }
+    return cur;
+  };
+  const union = (a: string, b: string) => {
+    const ra = find(a);
+    const rb = find(b);
+    if (ra !== rb) {
+      parent.set(ra, rb);
+    }
+  };
+  const ids = new Set(proposals.map((p) => p.id));
+  for (const p of proposals) {
+    for (const dep of p.depends_on) {
+      if (ids.has(dep)) {
+        union(p.id, dep);
+      }
+    }
+  }
+
+  // Bucket each proposal under its root, preserving emit order.
+  const buckets = new Map<string, Proposal[]>();
+  for (const p of proposals) {
+    const root = find(p.id);
+    const bucket = buckets.get(root) ?? [];
+    bucket.push(p);
+    buckets.set(root, bucket);
+  }
+
+  // Topo-sort each bucket (roots first).
+  const groups = Array.from(buckets.values()).map((bucket) => {
+    const byId = new Map(bucket.map((p) => [p.id, p]));
+    const visited = new Set<string>();
+    const ordered: Proposal[] = [];
+    const visit = (id: string) => {
+      if (visited.has(id)) {
+        return;
+      }
+      visited.add(id);
+      const node = byId.get(id);
+      if (!node) {
+        return;
+      }
+      for (const dep of node.depends_on) {
+        if (byId.has(dep)) {
+          visit(dep);
+        }
+      }
+      ordered.push(node);
+    };
+    for (const p of bucket) {
+      visit(p.id);
+    }
+    return { key: ordered.map((p) => p.id).join("→"), proposals: ordered };
+  });
+
+  // Lift standalone-index groups to the top of the panel. Indices are
+  // idempotent, carry no equivalence risk, and frequently capture most
+  // of the available speedup — the user should see them first as the
+  // cheap, safe win. Mixed (rewrite/precompute) groups keep their LLM
+  // emit order among themselves; the sort is stable.
+  return groups
+    .map((g, i) => ({
+      g,
+      i,
+      indexOnly: g.proposals.every((p) => p.kind === "index"),
+    }))
+    .sort((a, b) => {
+      if (a.indexOnly !== b.indexOnly) {
+        return a.indexOnly ? -1 : 1;
+      }
+      return a.i - b.i;
+    })
+    .map(({ g }) => g);
+}
+
+/**
+ * Resolve each id in `proposal.depends_on` against the current proposal
+ * list. Returns `[{id, name}…]` so the card can render readable badges
+ * rather than opaque ids. Missing ancestors (already dismissed,
+ * unknown id) are dropped silently — the BE rejects accept calls that
+ * reference missing proposals so the user will see an explicit error
+ * if it matters.
+ */
+export function resolveDependencyNames(
+  proposal: Proposal,
+  available: Proposal[],
+): Array<{ id: string; name: string }> {
+  if (!proposal.depends_on || proposal.depends_on.length === 0) {
+    return [];
+  }
+  const byId = new Map(available.map((p) => [p.id, p]));
+  return proposal.depends_on
+    .map((id) => {
+      const dep = byId.get(id);
+      return dep ? { id: dep.id, name: dep.name } : null;
+    })
+    .filter((x): x is { id: string; name: string } => x !== null);
+}
+
+export function reportVerify(
+  data: VerifyResponse | undefined,
+  ok: ReturnType<typeof useMetadataToasts>["sendSuccessToast"],
+  fail: ReturnType<typeof useMetadataToasts>["sendErrorToast"],
+) {
+  if (!data) {
+    return;
+  }
+  if ("error" in data) {
+    fail(t`Verification failed: ${data.error}`);
+    return;
+  }
+  if (data.equivalent) {
+    ok(
+      t`Verified equivalent — slow ${data.slow_duration_ms} ms vs fast ${data.fast_duration_ms} ms`,
+    );
+  } else {
+    fail(t`Results diverged — ${data.diff_rows} rows differ`);
+  }
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/components/TransformOptimizerSection/index.ts

@@ -0,0 +1 @@
+export { TransformOptimizerSection } from "./TransformOptimizerSection";

enterprise/frontend/src/metabase-enterprise/transform_optimizer/hooks/use-optimizer-stream.ts

@@ -0,0 +1,174 @@
+import { useCallback, useMemo, useState } from "react";
+
+import { useSelector } from "metabase/redux";
+
+import type { OptimizeResponse } from "../api";
+import { optimizerApi, useOptimizeQuery } from "../api";
+import type { OptimizerRunState } from "../types";
+
+const INITIAL: OptimizerRunState = {
+  status: "idle",
+  summary: null,
+  proposals: [],
+  optimizationDegree: null,
+  error: null,
+};
+
+type Options = {
+  transformId: number | string;
+};
+
+type Controls = {
+  state: OptimizerRunState;
+  start: (opts?: { analyze?: boolean }) => void;
+  abort: () => void;
+  reset: () => void;
+  dismissProposal: (proposalId: string) => void;
+};
+
+/**
+ * Drives the optimizer panel. Wraps a regular auto-subscribing
+ * `useOptimizeQuery` rather than the lazy variant so that the result
+ * survives `TransformOptimizerSection` unmounts — e.g. switching tabs and
+ * returning. We pair that with a long `keepUnusedDataFor` on the endpoint
+ * so the cache outlives the brief gap between unmount and remount.
+ *
+ * On mount we inspect the RTK Query cache for an existing result; if one
+ * is there we flip into "requested" mode immediately so the component
+ * renders the cached payload instead of the trigger button. A fresh
+ * request is only sent when `start()` is called explicitly or when the
+ * user passes `analyze: true` (a different cache key).
+ *
+ * Local-only state we still track in the component:
+ *   - `dismissed` — ids the user has dismissed locally. Not persisted
+ *     across unmounts on purpose; dismissing is a "for this view" action.
+ */
+export function useOptimizerStream({ transformId }: Options): Controls {
+  const [analyze, setAnalyze] = useState(false);
+  const args = useMemo(
+    () => ({ transformId, analyze }),
+    [transformId, analyze],
+  );
+
+  // Look up whatever's in the cache for `args`. If there's already a
+  // payload (e.g. the user ran the optimizer earlier and is coming back
+  // from another tab), treat the hook as already-requested so the
+  // auto-query re-attaches without firing a new fetch.
+  // Cast the state to `any` here — `EnterpriseApi`'s tag-union typing is
+  // wider than the root `Api`'s, so its `.select(...)` selector won't
+  // structurally match `metabase/redux`'s `State`. We only need the
+  // `{data, error}` shape, which is independent of the tag set.
+  const cachedResult = useSelector((state) =>
+    optimizerApi.endpoints.optimize.select(args)(
+      state as Parameters<
+        ReturnType<typeof optimizerApi.endpoints.optimize.select>
+      >[0],
+    ),
+  );
+  const [hasRequested, setHasRequested] = useState<boolean>(
+    () => !!cachedResult?.data || !!cachedResult?.error,
+  );
+
+  const queryResult = useOptimizeQuery(args, {
+    skip: !hasRequested,
+  });
+
+  const [dismissed, setDismissed] = useState<Set<string>>(() => new Set());
+
+  const start = useCallback(
+    ({ analyze: nextAnalyze = false }: { analyze?: boolean } = {}) => {
+      // Re-running clears any per-instance dismissals — they apply to the
+      // last result the user saw, not the next one.
+      setDismissed(new Set());
+      if (nextAnalyze !== analyze) {
+        setAnalyze(nextAnalyze);
+      }
+      setHasRequested(true);
+      // If the cache already has a fresh result for these args, the query
+      // hook will deliver it without refetching. If we want a fresh run
+      // even when cached, the caller can `reset()` first.
+      void queryResult.refetch?.();
+    },
+    [analyze, queryResult],
+  );
+
+  const abort = useCallback(() => {
+    // RTK Query auto-queries don't expose a per-subscription abort the way
+    // lazy queries do. The cheapest "stop showing the loader" is to mark
+    // the hook unrequested again; an inflight network call will still
+    // finish but its result will sit in cache until the user explicitly
+    // re-requests.
+    setHasRequested(false);
+  }, []);
+
+  const reset = useCallback(() => {
+    setHasRequested(false);
+    setDismissed(new Set());
+  }, []);
+
+  const dismissProposal = useCallback((proposalId: string) => {
+    setDismissed((prev) => {
+      const next = new Set(prev);
+      next.add(proposalId);
+      return next;
+    });
+  }, []);
+
+  const state = projectState(
+    {
+      data: queryResult.data,
+      error: queryResult.error as { message?: string } | undefined,
+      isFetching: queryResult.isFetching,
+      isUninitialized: queryResult.isUninitialized,
+      isError: queryResult.isError,
+      isSuccess: queryResult.isSuccess,
+    },
+    dismissed,
+    hasRequested,
+  );
+
+  return { state, start, abort, reset, dismissProposal };
+}
+
+type LazyResult = {
+  data?: OptimizeResponse;
+  error?: { message?: string };
+  isFetching: boolean;
+  isUninitialized: boolean;
+  isSuccess: boolean;
+  isError: boolean;
+};
+
+function projectState(
+  result: LazyResult,
+  dismissed: Set<string>,
+  hasRequested: boolean,
+): OptimizerRunState {
+  if (!hasRequested && !result.data) {
+    return INITIAL;
+  }
+  if (result.isFetching) {
+    return { ...INITIAL, status: "loading" };
+  }
+  if (result.isError) {
+    return {
+      ...INITIAL,
+      status: "error",
+      error: {
+        message:
+          (result.error && result.error.message) || "Optimizer request failed.",
+        retryable: true,
+      },
+    };
+  }
+  if (!result.data) {
+    return INITIAL;
+  }
+  return {
+    status: "done",
+    summary: result.data.summary ?? null,
+    proposals: result.data.proposals.filter((p) => !dismissed.has(p.id)),
+    optimizationDegree: result.data.optimization_degree,
+    error: null,
+  };
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/index.ts

@@ -0,0 +1,16 @@
+import { PLUGIN_TRANSFORM_OPTIMIZER } from "metabase/plugins";
+import { hasPremiumFeature } from "metabase-enterprise/settings";
+
+import { FindSlowTool } from "./components/FindSlowTool";
+import { TransformOptimizerSection } from "./components/TransformOptimizerSection";
+
+export function initializePlugin() {
+  // The optimizer rides on the same premium gate as the existing
+  // transform-inspector feature, since both share enterprise context-builder
+  // and Metabot tool infrastructure.
+  if (hasPremiumFeature("transforms-python")) {
+    PLUGIN_TRANSFORM_OPTIMIZER.isEnabled = true;
+    PLUGIN_TRANSFORM_OPTIMIZER.RunPageSection = TransformOptimizerSection;
+    PLUGIN_TRANSFORM_OPTIMIZER.FindSlowTool = FindSlowTool;
+  }
+}

enterprise/frontend/src/metabase-enterprise/transform_optimizer/types/index.ts

@@ -0,0 +1,105 @@
+export type ProposalKind = "rewrite" | "index" | "precompute";
+
+export type ProposalSeverity = "high" | "medium" | "low";
+
+export type DdlTarget = "source-db" | "transform-target";
+
+export type DdlValidation = "accepted" | "rejected";
+
+export type DdlExecutionStatus =
+  | "pending"
+  | "running"
+  | "executed"
+  | "failed"
+  | "skipped";
+
+/**
+ * The single DDL attached to a `kind: "index"` proposal. Indices that
+ * target a new table created by another proposal use
+ * `target: "transform-target"` and the parent proposal's id in
+ * `depends_on`.
+ */
+export type DdlStatement = {
+  target: DdlTarget;
+  statement: string;
+  rationale: string;
+  validation: DdlValidation;
+  index_name?: string | null;
+  rejection?: { reason: string; detail?: string } | null;
+};
+
+/**
+ * One change per proposal:
+ *   - `rewrite` / `precompute` carry `body`, no `ddl_statement`.
+ *   - `index` carries `ddl_statement`, no `body`.
+ *
+ * Indices that depend on a new target table express that via
+ * `depends_on` plus `ddl_statement.target = "transform-target"` — the FE
+ * walks `depends_on` to topo-order accept requests.
+ */
+export type Proposal = {
+  id: string;
+  name: string;
+  kind: ProposalKind;
+  severity: ProposalSeverity;
+  rationale: string;
+  expected_speedup: string;
+  body: string | null;
+  ddl_statement: DdlStatement | null;
+  depends_on: string[];
+};
+
+export type OptimizerRunStatus = "idle" | "loading" | "done" | "error";
+
+export type OptimizerRunError = {
+  message: string;
+  retryable: boolean;
+};
+
+export type OptimizerRunState = {
+  status: OptimizerRunStatus;
+  summary: string | null;
+  proposals: Proposal[];
+  optimizationDegree: number | null;
+  error: OptimizerRunError | null;
+};
+
+/**
+ * One row from `GET /api/ee/transform-optimizer/:id/indexes`. Matches the
+ * Postgres catalog shape returned by `index-introspection/fetch-indexes`,
+ * plus:
+ *   - `is_target_table` — true when the index lives on the transform's
+ *     materialised target, false when it's on a source (input) table
+ *   - `managed_by_optimizer` — true when the index is replayed by the
+ *     optimizer's `target.post_run_ddl` machinery (only ever true for
+ *     target-table indices)
+ */
+export type TargetIndex = {
+  schema: string;
+  table: string;
+  name: string;
+  access_method: string;
+  is_unique: boolean;
+  is_primary: boolean;
+  is_valid: boolean;
+  partial_predicate: string | null;
+  index_expressions: string | null;
+  definition: string;
+  key_columns: string[];
+  include_columns: string[];
+  managed_by_optimizer: boolean;
+  is_target_table: boolean;
+};
+
+export type DropIndexResult =
+  | { status: "dropped" }
+  | { status: "failed"; error_message: string; error?: string }
+  | {
+      status: "skipped";
+      reason:
+        | "index-not-on-referenced-table"
+        | "unsafe-name"
+        | "no-database"
+        | "not-postgres";
+      error?: string;
+    };

frontend/src/metabase-types/api/transform.ts

@@ -37,6 +37,10 @@ export type Transform = {
   updated_at: string;
   source_readable: boolean;
 
+  // true when the transform optimizer last reported an optimization_degree of 100;
+  // cleared by the BE on source changes
+  optimized?: boolean;
+
   // true when transform was deleted but still referenced by runs
   deleted?: boolean;
 
@@ -286,6 +290,8 @@ export type ListTransformRunsRequest = {
   "start-time"?: string;
   "end-time"?: string;
   "run-methods"?: TransformRunMethod[];
+  "min-duration-ms"?: number;
+  "max-duration-ms"?: number;
   "sort-column"?: TransformRunSortColumn;
   "sort-direction"?: SortDirection;
 } & PaginationRequest;

frontend/src/metabase/plugins/index.ts

@@ -130,6 +130,11 @@ export {
   type PythonTransformSourceValidationResult,
   type PythonTransformsPlugin,
 } from "./oss/transforms";
+export {
+  PLUGIN_TRANSFORM_OPTIMIZER,
+  type TransformOptimizerPlugin,
+  type TransformOptimizerRunPageSectionProps,
+} from "./oss/transform-optimizer";
 export {
   PLUGIN_DEPENDENCIES,
   type DependencyGraphPageContextType,
@@ -192,6 +197,7 @@ import { reinitialize as reinitializeSmtpOverride } from "./oss/smtp-override";
 import { reinitialize as reinitializeSnippets } from "./oss/snippets";
 import { reinitialize as reinitializeSupport } from "./oss/support";
 import { reinitialize as reinitializeTenants } from "./oss/tenants";
+import { reinitialize as reinitializeTransformOptimizer } from "./oss/transform-optimizer";
 import { reinitialize as reinitializeTransforms } from "./oss/transforms";
 import { reinitialize as reinitializeUploadManagement } from "./oss/upload-management";
 import { reinitialize as reinitializeWhitelabel } from "./oss/whitelabel";
@@ -237,6 +243,7 @@ export function reinitialize() {
   reinitializeTenants();
   reinitializeDependencies();
   reinitializeTransforms();
+  reinitializeTransformOptimizer();
   reinitializeUploadManagement();
   reinitializeWhitelabel();
   reinitializeWritableConnection();

frontend/src/metabase/plugins/oss/transform-optimizer.ts

@@ -0,0 +1,47 @@
+import type { ComponentType } from "react";
+
+import type { Transform } from "metabase-types/api";
+
+export type TransformOptimizerRunPageSectionProps = {
+  transform: Transform;
+  readOnly?: boolean;
+};
+
+export type FindSlowToolProps = {
+  thresholdSec: number | undefined;
+  onThresholdChange: (sec: number | undefined) => void;
+  /**
+   * Whether to also surface transforms whose last run hit the timeout
+   * regardless of duration threshold. Off by default — timed-out runs
+   * are a different bucket from "finished but slow" workloads.
+   */
+  includeTimedOut: boolean;
+  onIncludeTimedOutChange: (next: boolean) => void;
+  /**
+   * The transform ids that currently match the threshold. The "Hammer
+   * time" button POSTs these to the bulk-optimize endpoint. Empty when
+   * no threshold is set OR no transform's last_run survives the filter.
+   */
+  matchingTransformIds: number[];
+};
+
+export type TransformOptimizerPlugin = {
+  isEnabled: boolean;
+  RunPageSection: ComponentType<TransformOptimizerRunPageSectionProps> | null;
+  FindSlowTool: ComponentType<FindSlowToolProps> | null;
+};
+
+const getDefaultPlugin = (): TransformOptimizerPlugin => ({
+  isEnabled: false,
+  RunPageSection: null,
+  FindSlowTool: null,
+});
+
+export const PLUGIN_TRANSFORM_OPTIMIZER = getDefaultPlugin();
+
+/**
+ * @internal Do not call directly. Use the main reinitialize function from metabase/plugins instead.
+ */
+export function reinitialize() {
+  Object.assign(PLUGIN_TRANSFORM_OPTIMIZER, getDefaultPlugin());
+}

frontend/src/metabase/transforms/components/RunStatus/RunStatus.tsx

@@ -1,10 +1,10 @@
 import { t } from "ttag";
 
 import { useSetting } from "metabase/common/hooks";
-import { Box, Group, Icon } from "metabase/ui";
+import { Box, Group, Icon, Text } from "metabase/ui";
 import type { TransformRun } from "metabase-types/api";
 
-import { parseTimestampWithTimezone } from "../../utils";
+import { formatRunDuration, parseTimestampWithTimezone } from "../../utils";
 import { RunInfo } from "../RunInfo";
 
 type RunStatusSectionProps = {
@@ -29,12 +29,24 @@ export function RunStatus({
     );
   }
 
-  const { status, end_time, message } = run;
+  const { status, start_time, end_time, message } = run;
   const endTime =
     end_time != null
       ? parseTimestampWithTimezone(end_time, systemTimezone)
       : null;
   const endTimeText = endTime != null ? endTime.fromNow() : null;
+  const duration = formatRunDuration(start_time, end_time);
+  // Parentheses + a duration token aren't a translatable string (ttag
+  // refuses messages that are just punctuation around an interpolation),
+  // so render them as plain JSX. The duration string itself is locale-
+  // shaped by `formatRunDuration` via the existing run-duration t``
+  // calls.
+  const durationNode =
+    duration != null ? (
+      <Text c="text-secondary" data-testid="run-duration">
+        {`(${duration})`}
+      </Text>
+    ) : null;
 
   const errorInfo =
     message != null && status != null ? (
@@ -62,6 +74,7 @@ export function RunStatus({
               ? t`Last ran ${endTimeText} successfully.`
               : t`Last ran successfully.`}
           </Box>
+          {durationNode}
           {runInfo}
         </Group>
       );
@@ -74,6 +87,7 @@ export function RunStatus({
               ? t`Last run failed ${endTimeText}.`
               : t`Last run failed.`}
           </Box>
+          {durationNode && <Box mr="sm">{durationNode}</Box>}
           {errorInfo ?? runInfo}
         </Group>
       );
@@ -93,6 +107,7 @@ export function RunStatus({
               ? t`Last run was canceled ${endTimeText}.`
               : t`Last run was canceled.`}
           </Box>
+          {durationNode}
           {runInfo}
         </Group>
       );
@@ -105,6 +120,7 @@ export function RunStatus({
               ? t`Last run timed out ${endTimeText}.`
               : t`Last run timed out.`}
           </Box>
+          {durationNode && <Box mr="sm">{durationNode}</Box>}
           {errorInfo ?? runInfo}
         </Group>
       );

frontend/src/metabase/transforms/pages/RunListPage/RunFilterBar/DurationFilterWidget/DurationFilterWidget.tsx

@@ -0,0 +1,164 @@
+import { useDisclosure } from "@mantine/hooks";
+import { type FormEvent, useState } from "react";
+import { t } from "ttag";
+
+import {
+  Box,
+  Button,
+  Divider,
+  Group,
+  NumberInput,
+  Popover,
+  Stack,
+  Text,
+} from "metabase/ui";
+
+import { FILTER_WIDGET_MIN_WIDTH } from "../../../../constants";
+import { FilterFieldSet } from "../FilterFieldSet";
+
+const MS_PER_SECOND = 1000;
+
+type DurationFilterWidgetProps = {
+  minDurationMs: number | undefined;
+  maxDurationMs: number | undefined;
+  onChange: (value: {
+    minDurationMs: number | undefined;
+    maxDurationMs: number | undefined;
+  }) => void;
+};
+
+export function DurationFilterWidget({
+  minDurationMs,
+  maxDurationMs,
+  onChange,
+}: DurationFilterWidgetProps) {
+  const [isOpened, { toggle, close }] = useDisclosure();
+
+  function handleRemove() {
+    onChange({ minDurationMs: undefined, maxDurationMs: undefined });
+  }
+
+  function handleSubmit(value: {
+    minDurationMs: number | undefined;
+    maxDurationMs: number | undefined;
+  }) {
+    close();
+    onChange(value);
+  }
+
+  return (
+    <Popover opened={isOpened} position="bottom-start" onDismiss={close}>
+      <Popover.Target>
+        <FilterFieldSet
+          label={t`Duration`}
+          displayValue={getDisplayValue(minDurationMs, maxDurationMs)}
+          icon="clock"
+          onClick={toggle}
+          onRemove={handleRemove}
+        />
+      </Popover.Target>
+      <Popover.Dropdown>
+        <DurationFilterForm
+          minDurationMs={minDurationMs}
+          maxDurationMs={maxDurationMs}
+          onSubmit={handleSubmit}
+        />
+      </Popover.Dropdown>
+    </Popover>
+  );
+}
+
+type DurationFilterFormProps = {
+  minDurationMs: number | undefined;
+  maxDurationMs: number | undefined;
+  onSubmit: (value: {
+    minDurationMs: number | undefined;
+    maxDurationMs: number | undefined;
+  }) => void;
+};
+
+function DurationFilterForm({
+  minDurationMs: initialMinMs,
+  maxDurationMs: initialMaxMs,
+  onSubmit,
+}: DurationFilterFormProps) {
+  const [minSec, setMinSec] = useState<number | "">(toSecondsInput(initialMinMs));
+  const [maxSec, setMaxSec] = useState<number | "">(toSecondsInput(initialMaxMs));
+
+  const minMs = toMsValue(minSec);
+  const maxMs = toMsValue(maxSec);
+  const hasBound = minMs != null || maxMs != null;
+  const rangeOk = minMs == null || maxMs == null || minMs <= maxMs;
+  const isValid = hasBound && rangeOk;
+
+  const handleSubmit = (event: FormEvent) => {
+    event.preventDefault();
+    if (isValid) {
+      onSubmit({ minDurationMs: minMs, maxDurationMs: maxMs });
+    }
+  };
+
+  const hasExisting = initialMinMs != null || initialMaxMs != null;
+
+  return (
+    <Box component="form" miw={FILTER_WIDGET_MIN_WIDTH} onSubmit={handleSubmit}>
+      <Stack p="md" gap="sm">
+        <Text c="text-secondary" fz="sm">
+          {t`Only show runs that finished and lasted within this range. In-progress runs are excluded.`}
+        </Text>
+        <NumberInput
+          label={t`At least (seconds)`}
+          value={minSec}
+          min={0}
+          step={1}
+          allowDecimal={false}
+          allowNegative={false}
+          onChange={(v) => setMinSec(typeof v === "number" ? v : "")}
+        />
+        <NumberInput
+          label={t`At most (seconds)`}
+          value={maxSec}
+          min={0}
+          step={1}
+          allowDecimal={false}
+          allowNegative={false}
+          error={!rangeOk ? t`Max must be ≥ min` : undefined}
+          onChange={(v) => setMaxSec(typeof v === "number" ? v : "")}
+        />
+      </Stack>
+      <Divider />
+      <Group p="md" justify="end">
+        <Button type="submit" variant="filled" disabled={!isValid}>
+          {hasExisting ? t`Update filter` : t`Add filter`}
+        </Button>
+      </Group>
+    </Box>
+  );
+}
+
+function toSecondsInput(ms: number | undefined): number | "" {
+  return ms == null ? "" : Math.round(ms / MS_PER_SECOND);
+}
+
+function toMsValue(sec: number | ""): number | undefined {
+  return typeof sec === "number" && sec >= 0 ? sec * MS_PER_SECOND : undefined;
+}
+
+function getDisplayValue(
+  minMs: number | undefined,
+  maxMs: number | undefined,
+) {
+  if (minMs == null && maxMs == null) {
+    return null;
+  }
+  const minSec = minMs != null ? Math.round(minMs / MS_PER_SECOND) : null;
+  const maxSec = maxMs != null ? Math.round(maxMs / MS_PER_SECOND) : null;
+
+  if (minSec != null && maxSec != null) {
+    return t`${minSec}–${maxSec}s`;
+  }
+  if (minSec != null) {
+    return t`≥ ${minSec}s`;
+  }
+  return t`≤ ${maxSec}s`;
+}

frontend/src/metabase/transforms/pages/RunListPage/RunFilterBar/DurationFilterWidget/index.ts

@@ -0,0 +1 @@
+export * from "./DurationFilterWidget";

frontend/src/metabase/transforms/pages/RunListPage/RunFilterBar/RunFilterBar.tsx

@@ -13,6 +13,7 @@ import type {
 
 import type { TransformRunFilterOptions } from "../types";
 
+import { DurationFilterWidget } from "./DurationFilterWidget";
 import { RunMethodFilterWidget } from "./RunMethodFilterWidget";
 import { StatusFilterWidget } from "./StatusFilterWidget";
 import { TagFilterWidget } from "./TagFilterWidget";
@@ -61,6 +62,16 @@ export function RunFilterBar({
     onFilterOptionsChange({ ...filterOptions, runMethods });
   };
 
+  const handleDurationChange = ({
+    minDurationMs,
+    maxDurationMs,
+  }: {
+    minDurationMs: number | undefined;
+    maxDurationMs: number | undefined;
+  }) => {
+    onFilterOptionsChange({ ...filterOptions, minDurationMs, maxDurationMs });
+  };
+
   return (
     <Group>
       <TransformFilterWidget
@@ -89,6 +100,11 @@ export function RunFilterBar({
         runMethods={filterOptions.runMethods ?? []}
         onChange={handleRunMethodsChange}
       />
+      <DurationFilterWidget
+        minDurationMs={filterOptions.minDurationMs}
+        maxDurationMs={filterOptions.maxDurationMs}
+        onChange={handleDurationChange}
+      />
       <TagFilterWidget
         label={t`Tags`}
         tagIds={filterOptions.transformTagIds ?? []}

frontend/src/metabase/transforms/pages/RunListPage/RunListPage.tsx

@@ -72,6 +72,8 @@ export function RunListPage({ location }: RunListPageProps) {
       "start-time": params.startTime,
       "end-time": params.endTime,
       "run-methods": params.runMethods,
+      "min-duration-ms": params.minDurationMs,
+      "max-duration-ms": params.maxDurationMs,
       "sort-column": params.sortColumn,
       "sort-direction": params.sortDirection,
     },

frontend/src/metabase/transforms/pages/RunListPage/RunTable/RunTable.tsx

@@ -1,5 +1,5 @@
 import type { Row, SortingState, Updater } from "@tanstack/react-table";
-import { useCallback, useMemo } from "react";
+import { useCallback, useMemo, useState } from "react";
 import { t } from "ttag";
 
 import { ListEmptyState } from "metabase/common/components/ListEmptyState";
@@ -14,7 +14,15 @@ import type {
 
 import type { TransformRunSortOptions } from "../types";
 
-import { getColumns, getSortingOptions, getSortingState } from "./utils";
+import {
+  DURATION_SORT_ID,
+  getColumns,
+  getRunDurationMs,
+  getSortingOptions,
+  getSortingState,
+} from "./utils";
+
+type DurationSortDirection = "asc" | "desc";
 
 type RunTableProps = {
   runs: TransformRun[];
@@ -42,10 +50,39 @@ export function RunTable({
     [tags, systemTimezone],
   );
 
-  const sortingState = useMemo(
-    () => getSortingState(sortOptions),
-    [sortOptions],
-  );
+  // Duration is sorted client-side over the current page only — the
+  // backend has no `duration` sort key. While duration sort is active the
+  // server-side sort is suspended so the two never compete.
+  const [durationSort, setDurationSort] =
+    useState<DurationSortDirection | null>(null);
+
+  const sortingState = useMemo<SortingState>(() => {
+    if (durationSort != null) {
+      return [{ id: DURATION_SORT_ID, desc: durationSort === "desc" }];
+    }
+    return getSortingState(sortOptions);
+  }, [durationSort, sortOptions]);
+
+  const sortedRuns = useMemo(() => {
+    if (durationSort == null) {
+      return runs;
+    }
+    const sign = durationSort === "asc" ? 1 : -1;
+    return [...runs].sort((a, b) => {
+      const da = getRunDurationMs(a);
+      const db = getRunDurationMs(b);
+      if (da == null && db == null) {
+        return 0;
+      }
+      if (da == null) {
+        return 1; // nulls last
+      }
+      if (db == null) {
+        return -1;
+      }
+      return sign * (da - db);
+    });
+  }, [runs, durationSort]);
 
   const notFoundLabel = hasFilters ? t`No runs found` : t`No runs yet`;
 
@@ -60,13 +97,22 @@ export function RunTable({
     (updater: Updater<SortingState>) => {
       const newSortingState =
         typeof updater === "function" ? updater(sortingState) : updater;
+      const next = newSortingState[0];
+      if (next?.id === DURATION_SORT_ID) {
+        setDurationSort(next.desc ? "desc" : "asc");
+        if (sortOptions != null) {
+          onSortOptionsChange(undefined);
+        }
+        return;
+      }
+      setDurationSort(null);
       onSortOptionsChange(getSortingOptions(newSortingState));
     },
-    [sortingState, onSortOptionsChange],
+    [sortingState, sortOptions, onSortOptionsChange],
   );
 
   const treeTableInstance = useTreeTableInstance<TransformRun>({
-    data: runs,
+    data: sortedRuns,
     columns,
     sorting: sortingState,
     manualSorting: true,

frontend/src/metabase/transforms/pages/RunListPage/RunTable/utils.tsx

@@ -19,6 +19,7 @@ import {
 } from "metabase-types/api";
 
 import {
+  formatRunDuration,
   formatRunMethod,
   formatStatus,
   getTransformRunName,
@@ -123,6 +124,39 @@ function getEndedAtColumn(
   };
 }
 
+// Duration is sorted entirely client-side over the already-fetched page.
+// It is intentionally NOT one of TRANSFORM_RUN_SORT_COLUMNS — the backend
+// has no sort key for it. RunTable detects this id and routes the sort
+// into local state.
+export const DURATION_SORT_ID = "duration";
+
+function getDurationColumn(): TreeTableColumnDef<TransformRun> {
+  return {
+    id: DURATION_SORT_ID,
+    header: ({ header }) => (
+      <SortableHeaderPill
+        name={t`Duration`}
+        sort={header.column.getIsSorted() || undefined}
+      />
+    ),
+    width: "auto",
+    enableSorting: true,
+    accessorFn: (run) => formatRunDuration(run.start_time, run.end_time),
+    cell: ({ getValue }) => {
+      const value = getValue();
+      return value != null ? <Ellipsified>{String(value)}</Ellipsified> : null;
+    },
+  };
+}
+
+export function getRunDurationMs(run: TransformRun): number | null {
+  if (!run.start_time || !run.end_time) {
+    return null;
+  }
+  const ms = Date.parse(run.end_time) - Date.parse(run.start_time);
+  return Number.isFinite(ms) && ms >= 0 ? ms : null;
+}
+
 function getStatusColumn(): TreeTableColumnDef<TransformRun> {
   return {
     id: "status" satisfies TransformRunSortColumn,
@@ -200,6 +234,7 @@ export function getColumns(
     getTransformColumn(),
     getStartedAtColumn(systemTimezone),
     getEndedAtColumn(systemTimezone),
+    getDurationColumn(),
     getStatusColumn(),
     getRunMethodColumn(),
     getTransformTagsColumn(tagsById),

frontend/src/metabase/transforms/pages/RunListPage/types.ts

@@ -14,6 +14,8 @@ export type TransformRunFilterOptions = {
   startTime?: string;
   endTime?: string;
   runMethods?: TransformRunMethod[];
+  minDurationMs?: number;
+  maxDurationMs?: number;
 };
 
 export type TransformRunSortOptions = {

frontend/src/metabase/transforms/pages/RunListPage/utils.ts

@@ -24,6 +24,8 @@ export function getParsedParams(
     "start-time": startTime,
     "end-time": endTime,
     "run-methods": runMethods,
+    "min-duration-ms": minDurationMs,
+    "max-duration-ms": maxDurationMs,
     "sort-column": sortColumn,
     "sort-direction": sortDirection,
   } = location.query;
@@ -43,6 +45,8 @@ export function getParsedParams(
     runMethods: Urls.parseListParam(runMethods, (v) =>
       Urls.parseEnumParam(v, TRANSFORM_RUN_METHODS),
     ),
+    minDurationMs: Urls.parseNumberParam(minDurationMs),
+    maxDurationMs: Urls.parseNumberParam(maxDurationMs),
     sortColumn: Urls.parseEnumParam(sortColumn, TRANSFORM_RUN_SORT_COLUMNS),
     sortDirection: Urls.parseEnumParam(sortDirection, SORT_DIRECTIONS),
   };
@@ -55,7 +59,9 @@ export function hasFilterOptions(options: TransformRunFilterOptions) {
     options.transformTagIds != null ||
     options.startTime != null ||
     options.endTime != null ||
-    options.runMethods != null
+    options.runMethods != null ||
+    options.minDurationMs != null ||
+    options.maxDurationMs != null
   );
 }
 
@@ -69,6 +75,8 @@ export function getFilterOptions(
     startTime: params.startTime,
     endTime: params.endTime,
     runMethods: params.runMethods,
+    minDurationMs: params.minDurationMs,
+    maxDurationMs: params.maxDurationMs,
   };
 }
 

frontend/src/metabase/transforms/pages/TransformListPage/TransformListPage.tsx

@@ -25,13 +25,18 @@ import CS from "metabase/css/core/index.css";
 import { DataStudioBreadcrumbs } from "metabase/data-studio/common/components/DataStudioBreadcrumbs";
 import { PageContainer } from "metabase/data-studio/common/components/PageContainer";
 import { PaneHeader } from "metabase/data-studio/common/components/PaneHeader";
-import { PLUGIN_REPLACEMENT, PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
+import {
+  PLUGIN_REPLACEMENT,
+  PLUGIN_TRANSFORM_OPTIMIZER,
+  PLUGIN_TRANSFORMS_PYTHON,
+} from "metabase/plugins";
 import { useSelector } from "metabase/redux";
 import { LockedTransformsBanner } from "metabase/transforms/components/LockedTransformsBanner/LockedTransformsBanner";
 import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";
 import { getShouldShowPythonTransformsUpsell } from "metabase/transforms/selectors";
 import { Ellipsified } from "metabase/ui";
 import {
+  Box,
   Card,
   EntityNameCell,
   Flex,
@@ -39,6 +44,7 @@ import {
   Icon,
   Stack,
   TextInput,
+  Tooltip,
   TreeTable,
   type TreeTableColumnDef,
   TreeTableSkeleton,
@@ -85,6 +91,52 @@ const NODE_ICON_COLORS: Record<TreeNode["nodeType"], ColorName> = {
 };
 
 const getNodeIconColor = (node: TreeNode) => NODE_ICON_COLORS[node.nodeType];
+
+/**
+ * "Slow" semantics for the find-slow filter.
+ *
+ * - timeout: opt-in via `includeTimedOut`. The run exceeded its allotted
+ *   budget, which is the strongest "slow" signal, but the user may want
+ *   to scope to "finished but slow" runs separately.
+ * - started (in flight): match when the elapsed wall-clock already
+ *   exceeds the threshold — a hung 6-hour run is unambiguously slow,
+ *   no point waiting for it to finish to find that out.
+ * - succeeded / failed: match when end - start ≥ threshold.
+ * - canceling / canceled: skip — user explicitly stopped it, the
+ *   duration isn't a property of the workload.
+ */
+function isTransformSlow(
+  transform: {
+    last_run?: {
+      status?: string;
+      start_time: string;
+      end_time: string | null;
+    } | null;
+  },
+  thresholdMs: number,
+  includeTimedOut: boolean,
+): boolean {
+  const run = transform.last_run;
+  if (!run?.start_time) {
+    return false;
+  }
+  if (run.status === "timeout") {
+    return includeTimedOut;
+  }
+  if (run.status === "canceling" || run.status === "canceled") {
+    return false;
+  }
+  const start = Date.parse(run.start_time);
+  if (!Number.isFinite(start)) {
+    return false;
+  }
+  const endRaw = run.status === "started" ? Date.now() : Date.parse(run.end_time ?? "");
+  if (!Number.isFinite(endRaw)) {
+    return false;
+  }
+  const elapsed = endRaw - start;
+  return elapsed >= thresholdMs;
+}
 const globalFilterFn = (
   row: { original: TreeNode },
   _columnId: string,
@@ -112,6 +164,18 @@ export const TransformListPage = ({
     Urls.extractEntityId(location.query?.collectionId) ?? null;
   const hasScrolledRef = useRef(false);
   const [searchQuery, setSearchQuery] = useState("");
+  // Threshold (in seconds) for the optional "find slow" filter — only
+  // transforms whose `last_run` duration is at least this many seconds
+  // appear in the tree. `undefined` means no threshold (show everything).
+  const [slowThresholdSec, setSlowThresholdSec] = useState<number | undefined>(
+    undefined,
+  );
+  // Whether the find-slow filter should also surface runs that hit the
+  // timeout regardless of threshold. Off by default — most "slow" hunts
+  // are looking for finished-but-slow workloads to optimise; timed-out
+  // runs are a different bucket (process killed before completion) the
+  // user can opt into.
+  const [includeTimedOut, setIncludeTimedOut] = useState(false);
   const hasPythonTransformsFeature = useHasTokenFeature("transforms-python");
   const isMeterLocked = useSetting("transforms-meter-locked");
 
@@ -145,8 +209,43 @@ export const TransformListPage = ({
 
   const warningsByTransformId = useGetTransformWarnings(transforms);
 
+  // Transforms whose `last_run` qualifies as slow. We do this client-side
+  // because the transforms list endpoint already hydrates `last_run` —
+  // see `isTransformSlow` above for the semantics (timeouts opt-in,
+  // in-flight runs counted by elapsed time, etc.).
+  const matchingSlowTransformIds = useMemo(() => {
+    if (slowThresholdSec == null) {
+      return [];
+    }
+    const minMs = slowThresholdSec * 1000;
+    return (transforms ?? []).reduce<number[]>((acc, t) => {
+      if (isTransformSlow(t, minMs, includeTimedOut)) {
+        acc.push(t.id);
+      }
+      return acc;
+    }, []);
+  }, [slowThresholdSec, includeTimedOut, transforms]);
+
+  const visibleTransforms = useMemo(() => {
+    if (slowThresholdSec == null) {
+      return transforms;
+    }
+    const matching = new Set(matchingSlowTransformIds);
+    return (transforms ?? []).filter((t) => matching.has(t.id));
+  }, [transforms, matchingSlowTransformIds, slowThresholdSec]);
+
   const treeData = useMemo(() => {
-    const data = buildTreeData(collections, transforms);
+    // When the slow filter is on, hide collections — show a flat list of
+    // matching transforms so the user can scan the hits directly. Pass
+    // `{ flat: true }` so transforms living inside a (now-hidden)
+    // collection still surface as roots; otherwise only those with a null
+    // collection_id would render and the rest would silently disappear.
+    const isFlat = slowThresholdSec != null;
+    const data = buildTreeData(
+      isFlat ? [] : collections,
+      visibleTransforms,
+      { flat: isFlat },
+    );
 
     // It will trigger the upsell modal if the feature isn't enabled.
     const shouldShowPythonLibraryRow =
@@ -169,8 +268,9 @@ export const TransformListPage = ({
     collections,
     hasPythonTransformsFeature,
     shouldShowPythonTransformsUpsell,
-    transforms,
+    slowThresholdSec,
     transformsDatabases.length,
+    visibleTransforms,
   ]);
 
   const defaultExpanded = useMemo(
@@ -316,7 +416,9 @@ export const TransformListPage = ({
   const hasNoResults = !hasNoData && treeTableInstance.rows.length === 0;
 
   const emptyMessage = hasNoData
-    ? t`No transforms yet`
+    ? slowThresholdSec != null
+      ? t`No transforms took at least ${slowThresholdSec}s on their last run.`
+      : t`No transforms yet`
     : hasNoResults && searchQuery
       ? t`No transforms found`
       : null;
@@ -341,6 +443,15 @@ export const TransformListPage = ({
             value={searchQuery}
             onChange={(e) => setSearchQuery(e.target.value)}
           />
+          {PLUGIN_TRANSFORM_OPTIMIZER.FindSlowTool && (
+            <PLUGIN_TRANSFORM_OPTIMIZER.FindSlowTool
+              thresholdSec={slowThresholdSec}
+              onThresholdChange={setSlowThresholdSec}
+              includeTimedOut={includeTimedOut}
+              onIncludeTimedOutChange={setIncludeTimedOut}
+              matchingTransformIds={matchingSlowTransformIds}
+            />
+          )}
           {transformsDatabases.length > 0 && (
             <>
               <CreateTransformMenu />
@@ -410,11 +521,29 @@ function getNameCell({
 
   const hasWarning = !!getWarningMessage();
 
+  // Optimized transforms swap the regular icon for the Sonic gif so the
+  // celebratory state is visible at-a-glance in the list. Warning icon still
+  // wins over the gif — if something is broken, that's more important to see.
+  const useSonicIcon = !hasWarning && row.original.optimized === true;
+
   return (
     <Group gap="sm" wrap="nowrap" miw={0}>
+      {useSonicIcon && (
+        <Tooltip label={t`Fully optimized`}>
+          <Box
+            component="img"
+            src="app/assets/img/sonic-gotta-go-fast.gif"
+            alt={t`Fully optimized`}
+            h={20}
+            w={20}
+            style={{ objectFit: "contain", flexShrink: 0 }}
+            data-testid="transform-optimized-badge"
+          />
+        </Tooltip>
+      )}
       <EntityNameCell
         data-testid="tree-node-name"
-        icon={hasWarning ? "warning" : row.original.icon}
+        icon={useSonicIcon ? undefined : hasWarning ? "warning" : row.original.icon}
         iconColor={hasWarning ? "warning" : getNodeIconColor(row.original)}
         name={row.original.name}
         ellipsifiedProps={{ ...getTooltipProps(getWarningMessage()) }}

frontend/src/metabase/transforms/pages/TransformListPage/types.ts

@@ -17,6 +17,7 @@ export type TreeNode = {
   collection?: Collection;
   url?: string;
   source_readable?: boolean;
+  optimized?: boolean;
   warningTooltip?: string;
 };
 

frontend/src/metabase/transforms/pages/TransformListPage/utils.ts

@@ -99,11 +99,37 @@ export function useGetTransformWarnings(transforms: Transform[] | undefined) {
 export function buildTreeData(
   collections: Collection[] | undefined,
   transforms: Transform[] | undefined,
+  options: { flat?: boolean } = {},
 ): TreeNode[] {
   if (!collections && !transforms) {
     return [];
   }
 
+  function buildTransformNode(transform: Transform): TreeNode {
+    return {
+      id: getTransformNodeId(transform.id),
+      name: transform.name,
+      nodeType: "transform",
+      icon: "transform",
+      updated_at: transform.updated_at,
+      target: transform.target,
+      owner: transform.owner,
+      owner_email: transform.owner_email,
+      transformId: transform.id,
+      source_readable: transform.source_readable,
+      optimized: transform.optimized,
+    };
+  }
+
+  // Flat mode: caller wants every supplied transform as a root-level node,
+  // ignoring its `collection_id`. Used by the "find slow" filter so a
+  // matching transform that lives inside a (now-hidden) collection still
+  // appears in the list. Without this branch, only transforms with a null
+  // collection_id would render and the rest would silently disappear.
+  if (options.flat) {
+    return (transforms ?? []).map(buildTransformNode);
+  }
+
   const transformsByCollectionId = new Map<CollectionId | null, Transform[]>();
   for (const transform of transforms ?? []) {
     const collectionId = transform.collection_id;
@@ -129,21 +155,6 @@ export function buildTreeData(
     };
   }
 
-  function buildTransformNode(transform: Transform): TreeNode {
-    return {
-      id: getTransformNodeId(transform.id),
-      name: transform.name,
-      nodeType: "transform",
-      icon: "transform",
-      updated_at: transform.updated_at,
-      target: transform.target,
-      owner: transform.owner,
-      owner_email: transform.owner_email,
-      transformId: transform.id,
-      source_readable: transform.source_readable,
-    };
-  }
-
   const rootFolders = (collections ?? []).map(buildCollectionNode);
   const rootTransforms = (transformsByCollectionId.get(null) ?? []).map(
     buildTransformNode,

frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx

@@ -10,7 +10,10 @@ import {
 } from "metabase/api";
 import { ConfirmModal } from "metabase/common/components/ConfirmModal";
 import { useMetadataToasts } from "metabase/metadata/hooks";
-import { PLUGIN_REMOTE_SYNC } from "metabase/plugins";
+import {
+  PLUGIN_REMOTE_SYNC,
+  PLUGIN_TRANSFORM_OPTIMIZER,
+} from "metabase/plugins";
 import { useSelector } from "metabase/redux";
 import { Anchor, Box, Card, Divider, Group, Stack } from "metabase/ui";
 import * as Urls from "metabase/urls";
@@ -39,6 +42,8 @@ export function RunSection({ transform, readOnly, noTitle }: RunSectionProps) {
     PLUGIN_REMOTE_SYNC.getIsRemoteSyncReadOnly,
   );
 
+  const OptimizerSection = PLUGIN_TRANSFORM_OPTIMIZER.RunPageSection;
+
   const content = (
     <>
       <Stack>
@@ -48,6 +53,12 @@ export function RunSection({ transform, readOnly, noTitle }: RunSectionProps) {
         </Group>
         <RunOutputSection transform={transform} />
       </Stack>
+      {OptimizerSection && (
+        <>
+          <Divider />
+          <OptimizerSection transform={transform} readOnly={readOnly} />
+        </>
+      )}
       <Divider />
       <Group p="lg" gap="lg">
         <Stack gap="sm">

frontend/src/metabase/transforms/utils.ts

@@ -1,4 +1,4 @@
-import { t } from "ttag";
+import { c, t } from "ttag";
 import _ from "underscore";
 
 import type { OmniPickerCollectionItem } from "metabase/common/components/Pickers/EntityPicker/types";
@@ -34,6 +34,40 @@ export function parseTimestampWithTimezone(
   }
 }
 
+/**
+ * Format a run's wall-clock duration as a compact "Xms" / "Xs" / "Xm Ys" /
+ * "Xh Ym" string. Returns null if either endpoint is missing or non-positive.
+ */
+export function formatRunDuration(
+  startTime: string | null | undefined,
+  endTime: string | null | undefined,
+): string | null {
+  if (!startTime || !endTime) {
+    return null;
+  }
+  const start = parseTimestamp(startTime);
+  const end = parseTimestamp(endTime);
+  const ms = end.diff(start);
+  if (!Number.isFinite(ms) || ms < 0) {
+    return null;
+  }
+  if (ms < 1000) {
+    return c("Sub-second run duration").t`${ms}ms`;
+  }
+  const totalSeconds = Math.round(ms / 1000);
+  if (totalSeconds < 60) {
+    return c("Run duration in seconds").t`${totalSeconds}s`;
+  }
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  if (minutes < 60) {
+    return c("Run duration in minutes and seconds").t`${minutes}m ${seconds}s`;
+  }
+  const hours = Math.floor(minutes / 60);
+  const remMinutes = minutes % 60;
+  return c("Run duration in hours and minutes").t`${hours}h ${remMinutes}m`;
+}
+
 export function formatStatus(status: TransformRunStatus | null) {
   switch (status) {
     case null:

frontend/src/metabase/urls/transforms.ts

@@ -101,6 +101,8 @@ export type TransformRunListParams = {
   startTime?: string;
   endTime?: string;
   runMethods?: TransformRunMethod[];
+  minDurationMs?: number;
+  maxDurationMs?: number;
   sortColumn?: TransformRunSortColumn;
   sortDirection?: SortDirection;
 };
@@ -113,6 +115,8 @@ export function transformRunList({
   startTime,
   endTime,
   runMethods,
+  minDurationMs,
+  maxDurationMs,
   sortColumn,
   sortDirection,
 }: TransformRunListParams = {}) {
@@ -138,6 +142,12 @@ export function transformRunList({
   runMethods?.forEach((runMethod) => {
     searchParams.append("run-methods", runMethod);
   });
+  if (minDurationMs != null) {
+    searchParams.set("min-duration-ms", String(minDurationMs));
+  }
+  if (maxDurationMs != null) {
+    searchParams.set("max-duration-ms", String(maxDurationMs));
+  }
   if (sortColumn != null) {
     searchParams.set("sort-column", sortColumn);
   }

resources/migrations/062/20260513_transform_optimized_flag.yaml

@@ -0,0 +1,22 @@
+## Add transform.optimized — set by the transform optimizer when it reports
+## an optimization_degree of 100 (no proposals worth shipping). Cleared by
+## the model's before-update hook whenever the transform's source changes.
+
+databaseChangeLog:
+  - logicalFilePath: migrations/062_update_migrations.yaml
+
+  - changeSet:
+      id: v62.2026-05-13T10:00:00
+      author: bronsa
+      comment: Add optimized flag to transform
+      changes:
+        - addColumn:
+            tableName: transform
+            columns:
+              - column:
+                  name: optimized
+                  type: ${boolean.type}
+                  defaultValueBoolean: false
+                  remarks: True when the transform optimizer last reported an optimization_degree of 100. Cleared when the transform's source changes.
+                  constraints:
+                    nullable: false

resources/transform_optimizer/prelude.md

@@ -0,0 +1,789 @@
+# Transform optimizer — system prelude
+
+You are the **Metabase Transform Optimizer**. You are given:
+
+- the SQL body of a *slow native Postgres transform*,
+- the schemas, foreign keys, and indexes of every table the SQL references,
+- the EXPLAIN plan that Postgres produced for the SQL,
+- the recent run history (durations) of this transform.
+
+Your job: produce **0 or more** equivalent-but-faster transforms. Output
+nothing else. If you find no meaningful optimization, return an empty
+proposal list with a short summary stating why — that is a valid answer.
+
+## Emit zero proposals when the query is already optimized
+
+**Reading this rule is more important than reading the rest of the
+prelude.** An empty `proposals` array is the *correct, expected, and
+celebrated* output when the input query is already in good shape.
+There is no quota, no minimum, no "look productive" expectation. The
+optimization_degree the server computes will be **100** if and only
+if you return zero proposals — that is the signal the UI uses to
+celebrate the user, and it is what the user wants to see for healthy
+transforms.
+
+A query is "already optimized" when **all** of the following are true:
+
+- The EXPLAIN plan shows no full table scans on tables > ~1M rows that
+  could be avoided by an index that is *not* already present.
+- No cosmetic-only rewrites apply (see "Canonical forms — do not
+  oscillate" below).
+- No precompute opportunity offers a measurable working-set reduction
+  (i.e. the fact-table scan dominates the cost and would not be
+  meaningfully shrunk by rolling up to a daily/cohort table).
+- The recent run history doesn't show a regression worth investigating.
+
+In that situation, **return `{"summary": "...", "proposals": []}` and
+stop**. Do not invent a `:low` rewrite to fill the array. Do not
+propose a cosmetic JOIN/IN swap. Do not propose wrapping a subquery
+in a CTE. Do not propose adding an index on a column that already has
+one. Each of those is *worse* than emitting nothing — it costs the
+user a click to dismiss, and it prevents the transform from being
+marked optimized.
+
+### Pure-formatting "rewrites" are forbidden
+
+A proposal whose `body` differs from the original SQL only in
+**whitespace, comments, casing of keywords, or trailing semicolons**
+is not an optimization. **Never** emit such a proposal — not as
+`:rewrite`, not as `:precompute` (re-labelling doesn't help), not
+under any rationale. The server filters these out post-hoc and you
+will burn an output slot for nothing. Specifically:
+
+- Removing or adding `;` at the end is not a rewrite.
+- Reflowing a query across multiple lines is not a rewrite.
+- Capitalising `select` to `SELECT` is not a rewrite.
+- Adding or removing `-- ...` or `/* ... */` comments is not a rewrite.
+
+If you find yourself writing a `body` that, when its semicolons and
+whitespace are normalised, is character-for-character identical to
+the original, **emit zero proposals instead.**
+
+The `summary` for an empty-proposal result should briefly state *why*
+nothing was found (one sentence — "no index gaps, no precompute
+opportunity, query is well-shaped"), not apologise for it.
+
+## What "equivalent" means
+
+Two transforms are equivalent when they produce the **same set of rows
+with the same multiplicity** (i.e. `EXCEPT ALL` in both directions yields
+zero). It is acceptable for the order of rows to differ unless the
+original specifies `ORDER BY`. **Equivalence trumps speedup** — a
+faster proposal that diverges in even a single edge case is worse than
+no proposal at all.
+
+You **may not**:
+
+- Change `GROUP BY` semantics, `DISTINCT` to non-`DISTINCT`, or vice versa
+  unless you can prove no row count changes.
+- Replace `count(DISTINCT x)` with `count(x)` (different semantics).
+- Drop a `WHERE` clause that the planner is "obviously" not using.
+- Introduce date functions (`NOW()`, `CURRENT_DATE`) that the original
+  didn't already have.
+
+When in doubt, prefer **adding an optimization step** (e.g. a precompute
+transform) to **changing the original semantics**.
+
+## Equivalence pitfalls — check these before proposing
+
+These are real footguns that produce *plausible-looking* rewrites that
+are not equivalent. Before proposing a rewrite, check each that applies
+and either (a) abandon the rewrite, (b) add the missing precondition to
+the rewritten SQL, or (c) downgrade the severity and document the
+precondition explicitly in `rationale`.
+
+### NULL semantics — `NOT IN` vs `NOT EXISTS` are NOT equivalent on nullable columns
+
+`x NOT IN (subquery)` returns **UNKNOWN** (i.e. row excluded) whenever
+the subquery returns *any* NULL — even for outer rows that are not in
+the non-NULL part of the subquery. `NOT EXISTS` has no such NULL trap.
+
+```sql
+-- NOT equivalent in general:
+WHERE c.id NOT IN (SELECT customer_id FROM events)        -- events.customer_id NULLable
+WHERE NOT EXISTS (SELECT 1 FROM events WHERE customer_id = c.id)
+```
+
+Only propose the `NOT IN → NOT EXISTS` rewrite when the inner column
+is **declared `NOT NULL`** in the schema you were given, **or the
+field-stats annotation shows `null=0%`** (see "Field stats annotation"
+below — fingerprints are sampled, so very low non-zero percentages
+are still risky), or the subquery explicitly filters NULLs with
+`WHERE col IS NOT NULL`. Otherwise the rewrite is unsafe.
+
+The same applies in reverse to `<> ALL` (subquery) and `IN` vs `EXISTS`
+on outer-NULL paths — be skeptical of any rewrite that touches NULL
+handling.
+
+### `DISTINCT ON` and ordering
+
+`SELECT DISTINCT ON (key) ... ORDER BY key, tiebreaker` returns a
+deterministic single row per key. If you rewrite to a window function
+or aggregate, the tiebreaker must be preserved exactly — otherwise the
+*set* of returned rows changes when there are ties.
+
+### Multi-aggregate rollups — `array_agg(DISTINCT)` + `UNNEST` is subtly different from `GROUP BY`
+
+When splitting a multi-aggregate query into a daily/hourly rollup,
+**do not** stuff distinct-value sets into `array_agg(DISTINCT ...)`
+and then `UNNEST` at the consumer level. The empty-array and NULL
+cases (e.g. a day with zero matching rows producing `NULL` for the
+array agg rather than an empty array) diverge subtly from the
+original GROUP BY semantics.
+
+Instead, emit **one rollup per independent aggregate** and join them
+at the final query level. Use `FULL OUTER JOIN` (or `COALESCE` on
+both sides) so groups that appear in only one of the rollups still
+appear in the result with `0` for the missing aggregate.
+
+```sql
+-- Risky: empty-array semantics differ from no-row semantics
+SELECT ... ARRAY_AGG(DISTINCT customer_id) FILTER (...) AS ids ... GROUP BY day
+-- Then later: LEFT JOIN LATERAL UNNEST(ids) … — divergent on empty days.
+
+-- Safe: two rollups, FULL OUTER JOIN at the final level
+WITH a AS (SELECT day, count(distinct customer_id) AS active FROM … GROUP BY day),
+     b AS (SELECT day, count(*) AS views FROM … GROUP BY day)
+SELECT COALESCE(a.day, b.day), COALESCE(a.active, 0), COALESCE(b.views, 0)
+FROM a FULL OUTER JOIN b ON a.day = b.day;
+```
+
+### Result-set type compatibility
+
+The verifier compares rows with `EXCEPT ALL`, which requires matching
+column types in matching ordinal positions. Casts are not implicit
+between integer / numeric / bigint. When your rewrite swaps a `COUNT`
+(returns `bigint`) for a `SUM` (may return `numeric`), add an explicit
+`::bigint` cast so the types align.
+
+### Top-N per group — `LATERAL` + index is a *scale-dependent* win
+
+Rewriting `ROW_NUMBER() OVER (PARTITION BY ...)` + `WHERE rn <= N` to
+a `CROSS JOIN LATERAL (... ORDER BY ... LIMIT N)` pattern only beats
+the window-function plan when:
+
+- the per-group cardinality is large (so `LIMIT N` early-terminates
+  meaningfully), and
+- an index supports the `ORDER BY ... DESC` inside the LATERAL
+  (otherwise the planner sorts per group, which is no win).
+
+If the EXPLAIN plan shows a cheap hash-join + window over a small
+intermediate, the LATERAL rewrite may *regress*. Check estimated
+group sizes from the context's `~row_count` figures and the EXPLAIN
+plan before proposing this. Downgrade severity to `low` if the gain
+is uncertain.
+
+### CTE inlining
+
+In modern Postgres (≥ 12) non-recursive CTEs are inlined by default,
+so wrapping a subquery in `WITH foo AS (...)` is a wash. In older
+Postgres versions CTEs were optimization fences. Don't claim CTE
+wrapping itself as an optimization — it isn't, unless paired with a
+real structural change.
+
+## Canonical forms — do not oscillate
+
+Postgres' planner normalises several syntactically different patterns
+to the *same* execution plan. Proposing a rewrite that swaps one of
+these forms for its sibling is **pure cosmetic churn** — there is no
+speedup, and across multiple optimize runs you can flip the same
+transform back and forth forever, never converging on "fully
+optimized". When the only change is one of the swaps below, **do not
+propose the rewrite at all** — return zero proposals instead. The
+table tells you the canonical form the optimizer prefers; treat the
+input as already optimal on this axis regardless of which form it
+arrived in.
+
+| Pair | Canonical form | Why |
+|---|---|---|
+| `WHERE x IN (SELECT y FROM t WHERE …)` *vs* `JOIN t ON t.y = x AND …` | **Whichever the user wrote.** Both compile to the same semi-join plan in Postgres. | Planner treats them identically; the swap is cosmetic. |
+| `WHERE EXISTS (SELECT 1 FROM t WHERE t.y = x AND …)` *vs* `JOIN t ON t.y = x AND …` *vs* `WHERE x IN (SELECT y …)` | **Whichever the user wrote.** All three are semi-joins. | Same plan; no speedup. |
+| `LEFT JOIN t ON … WHERE t.col IS NULL` *vs* `WHERE NOT EXISTS (SELECT 1 FROM t WHERE …)` | **`NOT EXISTS`.** | NULL-safe and idiomatic. Don't rewrite an existing `NOT EXISTS` to the LEFT JOIN form. (Note: `NOT IN` is *not* in this set — see the NULL-semantics pitfall above.) |
+| `COUNT(*) FILTER (WHERE cond)` *vs* `SUM(CASE WHEN cond THEN 1 ELSE 0 END)` | **`FILTER`.** | Same plan, more readable; don't rewrite either direction. |
+| Subquery in `FROM` *vs* non-recursive CTE | **Whichever the user wrote.** | Inlined identically since PG 12 (see CTE inlining above). |
+| Two-arg `coalesce(a, b)` *vs* `CASE WHEN a IS NULL THEN b ELSE a END` | **`coalesce`.** | Same plan; never rewrite either way. |
+
+If a proposal's only change is one of the cosmetic swaps in this
+table, drop it. A "rewrite" that merely swaps `IN (...)` for `JOIN`
+on the *same* table with the *same* predicates is not an optimization
+— the EXPLAIN plan is byte-identical.
+
+You may still propose a rewrite that *also* involves one of these
+swaps if there is a *separate* structural win driving it (different
+join order, materialised intermediate, index usage that's only
+reachable from one form because of a planner quirk in the *given*
+EXPLAIN plan). In that case, lead the `rationale` with the structural
+win — not the cosmetic swap.
+
+## Field stats annotation
+
+Fields that the SQL actually references (FROM, WHERE, JOIN, GROUP BY,
+ORDER BY, SELECT) carry a compact stats annotation in `{ … }` after
+their type / index / FK markers. Example:
+
+```
+  - occurred_at : type/DateTime [indexed] {ndv≈8421 null=0.1% time=[2024-01-01..2026-05-12]}
+  - customer_id : type/Integer [FK → shop.customers.id] {ndv≈12534 null=0%}
+  - amount      : type/Decimal {ndv≈210 null=0% range=[0.01,9999.99] avg=42.7}
+```
+
+Only fields referenced by the query get stats — unreferenced columns
+on the same table render without the `{ … }` block to save tokens.
+**Absence of stats means "not referenced," not "no data."**
+
+Stats are derived from sampled fingerprints, so treat them as
+*estimates* — `ndv≈` is rounded, `null=` is sampled (a column with
+true 0 NULLs and one with very few may both show `null=0%`), and the
+range may miss outliers. Use them for **direction**, not certainty.
+
+| Key | Meaning | Typical use |
+|---|---|---|
+| `ndv≈<int>` | Estimated number of distinct values. | Selectivity: `ndv` close to row count ⇒ near-unique ⇒ index on this column would be highly selective. Low `ndv` (≲ 100) ⇒ a B-tree index is usually *worse* than a seq-scan; consider a partial index instead. |
+| `null=<pct>%` | Sampled percentage of NULL values. | The `NOT IN` ↔ `NOT EXISTS` pitfall — see above. Also informs whether `LEFT JOIN`-with-`IS NULL` anti-join is safe. |
+| `range=[min,max]` | Sampled min/max for numeric columns. | Filter selectivity (`WHERE x > k`). Detect bounded vs unbounded growth (`id` columns up to ~current scale; surrogate keys with tight `max`). |
+| `avg=<num>` | Sampled mean. | Tail vs body decisions for aggregate strategies; rarely the deciding factor alone. |
+| `time=[earliest..latest]` | Sampled earliest/latest temporal value. | Crucial for `WHERE ts >= NOW() - INTERVAL '90 days'` style filters — tells you *what fraction of the table* survives the filter, and so whether a precompute / partial index is worth proposing. Also informs precompute granularity (daily vs hourly vs weekly). |
+
+When you cite a stat in your `rationale`, write the actual number
+(`"events.occurred_at spans 2.3 years, so the 90-day window is ~10%
+of the table"`) — that's the kind of justification that makes a
+proposal land.
+
+## Output schema
+
+**One proposal = one change.** Each proposal is either a rewrite/precompute
+(carries `body`, no `ddl_statement`) or a single index (carries
+`ddl_statement`, no `body`). Never both. This lets the user accept and
+verify each change independently.
+
+### Kind semantics — what each kind *means*
+
+| Kind | Meaning | Accept behaviour |
+|---|---|---|
+| `rewrite` | A **new shape for the original transform's query**. May replace the original transform's source in place, **or** be created as a sibling. May reference tables produced by `:precompute` proposals in the same batch (use `depends_on` to point at them). | User picks "Replace source" or "Create as new transform". |
+| `precompute` | A **brand-new intermediate transform** the user didn't have before — typically a per-day / per-customer / per-bucket rollup that other queries can read from. **Never replaces** the original. | Always "Create as new transform". |
+| `index` | A single `CREATE INDEX` against the source DB or a transform target. | Single "Run index" action. |
+
+⚠ **Common mistake**: the *leaf* of a precompute DAG — the query that
+reads from the new rollup tables and produces the final result — is a
+`rewrite` (it replaces or shadows the original transform's query),
+not a `precompute`. Only the intermediate rollup tables are
+`precompute`.
+
+### Precompute naming contract (READ THIS BEFORE WRITING ANY BODY)
+
+Every `:precompute` proposal **must** include a `target_table` field — a
+bare snake_case identifier (no schema prefix, no spaces, no quotes,
+≤ 56 chars). The BE materialises the precompute into a table at
+`<original transform's target schema>.<target_table>`. Any dependent
+proposal that reads from it **must reference that exact string** in its
+FROM clauses.
+
+Concretely, the workflow is:
+
+1. Pick a `target_table` for the precompute, e.g. `daily_events_rollup`.
+2. Use **the same identifier verbatim** in any dependent rewrite or
+   precompute body — qualified with the original transform's target
+   schema (shown in the context block, usually `public.`):
+
+   ```sql
+   FROM public.daily_events_rollup d
+   ```
+
+3. Never invent a different name in the FROM clause. Renaming between
+   proposals breaks acceptance — the precompute creates one table, the
+   rewrite SELECTs from another, Postgres returns `relation does not
+   exist`, the user's run fails.
+
+If you cannot commit to a single name across the proposals, do not emit
+the precompute split — fall back to a single-step rewrite or an index.
+
+### When to use `depends_on`
+
+`depends_on` is **a materialisation order constraint**, not a logical
+hint. Use it only when accepting the dependent proposal requires the
+listed proposals to be accepted first because the target table doesn't
+exist yet:
+
+| Situation | `depends_on` |
+|---|---|
+| Rewrite proposal that stands alone | `[]` |
+| Rewrite leaf of a precompute DAG (reads from new rollup tables) | `[<precompute ids>]` |
+| Precompute proposal | `[]` or other precomputes whose target tables it reads from |
+| Index on `source-db` (table already exists) | **`[]` always** — even if the index supports another proposal's rewrite. The user must be able to accept the index alone without dragging the rewrite into the accept. |
+| Index on `transform-target` (table doesn't exist yet) | `[<id of the proposal that creates the target table>]` |
+
+When a rewrite has a supporting source-DB index, emit them as **two
+independent proposals**, both with `depends_on: []`. The user accepts
+each one independently and the order doesn't matter. Cross-reference
+them in `rationale` text if useful — that doesn't trigger any
+materialisation coupling.
+
+Emit JSON matching exactly this shape:
+
+```json
+{
+  "summary": "one paragraph diagnosis of the original transform",
+  "proposals": [
+    {
+      "id": "p1",
+      "name": "human-readable proposal name",
+      "kind": "rewrite | index | precompute",
+      "severity": "high | medium | low",
+      "rationale": "why this is an improvement",
+      "expected_speedup": "≥100×",
+
+      /* exactly one of these two must be present: */
+      "body": "SELECT ...",        /* present for kind = rewrite | precompute */
+      "ddl_statement": {            /* present for kind = index */
+        "target": "source-db" | "transform-target",
+        "statement": "CREATE INDEX IF NOT EXISTS idx_… ON schema.table (col1, col2) ...",
+        "rationale": "what this index supports"
+      },
+
+      "target_table": "snake_case_table_name",  /* REQUIRED for kind = precompute. The literal
+                                                   bare table name the BE will materialize this
+                                                   precompute into. Dependent rewrites/precomputes
+                                                   MUST reference it as `<original schema>.<target_table>`
+                                                   verbatim. Omit for kind = rewrite | index. */
+
+      "depends_on": []   /* ids of earlier proposals this depends on */
+    }
+  ]
+}
+```
+
+The server computes the `optimization_degree` from the proposal set —
+you do not emit it.
+
+## When to propose alternatives (not just the simplest fix)
+
+"Don't pad" is **not** "don't show alternatives." When two materially
+different optimization approaches both apply to the same query, emit
+**both** as separate proposals so the user can choose. Examples:
+
+- **A rewrite *and* a supporting index.** Each can be accepted alone;
+  together they multiply. Per "one proposal = one change," that's two
+  proposals with no `depends_on` between them.
+- **A targeted rewrite *and* a covering index.** When the rewrite
+  reshapes the query (different join, eliminated subquery) AND an
+  index resolves the access pattern, emit both — the user can accept
+  one or both.
+
+What "don't pad" *does* mean: don't propose a `:low` rewrite that won't
+materially help, and don't restate the same optimization in two slightly
+different syntactic forms. Two proposals must be *meaningfully*
+different.
+
+### Do not propose precomputes by default
+
+`:precompute` proposals split one transform into N — the materialised
+rollup runs on its own schedule, then the leaf rewrite reads from it.
+That sounds like it saves work, but for transforms (which run on a
+fixed schedule, not on every dashboard view), **the rollup itself
+still has to run** end-to-end on each refresh, and now there's an
+extra transform to maintain. **In practice the precomputed pipeline
+is slower overall than the original** — you're paying the same
+fact-table scan, plus the rollup write, plus the join in the leaf.
+
+Default to **no precompute proposals**. The only situation where a
+precompute is a net win is when the leaf will be read *much* more
+often than the rollup is materialised — and that's a property of the
+*caller* (a dashboard hit live, a downstream query) not the transform
+itself. Since you can't see that from the EXPLAIN plan, you can't
+make that call.
+
+If you find yourself reaching for a precompute, **emit a `:rewrite`
+or `:index` instead**. They deliver the same access-pattern win
+without forking the schedule. Examples:
+
+| Tempting precompute | Use this instead |
+|---|---|
+| Per-day rollup of an events fact table | Covering index on `(date_trunc('day', occurred_at), …)`. |
+| Cohort dimension precompute joined repeatedly | Rewrite that wraps the cohort calc in a single CTE inside the same transform. |
+| `COUNT(DISTINCT)` per group precompute | Covering index on `(group_key, distinct_col)` so the scan is index-only. |
+| Same fact table scanned by N CTEs | Single CTE that aggregates once + a covering index on the join key. |
+
+## Severity rubric
+
+- **high**: ≥100× expected speedup, *or* removes an unbounded-time risk
+  (full-scan + cross join, NOT IN on a nullable column, `ILIKE '%…%'` on
+  a large unindexed text column).
+- **medium**: 10×–100× speedup, *or* eliminates re-computation by adding
+  one precompute step.
+- **low**: <10× speedup, cosmetic, speculative. **Do not produce `:low`
+  proposals just to look productive.** An empty proposal list is correct
+  when no real improvement is available.
+
+### Prefer indices over rewrites when both apply
+
+When the EXPLAIN plan points at a missing index, an **index proposal is
+almost always the right answer** — it's idempotent, applies in seconds,
+carries no equivalence risk, and survives schema evolution. A rewrite
+that touches the SQL is higher risk (every rewrite has to clear the
+"equivalence pitfalls" bar) and harder to audit.
+
+**Severity must be measured against the indexed baseline, not the
+current baseline.** Walk it through:
+
+1. Identify the dominant bottleneck in the EXPLAIN plan.
+2. Estimate what a covering index alone would achieve (call this the
+   *indexed baseline*).
+3. Calibrate the rewrite's severity against the indexed baseline:
+   - If the rewrite's marginal win over the indexed baseline is ≥10×,
+     it's still :high or :medium.
+   - If the marginal win is <10×, the rewrite is **:low** at most,
+     and frequently should be **dropped entirely**.
+
+### Hard ceiling rule (apply this *before* picking a severity)
+
+If any single index proposal you're emitting would alone resolve the
+dominant bottleneck in the EXPLAIN plan, then:
+
+- **Any rewrite proposal you also emit is capped at `:low`.** It cannot
+  be `:high` and almost never `:medium`. The `:high` slot is already
+  occupied by the index — the user's reading order is top-down and you
+  must not lead with a rewrite when the cheap fix is right there.
+- **If the rewrite's only mechanism is "fewer scans of the same data"
+  (e.g. collapsing correlated subqueries into a JOIN/aggregate), drop
+  it.** Once the covering index turns each subquery into an index-only
+  scan, the planner handles N of them efficiently — the rewrite is no
+  longer a structural win, just cosmetic restructuring.
+- A rewrite is only justified alongside a sufficient index when it
+  *also* delivers something the index can't: shrinking the row set
+  before a heavy aggregate, avoiding a cross-join, switching to a
+  point-lookup access pattern the index couldn't be made to support.
+
+#### Concrete examples we want you to internalise
+
+**Example A — indices alone are sufficient.** A 3-minute query becomes
+90 ms with two indices (≈2000×); a rewrite would shave it to 30 ms
+(3× over the indexed baseline). The indices are `:high`; the rewrite
+is `:low` or omitted.
+
+**Example B — correlated subqueries, covering index.** The slow query
+has three correlated subqueries against `shop.orders` per outer row,
+each doing a full seq-scan. Proposed fix #1: covering index
+`orders(customer_id) INCLUDE (...)` — each subquery becomes an
+index-only scan, the planner does N of them in microseconds, the
+query goes from minutes to tens of ms. Proposed fix #2: a rewrite
+that collapses the subqueries into a `LEFT JOIN (… GROUP BY …)`.
+**The rewrite is redundant** — once the index resolves the access
+pattern, the subqueries-vs-join shape is a wash (both touch the same
+rows). Emit the index at `:high`; **do not emit the rewrite at all.**
+
+**Example C — rewrite genuinely earns its slot.** The slow query joins
+a 100M-row fact table to itself with a window function, then filters
+the result. A covering index helps the row fetch but doesn't change
+the O(N²) work. A rewrite that adds a precompute pre-aggregating per
+day, then joins the small rollup, is the real structural win.
+Indices and rewrites both at `:high`, because the rewrite delivers a
+mechanism the index alone cannot.
+
+When you emit both, order matters too: the user reads top to bottom
+and the BE renders the index proposals first. **Lead with the index**
+in your `summary` line as well, not the rewrite.
+
+## Constraints on emitted DDL
+
+Every `:index` proposal's `ddl_statement.statement` must be a **single
+`CREATE INDEX` statement**:
+
+- Must include `IF NOT EXISTS` so accept-and-execute is idempotent.
+- For `target: "source-db"`, prefer `CONCURRENTLY` so the source table
+  isn't locked.
+- Schema-qualified table name (e.g. `shop.orders`, not just `orders`).
+- Use the schema from the referenced-tables list — do **not** invent
+  schemas.
+- Use the `idx_<table>_<cols>` naming convention so duplicates across
+  proposals are detectable.
+- When the index targets a table created by another proposal (a new
+  transform's target), set `target: "transform-target"` and use
+  `depends_on` to point at that proposal's id.
+
+You may not emit `DROP`, `ALTER`, `GRANT`, or multi-statement DDL. The
+server will reject anything else.
+
+## Writing style (HARD LIMITS — the UI shows every word)
+
+The UI renders `summary`, every proposal's `rationale`, and every
+`ddl_statement.rationale` verbatim. Verbose answers are not a sign of
+thoroughness — they make the panel unusable. Stay terse.
+
+| Field | Limit |
+|---|---|
+| `summary` | **≤ 3 sentences**, ≤ 60 words. Name the main bottleneck and that's it. **Do NOT enumerate the proposals** — the user sees each one as a card directly below; recapping is pure noise. |
+| `proposals[].rationale` | **≤ 2 sentences**, ≤ 35 words. Lead with the one-line WHY of the change. |
+| `proposals[].ddl_statement.rationale` | **≤ 1 sentence**, ≤ 20 words. |
+| `proposals[].expected_speedup` | **≤ 6 tokens**, e.g. `"≥100×"`, `"10–50× at scale"`. Not a paragraph. |
+| `proposals[].name` | A short title, ≤ 10 words. |
+
+### Things to NOT do
+
+- No "Equivalence notes (1) … (2) … (3) …" enumerations inside
+  `rationale`. If a precondition matters, drop the proposal severity to
+  `:medium` and state the precondition in **one** short clause (e.g.
+  `"Safe because orders.customer_id is NOT NULL."`).
+- No re-explaining the rewrite SQL line-by-line in the rationale — the
+  UI shows the SQL diff already.
+- No restating the EXPLAIN plan numbers in every field. Pick one place
+  (the summary) for the headline cost / runtime, in **one** number.
+- No cross-references between proposals ("Wins meaningfully only when
+  the supporting index from p2 is in place" — say it in a single
+  sub-clause if at all).
+
+### Good vs bad
+
+```text
+GOOD summary:
+  "Three correlated subqueries on shop.orders cause an O(C×N) scan.
+   No supporting indexes on customers(segment, country) or orders(customer_id).
+   Runtime ≈ 5.7 s; expected ≤ 100 ms after fixes."
+
+BAD summary (don't do this):
+  "The transform runs three correlated subqueries against shop.orders — one
+   for COUNT(*), one for SUM(total_cents) filtered by status, and one for
+   MAX(ordered_at) — for every row in shop.customers that matches
+   segment='enterprise' AND country='US'. The EXPLAIN plan confirms that
+   each subquery triggers a full sequential scan of orders (cost ~11k–13k
+   each)… [then enumerates p1, p2, p3, then concludes]"
+
+GOOD rationale:
+  "Collapses three correlated subqueries into one LEFT JOIN aggregation
+   over orders. O(C×N) → O(N+M)."
+
+BAD rationale (don't do this):
+  "The current plan executes three independent correlated subqueries per
+   customer row, each performing a full sequential scan of shop.orders
+   (~11k–13k cost units each). With 99 matching customers the planner
+   runs up to 3 × 99 = 297 full scans of orders. Replacing all three…
+   [then five sentences of equivalence notes]"
+```
+
+## Examples
+
+### Example 1 — kind: rewrite (no index, no DDL)
+
+**Slow** (correlated subqueries — Postgres re-runs each per outer row):
+
+```sql
+SELECT
+  c.id, c.name,
+  (SELECT count(*)         FROM shop.orders o WHERE o.customer_id = c.id) AS order_count,
+  (SELECT sum(o.total_cents) FROM shop.orders o WHERE o.customer_id = c.id) AS lifetime
+FROM shop.customers c;
+```
+
+**Fast** (single GROUP BY join — one pass over orders):
+
+```sql
+SELECT c.id, c.name,
+       COALESCE(agg.order_count, 0) AS order_count,
+       COALESCE(agg.lifetime,    0) AS lifetime
+FROM shop.customers c
+LEFT JOIN (
+  SELECT customer_id, count(*) AS order_count, sum(total_cents) AS lifetime
+  FROM shop.orders
+  GROUP BY customer_id
+) agg ON agg.customer_id = c.id;
+```
+
+Rationale: O(N×M) → O(N+M). High severity; ≥100× on 5M-row orders + 500k
+customers.
+
+### Example 2 — kind: index (no SQL change)
+
+**Slow** (sequential scan of orders to filter status + date):
+
+```sql
+SELECT date_trunc('month', ordered_at) AS month,
+       sum(total_cents) AS gross
+FROM shop.orders
+WHERE status IN ('shipped','delivered')
+  AND ordered_at >= NOW() - INTERVAL '12 months'
+GROUP BY 1;
+```
+
+Proposal:
+
+```json
+{
+  "id": "p1",
+  "kind": "index",
+  "severity": "high",
+  "rationale": "Index covers the filter prefix and pulls total_cents from the leaf; planner switches from seq-scan to index-only scan.",
+  "expected_speedup": "≥100×",
+  "ddl_statement": {
+    "target": "source-db",
+    "statement": "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_orders_status_ordered_at_customer ON shop.orders (status, ordered_at, customer_id) INCLUDE (total_cents);",
+    "rationale": "Covers the WHERE prefix; INCLUDE total_cents avoids heap fetches."
+  },
+  "depends_on": []
+}
+```
+
+### Example 3 — rewrite + supporting index (two proposals)
+
+**Slow** (top-N per category via a window function over 10k products
+joined to a large order_items table):
+
+```sql
+SELECT category_id, product_id, name, gross
+FROM (
+  SELECT p.category_id, p.id AS product_id, p.name,
+         sum((oi.quantity * oi.unit_price_cents) - oi.discount_cents) AS gross,
+         row_number() OVER (PARTITION BY p.category_id ORDER BY sum(...) DESC) rn
+  FROM shop.products p
+  JOIN shop.order_items oi ON oi.product_id = p.id
+  GROUP BY p.category_id, p.id, p.name
+) t
+WHERE rn <= 5;
+```
+
+**Two fully independent proposals** — both with `depends_on: []` because
+the source-DB index can be created at any time, regardless of whether the
+rewrite is in place. Cross-reference them in `rationale` text instead so
+the user understands the relationship; the wire-level `depends_on` is
+reserved for *materialisation* order (see the table above), not for hints.
+
+```json
+{
+  "proposals": [
+    {
+      "id": "p1",
+      "name": "Top-N per category via LATERAL + small revenue rollup",
+      "kind": "rewrite",
+      "severity": "medium",
+      "rationale": "Replace window function with a small per-product rollup + LATERAL top-5 per category. Wins meaningfully only when the supporting index from p2 is in place.",
+      "expected_speedup": "≥50×",
+      "body": "WITH product_revenue AS (SELECT product_id, sum((quantity * unit_price_cents) - discount_cents) AS gross FROM shop.order_items GROUP BY product_id) SELECT c.id, top.product_id, p.name, top.gross FROM shop.categories c CROSS JOIN LATERAL (SELECT pr.product_id, pr.gross FROM product_revenue pr JOIN shop.products p2 ON p2.id = pr.product_id WHERE p2.category_id = c.id ORDER BY pr.gross DESC LIMIT 5) top JOIN shop.products p ON p.id = top.product_id",
+      "depends_on": []
+    },
+    {
+      "id": "p2",
+      "name": "Index order_items(product_id) INCLUDE quantity/price/discount",
+      "kind": "index",
+      "severity": "medium",
+      "rationale": "Backs the per-product aggregation in p1's rewrite. Useful on its own for other queries too.",
+      "expected_speedup": "5–10× standalone, multiplicative with p1",
+      "ddl_statement": {
+        "target": "source-db",
+        "statement": "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_order_items_product_id ON shop.order_items (product_id) INCLUDE (quantity, unit_price_cents, discount_cents);",
+        "rationale": "Covers product_id grouping and the gross aggregate."
+      },
+      "depends_on": []
+    }
+  ]
+}
+```
+
+`p1.depends_on` is empty: the rewrite is valid SQL with or without the
+index — the user can verify it independently. p2 is also independent.
+The user can choose to accept both, just one, or neither.
+
+### Example 4 — kind: precompute (DAG split with supporting indexes)
+
+A cohort-retention query scans orders twice. Split into precompute
+transforms plus a final query — and emit each precompute index as its
+own `:index` proposal that `depends_on` the precompute it supports.
+
+```json
+{
+  "summary": "Scans orders twice. Split into per-customer cohort rollup + monthly activity rollup, plus their join indexes.",
+  "proposals": [
+    {
+      "id": "p1",
+      "name": "customer_first_purchase precompute",
+      "kind": "precompute",
+      "severity": "medium",
+      "rationale": "Compact rollup, reusable.",
+      "expected_speedup": "≥10×",
+      "target_table": "customer_first_purchase",
+      "body": "SELECT customer_id, DATE_TRUNC('month', MIN(ordered_at))::DATE AS cohort_month FROM shop.orders WHERE status IN ('paid','shipped','delivered') GROUP BY customer_id",
+      "depends_on": []
+    },
+    {
+      "id": "p2",
+      "name": "Index on customer_first_purchase(customer_id)",
+      "kind": "index",
+      "severity": "low",
+      "rationale": "Backs the join in p5.",
+      "expected_speedup": "10×",
+      "ddl_statement": {
+        "target": "transform-target",
+        "statement": "CREATE INDEX IF NOT EXISTS idx_cfp_customer_id ON public.customer_first_purchase (customer_id)",
+        "rationale": "Join key for the final query."
+      },
+      "depends_on": ["p1"]
+    },
+    {
+      "id": "p3",
+      "name": "customer_monthly_activity precompute",
+      "kind": "precompute",
+      "severity": "medium",
+      "rationale": "Distinct (customer, month) pairs.",
+      "expected_speedup": "≥10×",
+      "target_table": "customer_monthly_activity",
+      "body": "SELECT DISTINCT customer_id, DATE_TRUNC('month', ordered_at)::DATE AS activity_month FROM shop.orders WHERE status IN ('paid','shipped','delivered')",
+      "depends_on": []
+    },
+    {
+      "id": "p4",
+      "name": "Index on customer_monthly_activity(customer_id)",
+      "kind": "index",
+      "severity": "low",
+      "rationale": "Backs the join in p5.",
+      "expected_speedup": "10×",
+      "ddl_statement": {
+        "target": "transform-target",
+        "statement": "CREATE INDEX IF NOT EXISTS idx_cma_customer_id ON public.customer_monthly_activity (customer_id)",
+        "rationale": "Join key for the final query."
+      },
+      "depends_on": ["p3"]
+    },
+    {
+      "id": "p5",
+      "name": "Cohort retention from rollups",
+      "kind": "rewrite",
+      "severity": "medium",
+      "rationale": "Reads the two small rollups instead of re-scanning orders. Note the FROM clauses reference `public.customer_first_purchase` and `public.customer_monthly_activity` — the literal target_table values from p1 and p3.",
+      "expected_speedup": "≥10×",
+      "body": "SELECT c.cohort_month, a.activity_month, count(*) AS active_customers FROM public.customer_first_purchase c JOIN public.customer_monthly_activity a ON a.customer_id = c.customer_id GROUP BY c.cohort_month, a.activity_month",
+      "depends_on": ["p1", "p3"]
+    }
+  ]
+}
+```
+
+Accepting `p5` (the rewrite leaf) topo-orders the whole DAG: it creates
+`p1`/`p3` as new precompute transforms, then either creates `p5` as a
+new transform *or* — if the user picks "Replace source" — replaces the
+original transform's source with `p5.body`. The precomputes always go
+in as new transforms (replace mode only ever targets the original).
+
+## Final reminders
+
+- **Stay terse.** `summary` ≤ 3 sentences; each `rationale` ≤ 2
+  sentences; each DDL rationale ≤ 1 sentence. The UI shows everything
+  verbatim — a long blob is unreadable, not impressive. Re-read the
+  "Writing style" section before emitting.
+- An empty `proposals` list is the right answer for an already-optimal
+  transform. Don't pad.
+- Lean on the indexes that **already exist** before proposing new ones —
+  the referenced-tables block lists every existing index.
+- **One change per proposal.** Never bundle a rewrite with an index in
+  the same proposal — split them so each can be tested and accepted
+  independently.
+- The `EXPLAIN` plan tells you what the planner is *currently* doing —
+  use it to spot full scans, missing-index scans, and wide hashes.
+- **Walk every proposal through the "Equivalence pitfalls" checklist
+  before emitting it.** If any pitfall applies and the precondition
+  isn't met, either fix the proposal (e.g. add `WHERE col IS NOT NULL`
+  to the inner subquery for a `NOT IN` rewrite) or drop it entirely.
+  A user-visible "✗ DIFF — 30 rows differ" from the verifier is a
+  worse outcome than no proposal.
+- When equivalence depends on a schema-level precondition (e.g. `NOT
+  NULL` on a referenced column, an existing unique constraint), state
+  the precondition explicitly in the `rationale` field so the user can
+  audit it. Treat that case as **at most `:medium`** severity, even if
+  the expected speedup is high.

src/metabase/transforms/models/transform.clj

@@ -120,7 +120,8 @@
     (collection/check-allowed-content :model/Transform new-collection))
   ;; The target db is recomputed when source changes because for MBQL transforms,
   ;; the source query's :database is the source of truth for the target database.
-  (let [target-changed? (or (:source (t2/changes transform)) (:target (t2/changes transform)))
+  (let [source-changed? (some? (:source (t2/changes transform)))
+        target-changed? (or source-changed? (:target (t2/changes transform)))
         target-db-id    (when target-changed?
                           ;; No database existence check added here, unlike for insert.
                           ;; Just allow updates for an invalid target to fail.
@@ -130,6 +131,12 @@
       (assoc :source_type (transforms-base.u/transform-source-type source)
              :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))
 
+      ;; Source actually changed in this update → any prior "fully optimized"
+      ;; verdict no longer applies. Gated on (t2/changes) so plain optimized-flag
+      ;; updates from optimize! aren't clobbered back to false.
+      source-changed?
+      (assoc :optimized false)
+
       target-changed?
       (assoc :target_db_id target-db-id)
 

src/metabase/transforms/models/transform_run.clj

@@ -193,10 +193,23 @@
     :transform-name [:transform [:= :transform_run.transform_id :transform.id]]
     nil))
 
+(defn- duration-ms-sql
+  "Raw SQL fragment that evaluates to (end_time - start_time) in milliseconds
+  for the current appdb driver. Returns NULL when end_time is NULL, so
+  in-progress runs are excluded from any duration filter."
+  []
+  (case (mdb/db-type)
+    :postgres "EXTRACT(EPOCH FROM (end_time - start_time)) * 1000"
+    :mysql    "TIMESTAMPDIFF(MICROSECOND, start_time, end_time) / 1000"
+    :h2       "DATEDIFF('MILLISECOND', start_time, end_time)"))
+
 (defn- paged-runs-where-clause
   "Builds a `:where` clause for transform runs from the given filter parameters."
-  [{:keys [start-time end-time run-methods transform-ids transform-tag-ids statuses user-id]}]
-  (let [where-cond (cond-> []
+  [{:keys [start-time end-time run-methods transform-ids transform-tag-ids statuses
+           min-duration-ms max-duration-ms user-id]}]
+  (let [duration-expr (when (or (some? min-duration-ms) (some? max-duration-ms))
+                        [:raw (duration-ms-sql)])
+        where-cond (cond-> []
                      (some? start-time)
                      (conj (timestamp-constraint :start_time start-time))
 
@@ -222,6 +235,12 @@
                           (nil? (next statuses)))
                      (conj [:= :is_active true])
 
+                     (some? min-duration-ms)
+                     (conj [:>= duration-expr [:inline (long min-duration-ms)]])
+
+                     (some? max-duration-ms)
+                     (conj [:<= duration-expr [:inline (long max-duration-ms)]])
+
                      (some? user-id)
                      (conj [:= :user_id user-id]))]
     (when (seq where-cond)

src/metabase/transforms_rest/api/transform.clj

@@ -80,6 +80,7 @@
    [:source_type :keyword]
    [:source_database_id {:optional true} [:maybe pos-int?]]
    [:source_readable {:optional true} [:maybe :boolean]]
+   [:optimized {:optional true} [:maybe :boolean]]
    [:entity_id [:maybe :string]]
    [:created_at :any]
    [:updated_at :any]
@@ -210,6 +211,8 @@
     [:start-time {:optional true} [:maybe ms/NonBlankString]]
     [:end-time {:optional true} [:maybe ms/NonBlankString]]
     [:run-methods {:optional true} [:maybe (ms/QueryVectorOf [:enum "manual" "cron"])]]
+    [:min-duration-ms {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]
+    [:max-duration-ms {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]
     [:user-id {:optional true} [:maybe ms/PositiveInt]]]]
   (api/check-data-analyst)
   (-> (transforms.core/paged-runs (assoc query-params

transform_optimizer/PLAN.md

@@ -0,0 +1,457 @@
+# Transform Optimizer — High-Level Plan
+
+## Goal
+
+Given a slow native (Postgres-only, for now) transform, propose one or more
+*equivalent* transforms that produce the same result faster. Most proposals are
+a single rewritten transform; some proposals split work across a DAG of
+transforms (precompute + final).
+
+Pipeline (in the end-state):
+
+```
+slow transform ──┐
+                 ├─► context builder ──► metabot agent ──► streamed { proposals[]    ──► UI
+schema/FK/idx ───┤                       (LLM + prelude        + optimization_degree }   (accept / verify
+EXPLAIN plan ────┘                        of examples)                                    / discard)
+```
+
+The streamed response carries two things:
+
+1. **Proposals** — 0..N alternative transforms (single rewrites, or DAGs
+   when split-for-precompute is warranted). Each carries a kind, rationale,
+   expected speedup, and severity. A proposal-free response is a valid
+   answer for an already-optimal transform.
+2. **Optimization degree** — a 0–100 score answering "how optimized is the
+   *current* transform?" Computed deterministically from the proposals
+   (see Phase 2). 100 ⇔ proposals is empty ⇔ "no further optimizations
+   found". This is shown first in the UI so the user immediately knows
+   whether to bother reading proposals.
+
+## Scope (this branch)
+
+- Source DB engine: **Postgres only**.
+- Source transform type: **native** (`{:type :query, :query {:native …, :database N}}`).
+- Output of a proposal: **new** transforms, never an update to the original.
+  Original stays around.
+- Verification is **optional**, user-triggered, and runs as a side workflow.
+
+## Architecture sketch
+
+### Touch points in the existing code
+
+From the survey (see `dataset/` for the dataset; references below are the
+files we'll hook into):
+
+- `metabase.transforms-base.query/run-query-transform!` — where the source SQL
+  for a `:query` transform is compiled. We do *not* modify execution; we just
+  re-use `compile-source` to get the SQL string for context.
+- `metabase.transforms-rest.api.transform` — where the new endpoint
+  `POST /api/transform/:id/optimize` will live.
+- `metabase.metabot.tools.deftool` + `metabase.metabot.tools` — where we
+  register the new `propose-transform-optimizations` tool.
+- `metabase.metabot.agent.core/run-agent-loop` — reused as-is for streaming.
+- `metabase.metabot.api/native-agent-streaming-request` — the streaming
+  contract we ride on (AI-SDK v4 SSE).
+- `metabase-enterprise.transforms-inspector.context/build-context` — **already**
+  assembles a per-transform context map (source/target tables, field metadata,
+  join structure, column mappings) for LLM consumption. We *extend* it: add
+  detailed index info, an `EXPLAIN` block, and recent-run durations.
+- `metabase-enterprise.transforms-inspector.query-analysis/analyze-native-query`
+  — **already** extracts referenced tables/columns from a native SQL string
+  via Macaw + `sql-tools`. We call it directly; we do not write our own parser.
+- `metabase.sync.fetch-metadata/index-metadata` and
+  `driver/describe-table-indexes` — existing index metadata sources used as
+  inputs/fallbacks. Full Postgres index shape comes from new catalog queries.
+
+### New code
+
+Roughly:
+
+```
+src/metabase/transform_optimizer/
+  core.clj                  ; public entry: optimize-transform!
+  index_introspection.clj   ; Postgres catalog queries for composite indexes,
+                            ;  INCLUDE columns, predicates, and index types
+  explain.clj               ; run EXPLAIN (FORMAT JSON, VERBOSE) safely
+  context.clj               ; wraps transforms-inspector.context/build-context,
+                            ;  adds indexes + EXPLAIN + run history
+  prelude.clj               ; embed the curated pre→post examples corpus
+  scoring.clj               ; proposals → optimization_degree (deterministic)
+  verify.clj                ; bounded pre/post equivalence check
+  ddl/
+    parse.clj               ; validate CREATE INDEX statements (allowlist)
+    execute.clj             ; run a single DDL on the right connection
+                            ;  (autocommit handling for CONCURRENTLY)
+  models/
+    proposal.clj            ; persisted proposal (1..N proposed transforms,
+                            ;  + optimization_degree, + ddl_statements,
+                            ;  + per-DDL execution status)
+  api.clj                   ; POST /api/transform/:id/optimize + GET proposals
+                            ; + POST /…/proposal/:pid/ddl/:did/execute
+                            ; + POST /…/proposal/:pid/accept
+
+src/metabase/metabot/tools/
+  transform_optimizer.clj   ; deftool wrapping core/optimize-transform!
+
+resources/transform_optimizer/
+  prelude.md                ; rendered into the system prompt
+  examples/                 ; .sql pairs that feed the prelude
+
+frontend/src/metabase/transform_optimizer/
+  …                         ; panel on the transform page that streams
+                            ;  the optimization_degree + proposals
+```
+
+We do **not** write our own SQL parser, but we do write Postgres-specific
+index introspection:
+- Table / column / FK info comes from the appdb (`Table`, `Field` with
+  `fk_target_field_id`) which sync already populates.
+- "Is this column indexed?" comes from `Field.database_indexed`.
+- "What's the full shape of the index (composite columns, INCLUDE list,
+  partial predicate, GIN/BRIN type)?" is the piece sync does *not*
+  persist. The existing `driver/describe-table-indexes` default is too
+  lossy for this feature, so `index_introspection.clj` queries Postgres
+  catalogs directly and treats `driver/describe-table-indexes` as a fallback.
+- Native SQL parsing reuses `transforms-inspector.query-analysis`.
+
+## Phased delivery
+
+### Phase 0 — Foundation (this commit)
+
+- Postgres dataset + DDL + seed (`dataset/`) sized so the slow queries reliably
+  span multiple seconds on a developer laptop.
+- A corpus of slow → fast pairs (`queries/`) covering distinct optimization
+  patterns. These do double duty: regression fixtures for the verifier *and*
+  raw material for the prelude.
+
+### Phase 1 — Read-only backend: context builder
+
+The context we feed the LLM is mostly assembled from existing utilities.
+What we add is index detail, EXPLAIN output, and recent-run timing.
+
+- **Reuse `transforms-inspector.context/build-context`** for the bulk of the
+  payload: source tables (with `column_count`, fingerprint-derived field
+  stats), target table, join structure, column-to-column mappings between
+  the transform output and its inputs.
+- **Reuse `transforms-inspector.query-analysis/analyze-native-query`** to
+  obtain the set of `{table-id, column-id}` references for a native SQL
+  body. The optimizer never invents its own SQL parser.
+- **FKs and "is this column indexed?"** come for free from the appdb:
+  `Field.fk_target_field_id` (a Toucan2 hydration step) and the
+  `Field.database_indexed` boolean (populated by `sync.sync-metadata.indexes`).
+- **`index_introspection.clj`** is the only new introspection code we write.
+  For Postgres, it queries `pg_index`, `pg_class`, `pg_namespace`,
+  `pg_attribute`, `pg_am`, and `pg_get_expr` at request time for each
+  referenced table to recover the shape sync drops on the floor:
+  composite key column order, `INCLUDE` payload, partial predicates,
+  uniqueness, and index type (btree / gin / brin / hash). Cached for the
+  duration of one optimizer request. If catalog introspection fails, it
+  falls back to the simplified `driver/describe-table-indexes` output and
+  marks the context as partial rather than pretending full index detail is
+  available.
+- **`explain.clj`**: `EXPLAIN (FORMAT JSON, VERBOSE)` against the source
+  DB. No `ANALYZE` by default — running the slow query is exactly what
+  we're trying to avoid. `:analyze?` opt-in flag for advanced callers.
+- **Approximate row counts**: prefer `Table` fingerprint row count when
+  fresh; fall back to `pg_class.reltuples` if missing. (We only need an
+  order-of-magnitude figure for the LLM.)
+- Unit-test all of the above against the seeded Postgres from Phase 0.
+
+### Phase 2 — Prompt + Metabot tool
+
+- `prelude.clj` reads `resources/transform_optimizer/prelude.md` and the
+  examples directory; renders into the system prompt.
+- `context.clj` builds the per-request payload:
+  ```
+  ## Transform
+  - id, name, target table
+  ## SQL
+  ```sql … ```
+  ## Referenced tables
+    table A — columns, FKs, indexes, ≈row count
+    …
+  ## EXPLAIN
+  ```json … ```
+  ## Run history
+  - last N runs: started_at, duration_ms, status
+  ```
+- `deftool` `propose-transform-optimizations` accepts `{transform_id}` and
+  streams a structured payload:
+  ```
+  {optimization_degree    ; integer in [0, 100]
+   summary                ; one-paragraph diagnosis
+   proposals [{id              ; stable per-stream id
+               name
+               kind            ; see below
+               severity        ; :high | :medium | :low
+               rationale
+               expected_speedup
+               body            ; SQL of the new transform, nil for pure-DDL
+               depends_on []   ; ids of sibling proposals in the same DAG
+               ddl_statements  ; 0..N — see "DDL statements" subsection
+                 [{id
+                   target       ; :source-db | :transform-target
+                                ;  | {:precompute-of <sibling proposal id>}
+                   statement    ; CREATE INDEX … (validated server-side)
+                   rationale}]}]}
+
+  kind     ∈ #{:rewrite           ; new body, no DDL
+               :index             ; no body change, DDL only
+               :rewrite+index     ; new body and DDL
+               :precompute}       ; DAG; DDL on precompute targets is common
+  severity ∈ #{:high :medium :low}  ; required on every proposal
+  ```
+  We instruct the LLM to annotate proposed indexes with any requirements
+  or caveats it depends on. We also explicitly allow `proposals` to be empty
+  when the transform is already optimal.
+
+#### DDL statements
+
+The LLM may decide that the best optimization is a new index — either on
+a source-DB table (q04: monthly revenue) or on a transform's
+target/precompute target (q07: cohort rollups). For this branch, DDL is a
+first-class **advisory** proposal artifact, not something Metabase executes.
+The UI shows the statement, rationale, target, and caveats so the user can
+review or copy it.
+
+**Constraints applied to every emitted statement** (enforced in
+`ddl/parse.clj`, not trusted from the LLM):
+
+- Must be a single `CREATE INDEX` statement (parsed; reject `DROP`,
+  `ALTER`, `GRANT`, multi-statement, etc.).
+- Must include `IF NOT EXISTS` so accept is idempotent.
+- For `target = :source-db`, prefer `CONCURRENTLY` so we don't lock the
+  source table out from under other readers. The LLM prompt requests
+  this; the validator does not require it.
+- Fully-qualified schema + table name. The schema must be one of those
+  reachable from the transform's referenced tables.
+- Index name follows a convention (`idx_<table>_<cols>_<hash>`) so we
+  can detect duplicates the LLM may suggest across proposals.
+
+**Execution semantics** (`ddl/execute.clj`):
+
+- One statement runs against one connection. `CREATE INDEX CONCURRENTLY`
+  cannot run inside a transaction; we use a fresh connection with
+  `setAutoCommit(true)` for that path.
+- Failures are surfaced to the UI with the Postgres error message;
+  partial execution of a multi-DDL proposal does *not* roll back the
+  succeeded statements (idempotent re-execution is the recovery path).
+- Each executed statement is recorded on the `Proposal` row
+  (`ddl_status[ddl_id] = :executed | :failed | :skipped`).
+
+**DDL targeting `transform-target`** has a known wrinkle: the standard
+transform flow recreates the target table on each run, so an index
+created post-materialization would be lost. We address this in two
+ways:
+
+1. The optimizer prefers proposing precompute splits as **incremental**
+   transforms (`:type :table-incremental`) when the inner SQL is
+   amenable. Incremental targets retain their indexes across runs.
+2. For non-incremental targets we register the DDL as
+   *post-materialization* on the new transform — the transform pipeline
+   runs the `CREATE INDEX IF NOT EXISTS` statements after each
+   materialization. Cheap if nothing changed; rebuilds when the table is
+   recreated. (Open question if this is enough for very large precompute
+   targets — see Open Questions.)
+
+#### Optimization degree
+
+The score is **deterministic** given the proposal set — the LLM does
+*not* emit it directly, so the same proposals always yield the same
+score and we don't have to babysit LLM calibration:
+
+```
+optimization_degree =
+  100 - clamp(Σ weight(severity_i), 0, 100)
+
+weight(:high)   = 30
+weight(:medium) = 15
+weight(:low)    =  5
+```
+
+Properties this gives us:
+
+- No proposals ⇒ score is 100 (the LLM saying "I looked and found
+  nothing worth changing"). The UI shows "✓ already optimized".
+- Three high-severity proposals ⇒ 10 (lots of room).
+- Two medium proposals ⇒ 70 ("there's some room, but it's not on fire").
+- Score depends only on what the LLM proposes — not on what it *says*.
+
+The LLM is given an explicit severity rubric in the system prompt:
+
+- **high**: ≥100× speedup expected, or removes an unbounded-time risk
+  (e.g. full-table LIKE-search).
+- **medium**: 10–100× speedup, or eliminates re-computation by adding
+  a precompute step.
+- **low**: <10× speedup, cosmetic rewrite, or speculative.
+
+**Severity cap on conditional rewrites** — when a proposed rewrite is
+only equivalent under a schema-level precondition (e.g. inner column is
+`NOT NULL` for a `NOT IN → NOT EXISTS` rewrite, or the partition key is
+unique for a particular precompute split), the LLM must state the
+precondition in `rationale` and cap severity at `:medium` regardless of
+expected speedup. The prelude lists the recurring equivalence pitfalls
+the optimizer has hit so far (NULL semantics, multi-aggregate rollup
+divergence, top-N LATERAL scale-dependence, COUNT-vs-SUM type
+mismatch).
+
+#### Streaming order
+
+The agent emits, in order:
+
+1. `summary` (one paragraph, lands quickly so the user has something to
+   read while proposals generate)
+2. `proposals[*]` one at a time (each as its own SSE event/data part)
+3. Final `optimization_degree`, computed from the accumulated proposals
+   when the LLM signals end-of-output. Computed *server-side* in
+   `scoring.clj` — not trusted from the model.
+
+### Phase 3 — UI
+
+- Button on the transform page: "Suggest optimizations".
+- Side panel opens with two regions:
+  - **Header**: a large optimization-degree dial (e.g. `73 / 100`,
+    colour-coded). While streaming, it shows a "Analyzing…" spinner;
+    when the LLM emits its summary the spinner becomes the summary
+    paragraph; when streaming ends, the dial animates to the computed
+    score. If the score is 100, the panel collapses to
+    "✓ Already optimized — nothing to suggest" and the proposal list
+    is hidden.
+  - **Proposals list**: each proposal renders as it arrives, as a card
+    with: severity badge, kind tag, diff (or full new SQL), rationale,
+    expected speedup.
+    - When `ddl_statements` is non-empty, an "Index changes" subsection
+      lists each statement with its rationale, the formatted DDL, and
+      per-statement state: `pending → running → executed | failed`.
+      Failed statements show the Postgres error inline with a "Retry"
+      button.
+    - Per-card actions:
+      - **Run DDL** — executes the proposal's DDL statements only.
+        Useful when the user wants the index but isn't ready to commit
+        to the rewrite. Pure-`:index` proposals: this is the primary
+        action.
+      - **Accept** — for `:rewrite` / `:rewrite+index`: creates the new
+        transform(s), then runs the DDL. For `:precompute`: creates the
+        whole DAG (in `depends_on` order), then runs the per-target DDL
+        (incremental targets up front, post-materialization DDL stored
+        on the transform).
+      - **Verify** — opens the equivalence-check flow (Phase 4).
+      - **Dismiss** — drops the proposal locally; can be re-fetched.
+- "Accept" creates N new transforms (linked through `depends_on`) and routes
+  the user to the first one. Original transform is untouched. Executed
+  DDL is not auto-reverted on dismiss — CREATE INDEX is durable on
+  purpose; the panel surfaces the resulting index name so the user can
+  drop it manually if they change their mind.
+
+### Phase 4 — Equivalence verification (optional, user-triggered)
+
+For a single-transform proposal, first compare the output schemas. Column
+count, names, and compatible database types must match before row comparison
+starts.
+
+Then materialize both sides into bounded temp tables using scratch target
+names, with a timeout and cleanup in `finally`. The row comparison should
+preserve duplicate row multiplicity. On Postgres, prefer `EXCEPT ALL`:
+
+```sql
+SELECT count(*) FROM (
+  (SELECT * FROM <slow_result>  EXCEPT ALL SELECT * FROM <fast_result>)
+  UNION ALL
+  (SELECT * FROM <fast_result>  EXCEPT ALL SELECT * FROM <slow_result>)
+) d;
+```
+
+Surface 0 rows = equivalent; otherwise show a row-sample diff. Also record
+both durations so the user sees the *actual* speedup vs the LLM's estimate.
+Queries using volatile functions such as `now()` or `random()` are marked
+with a verification caveat because repeated execution can produce different
+results even when a rewrite is logically valid.
+
+For DAG proposals, verification is deferred until DAG creation/acceptance is
+designed.
+
+### Phase 5 — Polish & guardrails
+
+- **Permission check for proposing**: the optimizer can always *propose*
+  DDL — but the proposal API marks each `ddl_statement` with
+  `advisory_only?`. We do *not* hide the proposals — read-only users can
+  still see what their team should run.
+- **DDL validator** (`ddl/parse.clj`): runs on every statement before
+  it's shown, regardless of how it got into the system. Reject
+  anything that isn't `CREATE INDEX [CONCURRENTLY] [IF NOT EXISTS]` on a
+  schema-qualified table belonging to the referenced set. This is the
+  primary defence against prompt injection coercing the LLM into
+  emitting an `ALTER USER … SUPERUSER`.
+- **Cost guard**: skip / warn on transforms whose latest run took longer
+  than a threshold (we should not run EXPLAIN ANALYZE against the
+  30-minute job).
+- **Telemetry**: log proposal acceptance rate and measured speedups; this
+  feeds the prelude over time.
+
+## Open questions (deferred)
+
+- Are some optimizations safe to apply automatically (e.g. `NOT IN` →
+  `NOT EXISTS` when the column is nullable)? **Settled: no.** The
+  rewrite is unsafe on nullable columns and the verifier confirmed
+  this in practice — see "Lessons learned" below. Auto-apply would
+  silently corrupt user-visible results.
+- DAG persistence: do proposed transforms form a real linked sub-DAG in the
+  jobs system, or are they just N independent transforms the user wires up?
+  Lean toward the latter for hackathon.
+- Multi-database: do we limit proposals to indexes/extensions the user
+  *actually has*, or just suggest and let them fail? Currently: suggest with a
+  clear "requires …" annotation.
+- Score calibration: do we want to factor *measured* (verifier-run) speedups
+  back into the score after Phase 4, so accepted proposals retroactively
+  re-grade the score? Lean no — the score should reflect the *as-presented*
+  diagnosis, not become a moving target.
+- Should the severity rubric also factor in the EXPLAIN cost ratio between
+  the slow and proposed plans, so the LLM has a less subjective anchor?
+  Worth a small experiment in Phase 2.
+- Post-materialization DDL on very large precompute targets: rebuilding a
+  GIN index on each transform run is expensive. Two ways out: (a) push
+  the user toward incremental transforms for these cases (we already
+  prefer this in the proposal), or (b) ship a smarter materialization
+  strategy ("CREATE TABLE … AS … then CREATE INDEX … once" vs. "TRUNCATE
+  + INSERT" to keep indexes). Out of scope this branch.
+
+## Lessons learned (folded into the prelude)
+
+These are concrete failure modes the verifier flagged during Phase-0
+harness shake-down. Each is now encoded in
+`resources/transform_optimizer/prelude.md` as an "equivalence pitfall"
+the LLM must check before emitting a rewrite — and locked in by
+anchor-string tests in `prelude_test.clj`:
+
+1. **`NOT IN → NOT EXISTS` is unsafe on nullable columns** (NULL in the
+   subquery turns the `NOT IN` outer comparison into UNKNOWN ⇒ excluded
+   row). Only safe when the inner column is `NOT NULL`-declared or the
+   subquery filters NULLs explicitly.
+2. **Multi-aggregate rollup via `array_agg(DISTINCT) FILTER` + `UNNEST`**
+   diverges from the original `GROUP BY` on empty-array / NULL days.
+   Prefer multiple independent rollups joined `FULL OUTER JOIN` at the
+   final query.
+3. **Top-N per group via `LATERAL` + index is scale-dependent** —
+   regresses against the window-function plan when per-group cardinality
+   is small. Severity should reflect what the EXPLAIN plan actually says.
+4. **Type compatibility** between `COUNT(bigint) → bigint` and
+   `SUM(bigint) → numeric` will trip `EXCEPT ALL`; the LLM must add
+   explicit casts when swapping these.
+5. **CTE inlining** in modern Postgres (≥12) makes "wrap in `WITH foo`"
+   a no-op — don't claim CTE wrapping itself as an optimization.
+
+Severity policy: any rewrite whose equivalence depends on a schema-level
+precondition (e.g. `NOT NULL`, uniqueness, sortedness) caps at
+`:medium` regardless of expected speedup, and the precondition must be
+stated explicitly in the proposal's `rationale`.
+
+## Phase 0 deliverables (this commit)
+
+- `dataset/01_schema.sql` — DDL, no helpful indexes, foreign keys retained.
+- `dataset/02_seed.sql` — bulk-load via `generate_series`; configurable scale.
+- `dataset/03_optimized_indexes.sql` — additional indexes referenced by some
+  optimized versions (kept separate so we can A/B with and without).
+- `queries/00_index.md` — table of pairs with one-line summary.
+- `queries/qNN_*.sql` — one file per pair with `-- @slow` / `-- @fast`
+  sections and a metadata header.

transform_optimizer/SUMMARY.md

@@ -0,0 +1,417 @@
+# Transform Optimizer — Implementation Status & Open Tasks
+
+Companion to [`PLAN.md`](./PLAN.md). PLAN describes *what* we're building;
+this doc tracks *where we are*. Keep it current — it's the parallel-work
+coordination surface.
+
+## Status by phase
+
+### Phase 0 — Foundation ✅ **done**
+
+| Artifact | Path | Notes |
+|---|---|---|
+| DDL | `dataset/01_schema.sql` | `shop` schema; FKs kept, no helper indexes |
+| Seed | `dataset/02_seed.sql` | 50k customers, 500k orders, 1.5M order_items, 2M events, 250k reviews — full harness run (load → verify) ~1–1.5 min on a dev laptop |
+| Optimized indexes | `dataset/03_optimized_indexes.sql` | btree + `pg_trgm` GIN; applied between Phase-2-slow and Phase-4-fast in the harness |
+| 8 slow→fast query pairs | `queries/qNN_*.sql` | `-- @meta` / `-- @slow` / `-- @fast` markers, one file per pair |
+| Validation harness | `harness/run.sh`, `harness/run.clj` | Loads schema+seed, runs all pairs, EXCEPT-ALL equivalence check, geomean speedup. Heartbeat every 3 s. |
+
+### Phase 1 — Read-only context builder ✅ **done**
+
+All under `enterprise/backend/src/metabase_enterprise/transform_optimizer/`:
+
+| Namespace | File | What it does |
+|---|---|---|
+| `index-introspection` | `index_introspection.clj` | Single-round-trip Postgres catalog query → full index shape (composite keys, INCLUDE, partial predicate, access method, raw `pg_get_indexdef`). Accepts either a flat `[schema table]` pair or a sequence of pairs. Falls back to nil on non-Postgres drivers. |
+| `explain` | `explain.clj` | `EXPLAIN (FORMAT JSON, VERBOSE)`. No `ANALYZE` by default. Postgres only. |
+| `context` | `context.clj` | Top-level builder. Wraps `transforms-inspector.context/build-context`, resolves FK edges via `Field.fk_target_field_id`, attaches indexes + EXPLAIN + last-10 runs with durations. |
+
+### Phase 2 — Prompt + LLM wiring ✅ **done**
+
+| Namespace | File | What it does |
+|---|---|---|
+| `prelude` | `prelude.clj` + `resources/transform_optimizer/prelude.md` | Static prelude (output schema, severity rubric, DDL constraints, 4 worked examples — one per `kind`). Loaded once and cached. |
+| `prompt` | `prompt.clj` | Renders context map to markdown for the user-side of the prompt. |
+| `scoring` | `scoring.clj` | Pure `proposals → optimization_degree` per the rubric (`100 − Σ severity_weights`; high=30, med=15, low=5). |
+| `ddl.parse` | `ddl/parse.clj` | Allowlist validator. Strips strings/comments, rejects multi-statement, rejects forbidden keywords, regex-matches canonical `CREATE INDEX`, asserts schema-qualified target is in referenced-tables set. |
+| `llm` | `llm.clj` | Calls `metabot.self/call-llm-structured` with our prelude+context as the user message and the proposals JSON schema. Inherits the metabot client's retry / telemetry / provider abstraction. Default model: `anthropic/claude-sonnet-4-6`. |
+| `core` | `core.clj` | Public entry `optimize!`. Stitches everything together. Server-side proposal post-processing (DDL validation tag + scoring) is wired in. |
+| `api` | `api.clj` | HTTP endpoints mounted at `/api/ee/transform-optimizer`. |
+
+### Phase 3 — UI ✅ **walking skeleton done** (mock-aware, BE-wired)
+
+All under `enterprise/frontend/src/metabase-enterprise/transform_optimizer/` unless noted:
+
+| Layer | Path | What it does |
+|---|---|---|
+| OSS plugin slot | `frontend/src/metabase/plugins/oss/transform-optimizer.ts` | New `PLUGIN_TRANSFORM_OPTIMIZER` with `RunPageSection` injection point. Re-exported + reinitialized in `frontend/src/metabase/plugins/index.ts`. |
+| OSS host | `frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx` | Renders `PLUGIN_TRANSFORM_OPTIMIZER.RunPageSection` between the run output and the tags row when the EE plugin registers a section. |
+| EE plugin init | `index.ts` | Gated on `transforms-python` premium feature; registers `TransformOptimizerSection` in the OSS slot. |
+| Types | `types/index.ts` | `Proposal`, `DdlStatement`, `OptimizerStreamEvent`, `OptimizerStreamState` matching the wire contract below. |
+| SSE client | `api/stream.ts` | `runOptimizerStream({transformId, analyze, signal, onEvent})`. Hand-rolled SSE parser (`event:` / `data:` framing, multi-chunk frames, comments, malformed JSON tolerated). Uses the legacy `api` client for basename + auth headers. Maps non-OK responses to a single synthetic `error` event. |
+| Mutations | `api/index.ts` | RTK Query mutations on `EnterpriseApi`: `useVerifyProposalMutation`, `useAcceptProposalMutation`. |
+| Stream hook | `hooks/use-optimizer-stream.ts` | Reducer-driven `{state, start, abort, reset, dismissProposal}`. Dedupes proposals by id, auto-cancels on unmount and on re-`start`, falls back to `done` if the server never emits a terminal event. |
+| Dial | `components/OptimizationDegreeDial/` | SVG ring with the 4 colour buckets per the table below. Spinner while streaming; the section hides the dial entirely on error (avoids double-rendering the error next to the alert). |
+| Proposal card | `components/ProposalCard/ProposalCard.tsx` | Severity badge, kind tag, expected-speedup chip, SQL `<Code block>`, accept/verify/dismiss actions with permission tooltips. |
+| Index changes | `components/ProposalCard/IndexChangesSection.tsx` | Per-DDL row: target label, validation badge (accepted / rejected with reason), execution-status badge (pending/running/executed/failed/skipped — only `pending` is reachable while DDL is advisory), statement code block, rationale, error detail. |
+| Section orchestrator | `components/TransformOptimizerSection/TransformOptimizerSection.tsx` | Trigger button (Suggest / Stop / Re-analyze), dial, summary paragraph, error `<Alert>` with Retry, proposal list. Collapses to "✓ Already optimized" when stream ends with score 100 and zero proposals. |
+| Unit test | `api/stream.unit.spec.ts` | 4 cases: ordered event parsing, chunk-split frames, HTTP-error → synthetic error event, malformed/comment frames ignored. All pass. |
+
+Maps to the FE task list:
+
+| Task | Status | Notes |
+|---|---|---|
+| FE-1 trigger button | ✅ | `<TriggerButton>` in `TransformOptimizerSection.tsx`. |
+| FE-2 section under RunSection | ✅ | Injected via the new `PLUGIN_TRANSFORM_OPTIMIZER` slot. |
+| FE-3 optimization-degree dial | ✅ | Colour buckets per "Severity → score mapping" table. Streaming spinner; collapse-to-✓ handled by the section, not the dial. |
+| FE-4 summary text region | ✅ | Renders below the dial as soon as the `summary` event arrives. Suppressed while erroring. |
+| FE-5 proposal card | ✅ | Severity, kind, rationale, expected speedup, SQL block. |
+| FE-6 index-changes subsection | ✅ | DDL state + validation status with rejection reason inline. |
+| FE-7 per-card actions | ✅ | Wired to `useVerifyProposalMutation` / `useAcceptProposalMutation`; dismiss is local. Buttons disabled-with-tooltip when `readOnly`. |
+| FE-8 stream error rendering | ✅ | Inline `<Alert color="error">` + Retry; toasts handled by `useMetadataToasts` for accept/verify failures. Dial is hidden in error state so the error renders once. |
+
+### Phase 4 — Equivalence verification 🟡 **partial (single-transform only)**
+
+| Namespace | File | What it does | Status |
+|---|---|---|---|
+| `verify` | `verify.clj` | Materialises slow + proposal SQL into a scratch schema, schema-compatibility check, `EXCEPT ALL` diff in both directions, sample-diff capture. Postgres only. | ✅ single proposals |
+| | | Precompute (DAG) verification — materialise each precompute body and rewrite the leaf's `<pN target>` references before EXCEPT-ALL. | ⛔ deferred (see C3) |
+
+### Phase 5 — Polish & guardrails 🟡 **partial**
+
+- ✅ DDL validator (`ddl/parse.clj`) runs on every emitted statement.
+- ✅ Accept executes every `:validation = :accepted` `CREATE INDEX` statement against the source DB on an autocommit connection (so `CONCURRENTLY` works). Per-statement status (`:executed | :failed | :skipped`) is returned in `ddl_statements`. Rejected statements pass through with their rejection reason so the UI can render the failure inline.
+- ⛔ Permission-checking against source-DB write perms (only `read-check` so far).
+- ⛔ Cost guard (skip-EXPLAIN-ANALYZE on long-running transforms).
+- ⛔ Telemetry on acceptance / measured speedups beyond what metabot-self already exports.
+
+---
+
+## Open BE tasks
+
+### Done
+
+| # | Task | Notes |
+|---|---|---|
+| BE-1 | ✅ Wire LLM call | `llm.clj` calls `metabot.self/call-llm-structured` with the structured-output schema. Default model `anthropic/claude-sonnet-4-6`. Prelude + context concatenated into one user message (call-llm-structured has no separate system slot). |
+| BE-2 | ✅ HTTP streaming endpoint | `api.clj` POST `/api/ee/transform-optimizer/:id/optimize` emits the SSE events per the **Streaming endpoint** contract below. Buffered today (LLM call completes before any event is emitted); upgrade to true streaming is **F3**. |
+| BE-4 | ✅ Verify endpoint (single proposal) | `verify.clj` materialises both sides into a scratch schema and EXCEPT-ALL diffs both directions. Precompute (DAG) verification deferred — see **C3**. |
+| BE-5 | ✅ Accept endpoint | `accept.clj` creates a transform per proposal that has a body; pure-`:index` proposals are skipped. DDL returned as advisory. DAG body-substitution still missing — see **C1**. |
+| BE-7 | ✅ `clj-kondo` module config | `enterprise/transform-optimizer` registered; `inspector.context` + `inspector.query-analysis` added to `enterprise/transforms-inspector`'s `:api` set. |
+
+### Correctness / completeness — ship-blockers
+
+These surfaced when I reviewed the just-written code; they're the gaps a
+real user would hit on day one.
+
+| # | Where | What | Effort |
+|---|---|---|---|
+| **C1** | `accept.clj` | **DAG body-substitution.** When `p3.body` says `… FROM <p1 target> …`, accept must replace `<p1 target>` with the actual `schema.table` of the just-created sibling. Without this, precompute accept silently emits broken transforms. | ~45 min |
+| **C2** | `accept.clj` | **Topological-order check + target-name collision detection.** Verify caller-given proposal order matches `depends_on`, reject duplicate target names within the batch, and refuse if a target name already exists in the destination DB. | ~30 min |
+| **C3** | `verify.clj` | **DAG verification.** Currently returns 422 on `kind = precompute`. Extend to materialise each precompute body to a scratch table, substitute its name into the leaf, then EXCEPT-ALL. | ~1 hr |
+| **C4** | `verify.clj` | **Scratch-schema isolation.** Two concurrent `/verify` calls both write `transform_optimizer_verify.slow` and clobber each other. Switch to a per-request scratch identifier (`slow_<uuid>` / `fast_<uuid>` or a randomly-named schema). | ~20 min |
+| **C5** | `api_routes/routes.clj` | **Correct feature flag.** We piggyback on `:transforms-python` as a placeholder. Define a proper `:transform-optimizer` premium feature in `metabase.premium-features` and switch the `premium-handler` over. | ~15 min |
+| **C6** | `accept.clj` | **Body sanity check** before insert. Reject empty / whitespace-only / unparseable `body` per-proposal rather than 500-ing inside `create-transform!`. | ~15 min |
+
+### Hardening
+
+| # | Where | What |
+|---|---|---|
+| **H1 / BE-9** | `api.clj` accept | Write-permission check on the target DB. Today only `read-check` runs on the source transform; a read-only user can call `/accept` and 500 in `create-transform!`. |
+| **H2 / BE-10** | `core.clj` | Cost guard: if the latest `transform_run.duration_ms` exceeds N minutes, decline `EXPLAIN ANALYZE` and the `/verify` materialise path. |
+| **H3** | `api.clj` optimize | Streaming-endpoint deadline. `call-llm-structured` retries internally but the HTTP request has no overall timeout — a wedged provider holds the connection forever. |
+| **H4** | `llm.clj` | Validate the LLM's structured response against the malli schema before passing to `finalise-proposals`. A schema-violating response today flows straight through and may NPE downstream. |
+| **H5** | `llm.clj` | Promote `model` / `temperature` to `defsetting` so the LLM can be tuned per-tenant without code edits. |
+| **H6** | all `api.clj` endpoints | Telemetry events: `transform-optimizer/proposed`, `/accepted`, `/verified` with severities, kinds, and measured speedups. Feeds the prelude over time. |
+| **H7** | `verify.clj` | Wrap each `DROP TABLE` in the `cleanup!` path in its own try/catch + log; today a failing cleanup leaks the scratch table for the rest of the session. |
+| **H8** | `api.clj` accept | Audit-log who accepted which proposal IDs for which transform. |
+| **H9** | `core.clj` | `source-tables-readable?` precheck (mirror `transforms-inspector`). User can read the transform but might lack read on a referenced table; today the optimizer would still call the LLM with their schema metadata. |
+
+### Future / deferred
+
+| # | Task | Depends on |
+|---|---|---|
+| **F1 / BE-6** | Persistence model `:model/TransformOptimizerProposal` (Liquibase migration + Toucan2 model). Stores the streamed payload + per-DDL `:validation` tags. | — |
+| **F2 / BE-3** | `GET /api/ee/transform-optimizer/:id/proposals` (fetch persisted, most-recent-first). | F1 |
+| **F3** | True incremental SSE streaming — parse Claude's `tool-input` deltas live and emit `summary` / `proposal` events as each JSON object completes, instead of buffering the full LLM response. | — |
+| **F4** | Post-accept verify — optionally run `verify.clj` against each just-created transform and include the measured speedup in the accept response. | C1, C3 |
+| **F5 / BE-8** | Integration tests for verify / accept / streaming endpoint against the seeded Postgres from the harness. Pure unit tests already cover scoring / parse / prompt / prelude / core. | — |
+| **F6** | Score-from-EXPLAIN anchor — feed slow vs proposed EXPLAIN cost ratios into the severity rubric so the LLM has a less subjective anchor (per PLAN open questions). | — |
+
+### Recommended next pass
+
+In this order — each step is largely independent of the next:
+
+1. **C5** (feature flag — 15 min, removes wrong-feature gating).
+2. **C4** (scratch isolation — 20 min, prevents corruption under concurrent verify).
+3. **H4** (validate LLM response — 20 min, prevents downstream NPEs).
+4. **C1 + C2 + C3** (full DAG support across accept + verify — together ~1.5–2 hr, unlocks the precompute kind end-to-end).
+5. **H1** (write-perm check at accept — 15 min, security hygiene).
+
+
+## Open FE tasks
+
+FE-1..FE-8 are landed (see Phase 3 above). What's left:
+
+| # | Task | Depends on | Notes |
+|---|---|---|---|
+| FE-9 | Empty-state polish | — | First-visit affordance before the user clicks "Suggest optimizations" (currently the section is just the trigger row). |
+| FE-10 | "Already optimized" collapse animation | — | Cosmetic; the panel currently swaps to the Alert without a transition. |
+---
+
+## Streaming endpoint — wire contract
+
+Backend exposes one endpoint that streams the optimizer's output. **Frontend
+should mock against this contract until BE-1 + BE-2 land.** Mockable with
+Mirage / MSW; sample fixture below.
+
+### Request
+
+```
+POST /api/ee/transform-optimizer/:id/optimize
+Accept: text/event-stream
+Content-Type: application/json
+```
+
+Body (optional):
+
+```json
+{
+  "analyze": false   // forwarded to EXPLAIN; off by default
+}
+```
+
+### Response
+
+`text/event-stream` (SSE). The response emits a sequence of events in this
+order:
+
+1. **One** `summary` event (lands quickly; gives the user something to read
+   while proposals are generated).
+2. **Zero or more** `proposal` events, one per proposal, in DAG-topological
+   order (`depends_on` references always precede their dependants).
+3. **Exactly one** `done` event at end-of-stream carrying the deterministic
+   `optimization_degree` (computed server-side from the accumulated
+   proposals — *not* trusted from the LLM).
+4. If anything fails mid-stream: a single `error` event, then the stream
+   closes. Partial proposals already emitted remain valid.
+
+#### Event: `summary`
+
+```
+event: summary
+data: {"text": "Three correlated subqueries in the SELECT list cause an O(C×N) scan over orders. Two precompute opportunities and one DDL index would close the gap."}
+```
+
+#### Event: `proposal`
+
+```
+event: proposal
+data: {
+  "id": "p1",
+  "name": "Use GROUP BY rollup instead of correlated subqueries",
+  "kind": "rewrite",                       // "rewrite" | "index" | "rewrite+index" | "precompute"
+  "severity": "high",                       // "high" | "medium" | "low"
+  "rationale": "Postgres re-runs each subquery once per outer row. A GROUP BY rollup is one pass.",
+  "expected_speedup": "≥100×",
+  "body": "SELECT c.id, c.name, ...",       // null for kind = "index"
+  "depends_on": [],                          // proposal ids this depends on (DAG)
+  "ddl_statements": [
+    {
+      "id": "ddl1",
+      "target": "source-db",                 // "source-db" | "transform-target" | { "precompute-of": "<sibling-id>" }
+      "statement": "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_orders_customer_id ON shop.orders (customer_id);",
+      "rationale": "Backs the GROUP BY rollup.",
+      "validation": "accepted",              // "accepted" | "rejected" — server-side allowlist result
+      "index_name": "idx_orders_customer_id",
+      "rejection": null                      // present only when validation = "rejected": { "reason": "<kw>", "detail": "..." }
+    }
+  ]
+}
+```
+
+#### Event: `done`
+
+```
+event: done
+data: { "optimization_degree": 73 }
+```
+
+#### Event: `error`
+
+```
+event: error
+data: { "message": "Source database is not Postgres; optimizer only supports Postgres in this branch.", "retryable": false }
+```
+
+### Severity → score mapping (for FE colour coding)
+
+The dial colour should follow the same buckets the BE computes from:
+
+| Score range | Suggested colour | Meaning |
+|---|---|---|
+| 100 | green | Already optimized; panel collapses |
+| 70 – 99 | green-yellow | Minor room |
+| 40 – 69 | orange | Real wins available |
+| 0 – 39 | red | Multiple high-severity issues |
+
+### Sample full stream (mock fixture)
+
+```
+event: summary
+data: {"text": "Three correlated subqueries scan orders once per customer. A single GROUP BY rollup eliminates the fan-out."}
+
+event: proposal
+data: {"id":"p1","name":"Rewrite to GROUP BY rollup","kind":"rewrite","severity":"high","rationale":"Eliminates O(C×N) fan-out.","expected_speedup":"≥100×","body":"SELECT c.id, c.name, COALESCE(agg.order_count, 0) ...","depends_on":[],"ddl_statements":[]}
+
+event: proposal
+data: {"id":"p2","name":"Index supporting the rollup","kind":"index","severity":"medium","rationale":"Speeds the GROUP BY pass; useful for other transforms.","expected_speedup":"5–10×","body":null,"depends_on":[],"ddl_statements":[{"id":"d1","target":"source-db","statement":"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_orders_customer_id ON shop.orders (customer_id);","rationale":"Backs customer_id grouping.","validation":"accepted","index_name":"idx_orders_customer_id","rejection":null}]}
+
+event: done
+data: {"optimization_degree":55}
+```
+
+---
+
+## Other endpoints (request/response only — no streaming)
+
+### Verify
+
+```
+POST /api/ee/transform-optimizer/:id/proposal/verify
+```
+
+Request body — just the proposal id from the streamed `proposal` event:
+
+```json
+{ "proposal_id": "p1" }
+```
+
+The full proposal payload (body, kind, depends_on, ddl_statements) is
+resolved server-side from the optimizer's per-user proposal cache,
+populated by the streaming `/optimize` endpoint. The cache TTL is
+1 hour and is process-local — if the FE gets a `404` here, the panel
+should silently re-run `/optimize`.
+
+Response (200):
+
+```json
+{
+  "equivalent": true,
+  "slow_duration_ms": 18314,
+  "fast_duration_ms":   94,
+  "speedup":  194.8,
+  "diff_rows": 0,
+  "sample_diff": null               // when equivalent=false, ≤10 row sample (each direction)
+}
+```
+
+Response (404) when the proposal id is no longer cached:
+
+```json
+{ "error": "proposal_not_found", "detail": "...re-run /optimize..." }
+```
+
+Response (422) for known failure modes (e.g. schema mismatch — different
+column count between slow and fast):
+
+```json
+{ "error": "schema_mismatch", "detail": "fast has 4 columns, slow has 5" }
+```
+
+### Accept
+
+```
+POST /api/ee/transform-optimizer/:id/proposal/accept
+```
+
+Creates the proposal's transforms (1 for `rewrite` / `index`, N for
+`precompute`) in `depends_on` order. The DDL list is returned for the
+user to inspect / copy — Metabase does **not** execute DDL in this branch.
+
+Request body — just the proposal ids from the streamed `proposal` events,
+in dependency order (single rewrites: one id; precompute DAGs: N ids,
+roots first):
+
+```json
+{
+  "proposal_ids":  ["p1", "p2", "p3"],
+  "collection_id": 42                    // optional, defaults to the source transform's collection
+}
+```
+
+Server resolves each id against the optimizer's per-user proposal
+cache. If any id is missing, the **entire request** fails with `404
+proposal_not_found` listing the missing ids — there is no partial
+accept.
+
+Response (200):
+
+```json
+{
+  "created_transforms": [
+    { "id": 88,  "name": "Original — customer_first_purchase",   "proposal_id": "p1", "kind": "precompute", "depends_on": [] },
+    { "id": 89,  "name": "Original — customer_monthly_activity", "proposal_id": "p2", "kind": "precompute", "depends_on": [] },
+    { "id": 90,  "name": "Original — cohort_retention",          "proposal_id": "p3", "kind": "precompute", "depends_on": ["p1","p2"] }
+  ],
+  "ddl_statements": [
+    {
+      "id":            "ddl1",
+      "proposal_id":   "p1",
+      "statement":     "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_cfp_customer_id ON shop.customer_first_purchase (customer_id);",
+      "target":        { "precompute-of": "p1" },
+      "rationale":     "Join key for the final query.",
+      "validation":    "accepted",
+      "index_name":    "idx_cfp_customer_id",
+      "status":        "executed"      // "executed" | "failed" | "skipped"
+    },
+    {
+      "id":            "ddl2",
+      "proposal_id":   "p2",
+      "statement":     "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_o ON shop.orders (customer_id)",
+      "target":        "source-db",
+      "validation":    "accepted",
+      "index_name":    "idx_o",
+      "status":        "failed",
+      "error_message": "ERROR: permission denied for table orders"
+    },
+    {
+      "id":            "ddl3",
+      "proposal_id":   "p3",
+      "statement":     "CREATE INDEX bad ON public.users (id)",
+      "validation":    "rejected",
+      "rejection":     { "reason": "unknown-table", "detail": "public.users is not in the referenced-tables set" }
+                       // no `status` — rejected statements never reach execution
+    }
+  ],
+  "skipped_proposals": []   // proposal ids that had no SQL body (kind=index); their DDL still runs via ddl_statements above
+}
+```
+
+Failed DDL does **not** roll back successfully-created transforms.
+`CREATE INDEX IF NOT EXISTS` is idempotent, so the user can re-run accept
+after fixing whatever the problem was (typically a permission grant).
+
+---
+
+## Parallelisation notes
+
+- **FE walking skeleton is live** against the wire contract. The SSE parser
+  in `api/stream.ts` is contract-driven and decoupled from any specific
+  backend implementation — keep the contract stable and the panel will keep
+  working if BE event shapes change downstream.
+- **Verify/accept** are wired to RTK Query against `EnterpriseApi`. Until
+  BE-6 (persistence) lands, the panel re-runs the optimizer on each open;
+  proposals are kept in component state for the lifetime of the page.
+- **BE-4 (verify)** is independent — can be built and tested in
+  isolation against the existing query pairs (Phase 0 harness already
+  validates exactly the EXCEPT-ALL equivalence form we'll use).
+- **BE-7 (module config)** has no dependencies; can be done at any time
+  but should land *before* the first CI run that touches kondo.
+
+## Where to read more
+
+- [`PLAN.md`](./PLAN.md) — design rationale, scope, open questions
+- [`harness/README.md`](./harness/README.md) — running the validation harness
+- [`queries/00_index.md`](./queries/00_index.md) — table of pairs
+- `resources/transform_optimizer/prelude.md` — the static LLM system prompt

transform_optimizer/dataset/01_schema.sql

@@ -0,0 +1,118 @@
+-- Transform Optimizer — base schema (Postgres).
+--
+-- Design intent
+-- =============
+-- This schema models a mid-size e-commerce / SaaS analytics stack at a scale
+-- where naive SQL takes multiple seconds on a developer laptop. It is paired
+-- with `02_seed.sql` (data) and `03_optimized_indexes.sql` (the indexes some
+-- of the optimized queries rely on).
+--
+-- We deliberately *omit* useful indexes on the base schema:
+--   * orders has no index on customer_id or ordered_at
+--   * order_items has no index on order_id or product_id
+--   * events has no index on customer_id, occurred_at, or type
+--   * reviews has no index on product_id, customer_id, or body
+--   * products has no index on category_id or supplier_id
+-- Foreign keys are kept because we want the introspection layer to be able to
+-- read them from pg_constraint, and the LLM prelude assumes FK info is
+-- available. Postgres does NOT auto-index the referencing side of an FK, so
+-- the indexes-missing-on-the-fact-table story stays intact.
+--
+-- Target scale (full):
+--   customers     500 000
+--   suppliers       2 000
+--   categories         30
+--   products       20 000
+--   orders      5 000 000
+--   order_items 15 000 000   (≈ 3 items per order)
+--   events     30 000 000
+--   reviews     2 000 000
+-- The seed script honors a scale knob; see its header.
+
+BEGIN;
+
+DROP SCHEMA IF EXISTS shop CASCADE;
+CREATE SCHEMA shop;
+SET search_path TO shop, public;
+
+-- ----------------------------------------------------------------------------
+-- Dimensions
+-- ----------------------------------------------------------------------------
+
+CREATE TABLE customers (
+  id           BIGINT PRIMARY KEY,
+  email        TEXT NOT NULL,
+  name         TEXT NOT NULL,
+  country      TEXT NOT NULL,
+  segment      TEXT NOT NULL,                              -- free | pro | enterprise
+  signed_up_at TIMESTAMPTZ NOT NULL
+);
+
+CREATE TABLE suppliers (
+  id      BIGINT PRIMARY KEY,
+  name    TEXT NOT NULL,
+  country TEXT NOT NULL,
+  active  BOOLEAN NOT NULL DEFAULT TRUE
+);
+
+CREATE TABLE categories (
+  id        BIGINT PRIMARY KEY,
+  name      TEXT NOT NULL,
+  parent_id BIGINT REFERENCES categories(id)
+);
+
+CREATE TABLE products (
+  id          BIGINT PRIMARY KEY,
+  name        TEXT NOT NULL,
+  category_id BIGINT NOT NULL REFERENCES categories(id),
+  supplier_id BIGINT NOT NULL REFERENCES suppliers(id),
+  price_cents BIGINT NOT NULL,
+  cost_cents  BIGINT NOT NULL,
+  active      BOOLEAN NOT NULL DEFAULT TRUE,
+  description TEXT NOT NULL
+);
+
+-- ----------------------------------------------------------------------------
+-- Facts
+-- ----------------------------------------------------------------------------
+
+CREATE TABLE orders (
+  id            BIGINT PRIMARY KEY,
+  customer_id   BIGINT NOT NULL REFERENCES customers(id),
+  ordered_at    TIMESTAMPTZ NOT NULL,
+  status        TEXT NOT NULL,                             -- placed | paid | shipped | delivered | cancelled | refunded
+  total_cents   BIGINT NOT NULL,
+  currency      TEXT NOT NULL DEFAULT 'USD',
+  shipping_country TEXT NOT NULL
+);
+
+CREATE TABLE order_items (
+  id              BIGINT PRIMARY KEY,
+  order_id        BIGINT NOT NULL REFERENCES orders(id),
+  product_id      BIGINT NOT NULL REFERENCES products(id),
+  quantity        INT    NOT NULL,
+  unit_price_cents BIGINT NOT NULL,
+  discount_cents  BIGINT NOT NULL DEFAULT 0
+);
+
+CREATE TABLE events (
+  id          BIGINT PRIMARY KEY,
+  customer_id BIGINT REFERENCES customers(id),             -- nullable: anonymous traffic
+  occurred_at TIMESTAMPTZ NOT NULL,
+  type        TEXT NOT NULL,                               -- page_view | add_to_cart | checkout_start | purchase | search
+  product_id  BIGINT REFERENCES products(id),
+  session_id  TEXT NOT NULL,
+  url_path    TEXT NOT NULL,
+  properties  JSONB NOT NULL DEFAULT '{}'::jsonb
+);
+
+CREATE TABLE reviews (
+  id          BIGINT PRIMARY KEY,
+  product_id  BIGINT NOT NULL REFERENCES products(id),
+  customer_id BIGINT NOT NULL REFERENCES customers(id),
+  rating      INT NOT NULL CHECK (rating BETWEEN 1 AND 5),
+  body        TEXT NOT NULL,
+  created_at  TIMESTAMPTZ NOT NULL
+);
+
+COMMIT;

transform_optimizer/dataset/02_seed.sql

@@ -0,0 +1,216 @@
+-- Transform Optimizer — seed data.
+--
+-- Loads roughly:
+--   customers       50 000   suppliers   2 000   categories      30
+--   products        10 000   orders    500 000   order_items ≈ 1 500 000
+--   events       2 000 000   reviews    250 000
+--
+-- Sizing rationale
+-- ================
+-- The dataset is sized so each un-optimized slow query lands in the 0.5–3 s
+-- range and the optimized versions stay well below 100 ms — a clean
+-- 10×–500× ratio that demonstrates the optimisation without making the run
+-- painful. Full developer-laptop run (seed → all 8 pairs → indexes → fast
+-- pairs → compare) finishes in about a minute.
+--
+-- The biggest constraint is q01 (NOT IN antijoin) and q02 (three correlated
+-- subqueries). Both need their hash subplans to fit in `work_mem`; without
+-- that Postgres falls back to per-row re-scans and the slow versions go
+-- from seconds to many minutes. The harness already sets
+-- `work_mem=128MB` on its connection — keep that or scale the dataset
+-- down further.
+--
+-- If you want a bigger dataset for stress-testing, multiply the
+-- generate_series stop values below — but expect q01/q02 to be the
+-- bottleneck.
+--
+-- Generation strategy
+-- ===================
+-- We use `generate_series` + `INSERT INTO … SELECT` for fast bulk loading.
+-- Randomness is derived deterministically from row indices (`(i * P) % M`)
+-- so reruns produce stable data. We tag a small fraction of `reviews.body`
+-- with hot keywords so the trigram-search example has a believable selectivity.
+-- After loading, ANALYZE all tables — the optimizer (and our EXPLAIN output)
+-- relies on accurate statistics.
+
+\set ON_ERROR_STOP on
+
+BEGIN;
+SET search_path TO shop, public;
+SET LOCAL synchronous_commit = OFF;        -- big speedup on bulk loads, safe within tx
+SET LOCAL work_mem = '256MB';
+SET LOCAL maintenance_work_mem = '512MB';
+
+-- ----------------------------------------------------------------------------
+-- Reference dimensions (tiny)
+-- ----------------------------------------------------------------------------
+
+INSERT INTO suppliers (id, name, country, active)
+SELECT
+  i,
+  'Supplier ' || i,
+  (ARRAY['US','UK','DE','FR','IT','ES','BR','JP','CA','AU','NL','SE','PL','MX','IN'])[1 + (i % 15)],
+  (i % 50) <> 0                                       -- ~2% inactive
+FROM generate_series(1, 2000) AS i;
+
+INSERT INTO categories (id, name, parent_id)
+SELECT
+  i,
+  'Category ' || i,
+  CASE WHEN i <= 5 THEN NULL ELSE 1 + ((i * 7) % 5) END
+FROM generate_series(1, 30) AS i;
+
+INSERT INTO customers (id, email, name, country, segment, signed_up_at)
+SELECT
+  i,
+  'user' || i || '@example.com',
+  'Customer ' || i,
+  -- country is driven by `(i / 100) % 15` rather than `i % 15` so it
+  -- decorrelates from segment (which depends on `i % 100`). With the latter,
+  -- gcd(15, 100) = 5 means segment='enterprise' AND country='US' is empty.
+  (ARRAY['US','UK','DE','FR','IT','ES','BR','JP','CA','AU','NL','SE','PL','MX','IN'])[1 + ((i / 100) % 15)],
+  CASE
+    WHEN (i % 100) < 80 THEN 'free'
+    WHEN (i % 100) < 97 THEN 'pro'
+    ELSE 'enterprise'
+  END,
+  TIMESTAMPTZ '2021-01-01 00:00:00+00'
+    + (((i * 1009) % (365 * 4 * 24 * 60)) || ' minutes')::interval
+FROM generate_series(1, 50000) AS i;
+
+INSERT INTO products (id, name, category_id, supplier_id, price_cents, cost_cents, active, description)
+SELECT
+  i,
+  'Product ' || i,
+  1 + ((i * 13) % 30),
+  1 + ((i * 17) % 2000),
+  500 + ((i * 31) % 50000),                            -- $5 — $505
+  200 + ((i * 23) % 20000),                            -- cost < price on average
+  (i % 25) <> 0,                                       -- ~4% inactive
+  CASE (i % 7)
+    WHEN 0 THEN 'A sturdy, well-reviewed product. Customers report fast delivery.'
+    WHEN 1 THEN 'Best seller. Often discounted in seasonal promotions.'
+    WHEN 2 THEN 'Premium tier. Includes a 2-year warranty.'
+    WHEN 3 THEN 'Replacement part. Compatible with most models.'
+    WHEN 4 THEN 'Refurbished unit. Tested and certified.'
+    WHEN 5 THEN 'Out of stock in some regions. Refunds available.'
+    ELSE        'New arrival. Limited initial inventory.'
+  END
+FROM generate_series(1, 10000) AS i;
+
+-- ----------------------------------------------------------------------------
+-- Orders (300k)
+--
+-- We bias `ordered_at` slightly toward recent dates and `customer_id` so that
+-- a handful of customers are heavy buyers (zipf-ish).
+-- ----------------------------------------------------------------------------
+
+INSERT INTO orders (id, customer_id, ordered_at, status, total_cents, currency, shipping_country)
+SELECT
+  i,
+  -- skew: 70% of orders go to the first 10k customers
+  CASE WHEN (i % 10) < 7
+       THEN 1 + ((i * 2654435761) % 10000)
+       ELSE 1 + ((i * 2654435761) % 50000)
+  END,
+  -- Orders span 2023-06-01 → ~2026-06-01 so that "last 12 months" filters
+  -- relative to today's NOW (≈ 2026-05) land inside the data.
+  TIMESTAMPTZ '2023-06-01 00:00:00+00'
+    + ((((i::bigint * 1019) + ((i * 7) % 9973)) % (365 * 3 * 24 * 60)) || ' minutes')::interval,
+  (ARRAY['placed','paid','paid','paid','shipped','shipped','delivered','delivered','delivered','cancelled','refunded'])[1 + (i % 11)],
+  500 + ((i * 97) % 80000),                            -- $5 — $805
+  (ARRAY['USD','USD','USD','EUR','GBP','BRL','JPY'])[1 + (i % 7)],
+  (ARRAY['US','UK','DE','FR','IT','ES','BR','JP','CA','AU','NL','SE','PL','MX','IN'])[1 + ((i * 11) % 15)]
+FROM generate_series(1, 500000) AS i;
+
+-- ----------------------------------------------------------------------------
+-- Order items (≈ 1.5M)
+--
+-- 1–5 items per order, deterministic. We use a row_number() global counter to
+-- assign primary keys.
+-- ----------------------------------------------------------------------------
+
+INSERT INTO order_items (id, order_id, product_id, quantity, unit_price_cents, discount_cents)
+SELECT
+  row_number() OVER (),
+  o.id,
+  1 + ((o.id * 7 + g) % 10000),
+  1 + ((g * 13 + o.id) % 5),
+  100 + ((o.id * 31 + g) % 10000),
+  CASE WHEN ((o.id + g) % 12) = 0 THEN ((g * 17) % 2000) ELSE 0 END
+FROM orders o,
+     LATERAL generate_series(1, 1 + (o.id % 5)) AS g;
+
+-- ----------------------------------------------------------------------------
+-- Events (2M)
+--
+-- Mostly anonymous page_views; ~15% authenticated; ~3% purchases.
+-- ----------------------------------------------------------------------------
+
+INSERT INTO events (id, customer_id, occurred_at, type, product_id, session_id, url_path, properties)
+SELECT
+  i,
+  CASE WHEN (i % 100) < 15
+       THEN 1 + ((i * 2654435761) % 50000)
+       ELSE NULL
+  END,
+  -- Events span 2024-06-01 → ~2026-06-01 so "last 90 days" relative to
+  -- today's NOW (≈ 2026-05) lands inside the data.
+  TIMESTAMPTZ '2024-06-01 00:00:00+00'
+    + (((i::bigint * 31) % (365 * 2 * 24 * 60 * 60)) || ' seconds')::interval,
+  (ARRAY['page_view','page_view','page_view','page_view','page_view','page_view','page_view',
+         'search','search','add_to_cart','add_to_cart','checkout_start','purchase'])[1 + (i % 13)],
+  CASE WHEN (i % 5) = 0 THEN 1 + ((i * 7) % 10000) ELSE NULL END,
+  'sess_' || ((i / 30) % 1000000),                     -- ≈30 events per session
+  '/' || (ARRAY['home','products','search','cart','checkout','account','help','blog'])[1 + (i % 8)]
+    || '/' || ((i * 11) % 10000),
+  jsonb_build_object('referrer', (ARRAY['google','direct','email','ad','social'])[1 + (i % 5)])
+FROM generate_series(1, 2000000) AS i;
+
+-- ----------------------------------------------------------------------------
+-- Reviews (250k)
+--
+-- We salt review bodies with hot keywords ("refund", "broken", "amazing",
+-- "fast delivery") at a low rate so the trigram-search example is realistic.
+-- ----------------------------------------------------------------------------
+
+INSERT INTO reviews (id, product_id, customer_id, rating, body, created_at)
+SELECT
+  i,
+  1 + ((i * 7) % 10000),
+  1 + ((i * 2654435761) % 50000),
+  1 + (i % 5),
+  CASE (i % 20)
+    WHEN 0  THEN 'Arrived broken. Had to request a refund — process took two weeks.'
+    WHEN 1  THEN 'Absolutely amazing product. Will buy again.'
+    WHEN 2  THEN 'Fast delivery and well packaged.'
+    WHEN 3  THEN 'Not what I expected based on the description. Considering a refund.'
+    WHEN 4  THEN 'Broken on arrival. Refund granted, no issues.'
+    WHEN 5  THEN 'Great value for the price. Five stars.'
+    WHEN 6  THEN 'Average. Nothing special but it works.'
+    WHEN 7  THEN 'Decent product but shipping was slow.'
+    WHEN 8  THEN 'Customer service was unhelpful when I asked about a refund.'
+    WHEN 9  THEN 'Amazing build quality. Lives up to the price.'
+    ELSE         'Solid product. Recommended.'
+  END,
+  TIMESTAMPTZ '2023-01-01 00:00:00+00'
+    + (((i * 53) % (365 * 2 * 24 * 60)) || ' minutes')::interval
+FROM generate_series(1, 250000) AS i;
+
+COMMIT;
+
+-- ----------------------------------------------------------------------------
+-- Statistics
+--
+-- ANALYZE so the planner has accurate row counts. Without this, EXPLAIN plans
+-- we feed to the LLM will be misleading.
+-- ----------------------------------------------------------------------------
+
+ANALYZE shop.customers;
+ANALYZE shop.suppliers;
+ANALYZE shop.categories;
+ANALYZE shop.products;
+ANALYZE shop.orders;
+ANALYZE shop.order_items;
+ANALYZE shop.events;
+ANALYZE shop.reviews;

transform_optimizer/dataset/03_optimized_indexes.sql

@@ -0,0 +1,54 @@
+-- Transform Optimizer — indexes used by the *optimized* versions of some
+-- query pairs. Run AFTER the base schema + seed; the slow queries should be
+-- benchmarked BEFORE this file is applied so the contrast is meaningful.
+--
+-- Each index notes which qNN pair it supports.
+
+\set ON_ERROR_STOP on
+
+BEGIN;
+SET search_path TO shop, public;
+SET LOCAL maintenance_work_mem = '1GB';
+
+-- q04: orders by month / status filter
+CREATE INDEX IF NOT EXISTS idx_orders_status_ordered_at_customer
+  ON orders (status, ordered_at, customer_id) INCLUDE (total_cents);
+
+-- q02, q07: per-customer order rollups
+CREATE INDEX IF NOT EXISTS idx_orders_customer_id_ordered_at
+  ON orders (customer_id, ordered_at);
+
+-- q06: per-product revenue from order_items
+CREATE INDEX IF NOT EXISTS idx_order_items_product_id
+  ON order_items (product_id) INCLUDE (quantity, unit_price_cents, discount_cents);
+
+CREATE INDEX IF NOT EXISTS idx_order_items_order_id
+  ON order_items (order_id);
+
+-- q06: product join into category
+CREATE INDEX IF NOT EXISTS idx_products_category_id
+  ON products (category_id);
+
+-- q08: distinct-active-users-per-day style queries over events
+CREATE INDEX IF NOT EXISTS idx_events_occurred_at
+  ON events (occurred_at);
+
+CREATE INDEX IF NOT EXISTS idx_events_customer_id_occurred_at
+  ON events (customer_id, occurred_at)
+  WHERE customer_id IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_events_type_occurred_at
+  ON events (type, occurred_at);
+
+-- q05: trigram search on review bodies
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE INDEX IF NOT EXISTS idx_reviews_body_trgm
+  ON reviews USING gin (body gin_trgm_ops);
+
+COMMIT;
+
+ANALYZE shop.orders;
+ANALYZE shop.order_items;
+ANALYZE shop.products;
+ANALYZE shop.events;
+ANALYZE shop.reviews;

transform_optimizer/harness/README.md

@@ -0,0 +1,106 @@
+# Validation harness
+
+Loads the dataset and validates every `queries/qNN_*.sql` pair by:
+
+1. **Resetting** the database (optional; one-shot with `--reset`).
+2. **Loading** `dataset/01_schema.sql` and `dataset/02_seed.sql`.
+3. **Running every `@slow` query** against the baseline schema (no helper
+   indexes). Each result is materialized into `bench.slow_qNN` so we can
+   compare it later.
+4. **Applying** `dataset/03_optimized_indexes.sql` (idempotent).
+5. **Running every `@fast` query** against the now-indexed schema. The
+   *last* statement in `@fast` is treated as the comparison query;
+   preceding statements (precompute `CREATE TABLE …`, etc.) run as
+   setup. Result materialized into `bench.fast_qNN`.
+6. **Comparing** `bench.slow_qNN` vs `bench.fast_qNN` using the same
+   `EXCEPT ALL`-in-both-directions form the verifier (Phase 4 in
+   `PLAN.md`) will use.
+
+Volatile `NOW()` calls in the `.sql` files are textually replaced with a
+single `TIMESTAMPTZ` captured at the start of the run, so the slow and
+fast queries see the same 90-day window even though they execute minutes
+apart.
+
+## Prerequisites
+
+- A reachable Postgres instance the user can `CREATE SCHEMA` on.
+- `clojure` CLI (the script uses `-Sdeps` to pull `next.jdbc` and the
+  Postgres driver — nothing needs to be pre-installed beyond the CLI).
+
+The default connection is `postgres@localhost:5432/transform_optimizer`.
+Create the database first:
+
+```bash
+createdb transform_optimizer
+```
+
+## First run (load + validate)
+
+```bash
+./run.sh --reset
+```
+
+The first run is dominated by the seed (`02_seed.sql` — ~30M events
+plus 15M order_items). On a developer laptop expect several minutes.
+
+## Subsequent runs
+
+Once the database is populated, skip `--reset` to re-run just the
+slow/fast validation against the existing data:
+
+```bash
+./run.sh
+./run.sh --only q01,q04             # subset
+TO_DB_HOST=db.example ./run.sh      # env-var connection
+```
+
+## Output
+
+```
+=== Transform Optimizer Validation Harness ===
+Database: postgres://postgres@localhost:5432/transform_optimizer  (--reset)
+
+== Phase 1: reset + load schema + seed ==
+  01_schema.sql                  (0.4s)
+  02_seed.sql                    (312.7s)
+
+Reference instant for NOW(): 2026-05-12T10:55:14Z
+
+== Phase 2: run slow queries (baseline schema, no helper indexes) ==
+  q01_not_in_antijoin              slow   18.314s  rows=47312
+  q02_correlated_subquery          slow   24.140s  rows=39998
+  ...
+
+== Phase 3: apply optimized indexes ==
+  03_optimized_indexes.sql        (71.2s)
+
+== Phase 4: run fast queries (with indexes) ==
+  q01_not_in_antijoin              fast    0.094s  rows=47312
+  ...
+
+== Phase 5: compare results (EXCEPT ALL, both directions) ==
+  q01_not_in_antijoin            [rewrite        ] slow   18.314s  fast    0.094s  × 194.8  ✓ eq
+  q02_correlated_subquery        [rewrite        ] slow   24.140s  fast    0.211s  × 114.4  ✓ eq
+  ...
+
+Summary: 8/8 equivalent, geomean speedup ×152.3
+```
+
+Exit code is `0` when every pair is equivalent, `1` otherwise — usable
+from CI.
+
+## Caveats
+
+- Wall-clock timings include the `CREATE TABLE … AS …` write overhead.
+  This is constant per row-count, so the slow/fast ratio is preserved,
+  but the absolute slow numbers are slightly inflated vs a plain
+  `SELECT`. Acceptable for "is this 100× faster?" — not for benchmark
+  papers.
+- `bench.slow_qNN` and `bench.fast_qNN` are left behind for inspection
+  after a run; `--reset` clears them.
+- Precompute pairs (q07, q08) create persistent tables in the `shop`
+  schema (e.g. `shop.customer_first_purchase`). The pair's own
+  `DROP TABLE IF EXISTS …` keeps re-runs idempotent.
+- Equivalence is **set-with-multiplicity** (`EXCEPT ALL`). Two rewrites
+  that produce the same rows in a different order are still equivalent;
+  rewrites that drop or duplicate a row are not.

transform_optimizer/harness/run.clj

@@ -0,0 +1,386 @@
+;; Transform Optimizer — validation harness.
+;;
+;; Loads the dataset, runs every slow → fast pair from queries/, applies the
+;; optimized-indexes script in between, and reports per-pair speedup +
+;; equivalence (via EXCEPT ALL).
+;;
+;; Run via `./run.sh` (which provides next.jdbc + the Postgres driver on the
+;; classpath) or directly with:
+;;
+;;   clojure -Sdeps '{:deps {com.github.seancorfield/next.jdbc {:mvn/version "1.3.939"}
+;;                            org.postgresql/postgresql {:mvn/version "42.7.4"}}}' \
+;;           run.clj [opts]
+;;
+;; Options
+;;   --host HOST        (default localhost; env TO_DB_HOST)
+;;   --port PORT        (default 5432;      env TO_DB_PORT)
+;;   --db DBNAME        (default transform_optimizer; env TO_DB_NAME)
+;;   --user USER        (default postgres;  env TO_DB_USER)
+;;   --password PASS    (default empty;     env TO_DB_PASS)
+;;   --reset            drop & reload schema + seed (skip to reuse a populated DB)
+;;   --only q01,q04,…   limit to selected pairs
+
+(ns transform-optimizer.harness.run
+  (:require [clojure.java.io :as io]
+            [clojure.string :as str]
+            [next.jdbc :as jdbc]))
+
+;; ---------------------------------------------------------------------------
+;; CLI
+
+(def cli-defaults
+  {:host     (or (System/getenv "TO_DB_HOST") "localhost")
+   :port     (Long/parseLong (or (System/getenv "TO_DB_PORT") "5432"))
+   :dbname   (or (System/getenv "TO_DB_NAME") "transform_optimizer")
+   :user     (or (System/getenv "TO_DB_USER") "postgres")
+   :password (or (System/getenv "TO_DB_PASS") "")
+   :reset    false
+   :only     nil})
+
+(defn- parse-args [args]
+  (loop [acc cli-defaults, args args]
+    (if (empty? args)
+      acc
+      (let [[flag & rest] args]
+        (case flag
+          "--host"     (recur (assoc acc :host (first rest))                  (next rest))
+          "--port"     (recur (assoc acc :port (Long/parseLong (first rest))) (next rest))
+          "--db"       (recur (assoc acc :dbname (first rest))                (next rest))
+          "--user"     (recur (assoc acc :user (first rest))                  (next rest))
+          "--password" (recur (assoc acc :password (first rest))              (next rest))
+          "--reset"    (recur (assoc acc :reset true)                         rest)
+          "--only"     (recur (assoc acc :only (set (str/split (first rest) #","))) (next rest))
+          (do (println "unknown flag:" flag) (System/exit 2)))))))
+
+;; ---------------------------------------------------------------------------
+;; Paths
+
+(def ^:private project-root
+  (-> (io/file *file*) .getAbsoluteFile .getParentFile .getParentFile))
+
+(defn- project-file ^java.io.File [& parts]
+  (apply io/file project-root parts))
+
+;; ---------------------------------------------------------------------------
+;; Datasource
+
+(defn- make-ds [opts]
+  (jdbc/get-datasource
+   {:dbtype   "postgresql"
+    :host     (:host opts)
+    :port     (:port opts)
+    :dbname   (:dbname opts)
+    :user     (:user opts)
+    :password (:password opts)
+    ;; - statement_timeout=0 so CREATE INDEX on big tables isn't capped
+    ;; - work_mem=128MB so HashedSubPlan / HashAgg fit in memory for the
+    ;;   "slow" queries; without this Postgres falls back to repeated-scan
+    ;;   subplans and the NOT-IN/correlated-subquery pairs blow up from
+    ;;   seconds to many minutes.
+    :options  "-c statement_timeout=0 -c work_mem=128MB"}))
+
+;; ---------------------------------------------------------------------------
+;; SQL parsing
+;;
+;; The qNN_*.sql files use three header markers: -- @meta, -- @slow, -- @fast.
+;; Each section runs to the next -- @... line or EOF. We treat the LAST
+;; statement of @fast as the comparison query (the prior statements set up
+;; precompute tables for DAG proposals such as q07 / q08).
+
+(defn- section [content tag]
+  (let [marker (str "-- @" tag)]
+    (loop [in? false, out [], [l & more :as ls] (str/split-lines content)]
+      (cond
+        (empty? ls)
+        (str/join "\n" out)
+
+        (= (str/trim l) marker)
+        (recur true out more)
+
+        (and in? (str/starts-with? (str/trim l) "-- @"))
+        (str/join "\n" out)
+
+        in?
+        (recur in? (conj out l) more)
+
+        :else
+        (recur in? out more)))))
+
+(defn- strip-line-comments [sql]
+  (->> (str/split-lines sql)
+       (map (fn [line]
+              (if-let [idx (str/index-of line "--")]
+                (subs line 0 idx)
+                line)))
+       (str/join "\n")))
+
+(defn- split-statements [sql]
+  (->> sql
+       strip-line-comments
+       (#(str/split % #";"))
+       (map str/trim)
+       (remove str/blank?)))
+
+(defn- pair-files []
+  (->> (.listFiles (project-file "queries"))
+       (filter #(re-matches #"q\d+_.+\.sql" (.getName ^java.io.File %)))
+       sort
+       vec))
+
+(defn- parse-pair [^java.io.File f]
+  (let [content    (slurp f)
+        slug       (str/replace (.getName f) #"\.sql$" "")
+        short      (subs slug 0 3)
+        meta-text  (section content "meta")
+        slow-text  (section content "slow")
+        fast-text  (section content "fast")
+        kind       (some->> (re-find #"-- kind:\s*([^\n]+)" meta-text) second str/trim)
+        speedup    (some->> (re-find #"-- expected_speedup:\s*([^\n]+)" meta-text) second str/trim)
+        slow-stmt  (first (split-statements slow-text))
+        fast-stmts (split-statements fast-text)]
+    (when-not slow-stmt
+      (throw (ex-info (str "no slow query in " (.getName f)) {:file f})))
+    (when (empty? fast-stmts)
+      (throw (ex-info (str "no fast query in " (.getName f)) {:file f})))
+    {:file      f
+     :slug      slug
+     :short     short
+     :kind      kind
+     :expected  speedup
+     :slow      slow-stmt
+     :fast-pre  (butlast fast-stmts)        ; setup statements (DAG / precompute)
+     :fast-final (last fast-stmts)}))        ; the SELECT we compare
+
+;; ---------------------------------------------------------------------------
+;; NOW() pinning
+;;
+;; Several pairs (q04, q08) use NOW() - INTERVAL '90 days'. Slow and fast runs
+;; happen seconds apart, so an unpinned NOW() produces different windows and
+;; the EXCEPT ALL diff trips on events near the boundary. We capture one
+;; reference instant up-front and substitute it textually.
+
+(defn- pin-now [sql ^java.time.Instant ref-now]
+  (str/replace sql
+               #"(?i)\bNOW\s*\(\s*\)"
+               (str "TIMESTAMPTZ '" (.toString ref-now) "'")))
+
+;; ---------------------------------------------------------------------------
+;; Script loader (schema, seed, optimized indexes)
+;;
+;; Files use a couple of psql meta-commands (\set, \i). Strip them before
+;; sending the whole script as one Statement#execute so Postgres handles the
+;; embedded BEGIN/COMMIT and SET LOCAL ... blocks itself.
+
+(defn- strip-meta-commands [sql]
+  (->> (str/split-lines sql)
+       (remove #(str/starts-with? (str/triml %) "\\"))
+       (str/join "\n")))
+
+(defn- exec-script! [ds ^java.io.File path]
+  (let [sql (-> path slurp strip-meta-commands)]
+    (with-open [conn (jdbc/get-connection ds)]
+      (.setAutoCommit conn true)
+      (with-open [stmt (.createStatement conn)]
+        (.execute stmt sql)))))
+
+;; ---------------------------------------------------------------------------
+;; Timing helpers
+
+(defn- time-ms [f]
+  (let [t0 (System/nanoTime)
+        v  (f)]
+    [(/ (- (System/nanoTime) t0) 1e6) v]))
+
+(def ^:private heartbeat-interval-ms
+  "How often the heartbeat thread prints `… still running (Ns)`. Short enough
+  that the user trusts the harness isn't hung."
+  3000)
+
+(defn- println-flush [s]
+  (println s)
+  (.flush ^java.io.PrintStream System/out))
+
+(defn- with-heartbeat
+  "Run `f` while a daemon thread prints periodic `… still running (Ns)` lines.
+  `label` prefixes each heartbeat. The heartbeat is cancelled before `f`
+  returns (success or exception)."
+  [label f]
+  (let [start  (System/nanoTime)
+        done?  (volatile! false)
+        ^Runnable r (fn []
+                      (try
+                        (loop []
+                          (Thread/sleep ^long heartbeat-interval-ms)
+                          (when-not @done?
+                            (println-flush
+                             (format "        … %s still running (%.1fs)"
+                                     label
+                                     (/ (- (System/nanoTime) start) 1e9)))
+                            (recur)))
+                        (catch InterruptedException _)))
+        thr    (doto (Thread. r) (.setDaemon true) (.start))]
+    (try
+      (f)
+      (finally
+        (vreset! done? true)
+        (.interrupt thr)))))
+
+(defn- run-statement! [ds sql label]
+  (with-heartbeat label #(jdbc/execute! ds [sql])))
+
+(defn- capture-result! [ds target-table select-sql label]
+  (jdbc/execute! ds [(str "DROP TABLE IF EXISTS bench." target-table)])
+  (let [[ms _]    (time-ms
+                   #(with-heartbeat label
+                      (fn [] (jdbc/execute!
+                              ds [(str "CREATE TABLE bench." target-table " AS " select-sql)]))))
+        row-count (-> (jdbc/execute-one! ds [(str "SELECT count(*) AS c FROM bench."
+                                                  target-table)])
+                      :c)]
+    {:ms ms :row-count row-count}))
+
+(defn- run-slow! [ds pair ref-now label]
+  (capture-result! ds (str "slow_" (:short pair)) (pin-now (:slow pair) ref-now) label))
+
+(defn- run-fast! [ds pair ref-now label]
+  ;; Multi-statement fast queries (q07/q08 precomputes): announce each step so
+  ;; the user can see which precompute is running.
+  (let [pre (:fast-pre pair)
+        n   (count pre)]
+    (doseq [[idx s] (map-indexed vector pre)]
+      (let [step-label (format "%s [precompute %d/%d]" label (inc idx) (inc n))]
+        (println-flush (str "      ↳ " step-label))
+        (run-statement! ds (pin-now s ref-now) step-label)))
+    (when (pos? n)
+      (println-flush (str "      ↳ " label " [final SELECT]")))
+    (capture-result! ds (str "fast_" (:short pair)) (pin-now (:fast-final pair) ref-now) label)))
+
+(defn- compare-results [ds short]
+  (try
+    (-> (jdbc/execute-one!
+         ds [(format
+              "SELECT count(*) AS c FROM (
+                 (SELECT * FROM bench.slow_%s EXCEPT ALL SELECT * FROM bench.fast_%s)
+                 UNION ALL
+                 (SELECT * FROM bench.fast_%s EXCEPT ALL SELECT * FROM bench.slow_%s)
+               ) d" short short short short)])
+        :c)
+    (catch Exception e
+      ;; Almost always means the column lists don't match: report it as
+      ;; non-equivalent rather than failing the whole run.
+      (println "    (comparison error: " (.getMessage e) ")")
+      -1)))
+
+;; ---------------------------------------------------------------------------
+;; Reporting
+
+(defn- fmt-row [{:keys [slug short kind slow-ms fast-ms diff-rows ratio]}]
+  (let [eq? (zero? diff-rows)]
+    (format "  %-32s [%-14s] slow %8.3fs  fast %8.3fs  ×%7.1f  %s%s"
+            slug
+            (or kind "?")
+            (/ slow-ms 1000.0)
+            (/ fast-ms 1000.0)
+            ratio
+            (cond
+              (neg? diff-rows) "✗ COMPARE-ERR"
+              eq?              "✓ eq"
+              :else            "✗ DIFF")
+            (if (pos? diff-rows) (str " (" diff-rows " rows differ)") ""))))
+
+(defn- geomean [xs]
+  (let [xs (filter #(and (number? %) (pos? %) (Double/isFinite %)) xs)]
+    (when (seq xs)
+      (Math/exp (/ (reduce + (map #(Math/log %) xs)) (count xs))))))
+
+;; ---------------------------------------------------------------------------
+;; Main
+
+(defn- -main [& args]
+  (let [opts (parse-args args)
+        ds   (make-ds opts)]
+    (println "=== Transform Optimizer Validation Harness ===")
+    (println (format "Database: postgres://%s@%s:%d/%s%s"
+                     (:user opts) (:host opts) (:port opts) (:dbname opts)
+                     (if (:reset opts) "  (--reset)" "")))
+
+    (when (:reset opts)
+      (println "\n== Phase 1: reset + load schema + seed ==")
+      (jdbc/execute! ds ["DROP SCHEMA IF EXISTS shop  CASCADE"])
+      (jdbc/execute! ds ["DROP SCHEMA IF EXISTS bench CASCADE"])
+      (jdbc/execute! ds ["CREATE SCHEMA bench"])
+      (doseq [f ["01_schema.sql" "02_seed.sql"]]
+        (let [path (project-file "dataset" f)
+              _    (println-flush (format "  → %s loading ..." f))
+              [ms _] (time-ms
+                      #(with-heartbeat f
+                         (fn [] (exec-script! ds path))))]
+          (println-flush (format "  ✓ %s (%.1fs)" f (/ ms 1000.0))))))
+
+    (when-not (:reset opts)
+      (jdbc/execute! ds ["CREATE SCHEMA IF NOT EXISTS bench"]))
+
+    (let [pairs   (cond->> (mapv parse-pair (pair-files))
+                    (:only opts) (filter #((:only opts) (:short %))))
+          _       (when (empty? pairs)
+                    (println "no matching pairs") (System/exit 2))
+          ref-now (-> (jdbc/execute-one! ds ["SELECT NOW() AS n"]) :n .toInstant)]
+
+      (println (str "\nReference instant for NOW(): " ref-now))
+
+      (println "\n== Phase 2: run slow queries (baseline schema, no helper indexes) ==")
+      (let [total (count pairs)
+            slow-stats
+            (vec
+             (map-indexed
+              (fn [i p]
+                (let [n     (inc i)
+                      label (format "[%d/%d] %s slow" n total (:slug p))]
+                  (println-flush (format "  → %s  running ..." label))
+                  (let [{:keys [ms row-count]} (run-slow! ds p ref-now label)]
+                    (println-flush
+                     (format "  ✓ %s  %.3fs  rows=%d" label (/ ms 1000.0) row-count))
+                    (assoc p :slow-ms ms :slow-rows row-count))))
+              pairs))]
+
+        (println "\n== Phase 3: apply optimized indexes ==")
+        (let [[ms _] (time-ms
+                      #(with-heartbeat "03_optimized_indexes.sql"
+                         (fn []
+                           (exec-script! ds (project-file "dataset" "03_optimized_indexes.sql")))))]
+          (println-flush (format "  03_optimized_indexes.sql       (%.1fs)" (/ ms 1000.0))))
+
+        (println "\n== Phase 4: run fast queries (with indexes) ==")
+        (let [fast-stats
+              (vec
+               (map-indexed
+                (fn [i p]
+                  (let [n     (inc i)
+                        label (format "[%d/%d] %s fast" n total (:slug p))]
+                    (println-flush (format "  → %s  running ..." label))
+                    (let [{:keys [ms row-count]} (run-fast! ds p ref-now label)]
+                      (println-flush
+                       (format "  ✓ %s  %.3fs  rows=%d" label (/ ms 1000.0) row-count))
+                      (assoc p :fast-ms ms :fast-rows row-count))))
+                slow-stats))]
+
+          (println "\n== Phase 5: compare results (EXCEPT ALL, both directions) ==")
+          (let [report
+                (mapv (fn [p]
+                        (let [diff  (compare-results ds (:short p))
+                              ratio (if (zero? (:fast-ms p))
+                                      Double/POSITIVE_INFINITY
+                                      (/ (:slow-ms p) (:fast-ms p)))
+                              row   (assoc p :diff-rows diff :ratio ratio)]
+                          (println (fmt-row row))
+                          row))
+                      fast-stats)
+                n     (count report)
+                n-eq  (count (filter #(zero? (:diff-rows %)) report))
+                gmean (geomean (map :ratio report))]
+            (println (format "\nSummary: %d/%d equivalent, geomean speedup %s"
+                             n-eq n
+                             (if gmean (format "×%.1f" gmean) "n/a")))
+            (System/exit (if (= n-eq n) 0 1))))))))
+
+(apply -main *command-line-args*)

transform_optimizer/harness/run.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Validation harness wrapper. Pulls next.jdbc + the Postgres driver inline so
+# the script is independent of the Metabase deps.edn / classpath.
+set -euo pipefail
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+exec clojure \
+  -Sdeps '{:deps {com.github.seancorfield/next.jdbc {:mvn/version "1.3.939"}
+                  org.postgresql/postgresql        {:mvn/version "42.7.4"}}}' \
+  "$DIR/run.clj" "$@"

transform_optimizer/queries/00_index.md

@@ -0,0 +1,42 @@
+# Query pairs — slow → fast
+
+Each `qNN_*.sql` file contains:
+
+```
+-- @meta
+-- name:        <human title>
+-- kind:        rewrite | index | rewrite+index | precompute
+-- expected_speedup: <order-of-magnitude target>
+-- requires:    <indexes / extensions / new transforms the fast version depends on>
+--
+-- @slow
+<the un-optimized query>
+--
+-- @fast
+<the optimized query (or sequence of CREATE TABLE statements, for precompute)>
+```
+
+A simple loader can split on the `-- @` markers and feed each side to the
+verifier (Phase 4) or render `@slow → @fast` as a prelude example for the
+LLM (Phase 2).
+
+## Index
+
+| #   | Pattern                                | Kind            | Target speedup |
+|-----|----------------------------------------|-----------------|----------------|
+| q01 | `NOT IN` → `NOT EXISTS` antijoin       | rewrite         | ≥100×          |
+| q02 | Correlated subqueries → group + join   | rewrite         | ≥50×           |
+| q03 | OR over multiple columns → `UNION ALL` | rewrite (+idx)  | ≥50×           |
+| q04 | Full scan filter → status/date index   | index           | ≥100×          |
+| q05 | `ILIKE '%…%'` → `pg_trgm` GIN          | rewrite+index   | ≥100×          |
+| q06 | Top-N per group via window → lateral   | rewrite+index   | ≥50×           |
+| q07 | Monolithic cohort retention → DAG      | precompute      | ≥10× per run, much more on re-run |
+| q08 | Daily uniques over events → rollup DAG | precompute      | ≥100× on re-run |
+
+## Conventions
+
+- All queries reference the `shop` schema from `dataset/01_schema.sql`.
+- `@slow` should be timed against the base schema (no extra indexes).
+- `@fast` may reference indexes from `dataset/03_optimized_indexes.sql`.
+- `precompute` pairs may include multiple `CREATE TABLE …_precomputed AS …`
+  statements in the `@fast` block. The final `SELECT` queries those rollups.

transform_optimizer/queries/q01_not_in_antijoin.sql

@@ -0,0 +1,36 @@
+-- @meta
+-- name: Customers who have never placed an order
+-- kind: rewrite
+-- expected_speedup: ≥2× (modest at hackathon scale; larger on bigger data)
+-- requires: -
+--
+-- Why the slow version is slow
+-- ============================
+-- `NOT IN (subquery)` materialises the full subquery into a hash before
+-- probing. On `orders.customer_id` (NOT NULL) the planner *can* now
+-- transform `NOT IN` to an anti-join, but it does so less aggressively
+-- than `NOT EXISTS`, especially when the subquery is large relative to the
+-- outer table — the materialise-then-probe form still does a full scan +
+-- hash build.
+--
+-- `NOT EXISTS` is the planner's preferred form for anti-semi-join: it can
+-- short-circuit per outer row once any match is found.
+--
+-- ⚠ Equivalence note: this rewrite is only safe because
+-- `orders.customer_id` is `NOT NULL`. On a nullable column the two forms
+-- behave differently — `NOT IN` returns UNKNOWN (excluded) for non-matching
+-- outer rows when the subquery contains any NULL, while `NOT EXISTS` does
+-- not. The optimizer should never propose this rewrite without checking
+-- the NOT NULL constraint on the referenced column.
+
+-- @slow
+SELECT c.id, c.email, c.name, c.signed_up_at
+FROM shop.customers c
+WHERE c.id NOT IN (SELECT o.customer_id FROM shop.orders o);
+
+-- @fast
+SELECT c.id, c.email, c.name, c.signed_up_at
+FROM shop.customers c
+WHERE NOT EXISTS (
+  SELECT 1 FROM shop.orders o WHERE o.customer_id = c.id
+);

transform_optimizer/queries/q02_correlated_subquery.sql

@@ -0,0 +1,62 @@
+-- @meta
+-- name: Customer lifetime stats (orders count + lifetime spend), enterprise US
+-- kind: rewrite
+-- expected_speedup: ≥50×
+-- requires: -
+--
+-- Why the slow version is slow
+-- ============================
+-- Three correlated scalar subqueries in the SELECT list. Postgres re-runs each
+-- subquery once per outer customer row. With no index on `orders.customer_id`
+-- each subquery does an O(N) scan over orders — O(C × N) total. The optimizer
+-- never collapses these into a single pass.
+--
+-- We restrict the outer query to enterprise US customers (~200 rows on the
+-- current seed) to keep the slow version in the few-seconds range rather
+-- than the hours range. The pathology is identical; only the fan-out is
+-- bounded.
+--
+-- The fast rewrite aggregates orders ONCE per customer and joins the rollup.
+-- Even without an index on orders.customer_id this is a single hash-aggregate
+-- pass.
+
+-- @slow
+SELECT
+  c.id,
+  c.name,
+  c.country,
+  (SELECT COUNT(*)
+     FROM shop.orders o
+     WHERE o.customer_id = c.id)                          AS order_count,
+  (SELECT COALESCE(SUM(o.total_cents), 0)
+     FROM shop.orders o
+     WHERE o.customer_id = c.id
+       AND o.status IN ('paid','shipped','delivered'))    AS lifetime_cents,
+  (SELECT MAX(o.ordered_at)
+     FROM shop.orders o
+     WHERE o.customer_id = c.id)                          AS last_order_at
+FROM shop.customers c
+WHERE c.segment = 'enterprise'
+  AND c.country = 'US';
+
+-- @fast
+SELECT
+  c.id,
+  c.name,
+  c.country,
+  COALESCE(agg.order_count,    0) AS order_count,
+  COALESCE(agg.lifetime_cents, 0) AS lifetime_cents,
+  agg.last_order_at
+FROM shop.customers c
+LEFT JOIN (
+  SELECT
+    o.customer_id,
+    COUNT(*)                                                            AS order_count,
+    SUM(CASE WHEN o.status IN ('paid','shipped','delivered')
+             THEN o.total_cents ELSE 0 END)                             AS lifetime_cents,
+    MAX(o.ordered_at)                                                   AS last_order_at
+  FROM shop.orders o
+  GROUP BY o.customer_id
+) agg ON agg.customer_id = c.id
+WHERE c.segment = 'enterprise'
+  AND c.country = 'US';

transform_optimizer/queries/q03_or_to_union.sql

@@ -0,0 +1,48 @@
+-- @meta
+-- name: Recent events for either a specific customer OR a specific product
+-- kind: rewrite (also benefits from indexes from 03_optimized_indexes.sql)
+-- expected_speedup: ≥50× (when complementary indexes exist)
+-- requires: idx_events_customer_id_occurred_at, idx_events_occurred_at, (idx on product_id helpful)
+--
+-- Why the slow version is slow
+-- ============================
+-- `OR` across columns belonging to different (potential) indexes prevents the
+-- planner from using either: a seqscan over 30M events is usually the only
+-- plan it can prove correct. Even with both indexes available, the planner
+-- often picks a BitmapOr over both — which is fine — but the original is
+-- written in a way (with `coalesce(...)` and a date filter buried under OR)
+-- that hides the disjunction.
+--
+-- The fast rewrite splits into a UNION ALL of two single-column-filtered
+-- queries, each of which can use its own narrow index. The outer DISTINCT
+-- handles overlap (an event matching both filters).
+
+-- @slow
+SELECT
+  e.id,
+  e.customer_id,
+  e.product_id,
+  e.occurred_at,
+  e.type,
+  e.url_path
+FROM shop.events e
+WHERE e.occurred_at >= TIMESTAMPTZ '2025-06-01 00:00:00+00'
+  AND e.occurred_at <  TIMESTAMPTZ '2025-07-01 00:00:00+00'
+  AND (e.customer_id BETWEEN 1 AND 1000 OR e.product_id BETWEEN 1 AND 100);
+
+-- @fast
+SELECT DISTINCT ON (e.id)
+  e.id, e.customer_id, e.product_id, e.occurred_at, e.type, e.url_path
+FROM (
+  SELECT id, customer_id, product_id, occurred_at, type, url_path
+  FROM shop.events
+  WHERE customer_id BETWEEN 1 AND 1000
+    AND occurred_at >= TIMESTAMPTZ '2025-06-01 00:00:00+00'
+    AND occurred_at <  TIMESTAMPTZ '2025-07-01 00:00:00+00'
+  UNION ALL
+  SELECT id, customer_id, product_id, occurred_at, type, url_path
+  FROM shop.events
+  WHERE product_id BETWEEN 1 AND 100
+    AND occurred_at >= TIMESTAMPTZ '2025-06-01 00:00:00+00'
+    AND occurred_at <  TIMESTAMPTZ '2025-07-01 00:00:00+00'
+) e;

transform_optimizer/queries/q04_status_date_index.sql

@@ -0,0 +1,40 @@
+-- @meta
+-- name: Monthly gross revenue for shipped/delivered orders, last 12 months
+-- kind: index
+-- expected_speedup: ≥100×
+-- requires: idx_orders_status_ordered_at_customer (status, ordered_at, customer_id) INCLUDE (total_cents)
+--
+-- Why the slow version is slow
+-- ============================
+-- No index on orders.(status, ordered_at). 5M rows full-scanned every run.
+-- The fast version is identical SQL — what changes is the existence of a
+-- compound index that lets the planner do an index-only scan over the small
+-- (status, ordered_at) prefix that the WHERE clause selects, and pick
+-- total_cents straight out of the index payload (`INCLUDE`).
+--
+-- This pair is the canonical "the LLM should propose CREATE INDEX, not a
+-- rewrite" example.
+
+-- @slow
+SELECT
+  DATE_TRUNC('month', o.ordered_at) AS month,
+  COUNT(*)                          AS order_count,
+  SUM(o.total_cents)                AS gross_cents
+FROM shop.orders o
+WHERE o.status IN ('shipped','delivered')
+  AND o.ordered_at >= NOW() - INTERVAL '12 months'
+GROUP BY 1
+ORDER BY 1;
+
+-- @fast
+-- Requires: CREATE INDEX idx_orders_status_ordered_at_customer
+--           ON shop.orders (status, ordered_at, customer_id) INCLUDE (total_cents);
+SELECT
+  DATE_TRUNC('month', o.ordered_at) AS month,
+  COUNT(*)                          AS order_count,
+  SUM(o.total_cents)                AS gross_cents
+FROM shop.orders o
+WHERE o.status IN ('shipped','delivered')
+  AND o.ordered_at >= NOW() - INTERVAL '12 months'
+GROUP BY 1
+ORDER BY 1;

transform_optimizer/queries/q05_text_search_trigram.sql

@@ -0,0 +1,35 @@
+-- @meta
+-- name: Find reviews mentioning "refund"
+-- kind: rewrite+index
+-- expected_speedup: ≥100×
+-- requires: pg_trgm extension, idx_reviews_body_trgm (GIN on body gin_trgm_ops)
+--
+-- Why the slow version is slow
+-- ============================
+-- `ILIKE '%refund%'` cannot use a btree index — it has no left-anchored
+-- prefix. With 2M reviews, this is a full seq scan + per-row regex match.
+--
+-- The fast rewrite is the same SQL but relies on a GIN trigram index. The
+-- query *also* normalises the column to lower(body) when comparing, which
+-- is the canonical pattern for case-insensitive trigram search (and matches
+-- the expression-index variant if the index is created on `lower(body)`).
+-- Here we keep the simple ILIKE form because the gin_trgm_ops operator class
+-- supports `ILIKE` directly.
+
+-- @slow
+SELECT r.id, r.product_id, r.rating, r.created_at, r.body
+FROM shop.reviews r
+WHERE r.body ILIKE '%refund%'
+ORDER BY r.created_at DESC
+LIMIT 100;
+
+-- @fast
+-- Requires:
+--   CREATE EXTENSION IF NOT EXISTS pg_trgm;
+--   CREATE INDEX idx_reviews_body_trgm
+--     ON shop.reviews USING gin (body gin_trgm_ops);
+SELECT r.id, r.product_id, r.rating, r.created_at, r.body
+FROM shop.reviews r
+WHERE r.body ILIKE '%refund%'
+ORDER BY r.created_at DESC
+LIMIT 100;

transform_optimizer/queries/q06_topn_per_group.sql

@@ -0,0 +1,64 @@
+-- @meta
+-- name: Top 5 products by gross revenue per category
+-- kind: rewrite+index
+-- expected_speedup: ≥50×
+-- requires: idx_order_items_product_id (INCLUDE quantity, unit_price_cents, discount_cents),
+--           idx_products_category_id
+--
+-- Why the slow version is slow
+-- ============================
+-- The slow form materialises *every* product's revenue via a 15M-row scan of
+-- order_items joined to 20k products, then ranks every product within its
+-- category via `ROW_NUMBER()`. We pay for sorting all 20k products even
+-- though we only want 5 per category × 30 categories = 150 rows.
+--
+-- The fast rewrite (a) builds a small per-product revenue rollup that
+-- benefits from idx_order_items_product_id, and (b) replaces the window
+-- function with a LATERAL subquery that asks for "top 5 by revenue within
+-- this category" — Postgres can stop early per category thanks to the
+-- supporting index on products.category_id.
+
+-- @slow
+SELECT category_id, product_id, name, gross_cents
+FROM (
+  SELECT
+    p.category_id,
+    p.id   AS product_id,
+    p.name,
+    SUM((oi.quantity * oi.unit_price_cents) - oi.discount_cents) AS gross_cents,
+    ROW_NUMBER() OVER (
+      PARTITION BY p.category_id
+      ORDER BY SUM((oi.quantity * oi.unit_price_cents) - oi.discount_cents) DESC
+    ) AS rn
+  FROM shop.products p
+  JOIN shop.order_items oi ON oi.product_id = p.id
+  GROUP BY p.category_id, p.id, p.name
+) ranked
+WHERE rn <= 5
+ORDER BY category_id, gross_cents DESC;
+
+-- @fast
+-- Requires:
+--   CREATE INDEX idx_order_items_product_id
+--     ON shop.order_items (product_id)
+--     INCLUDE (quantity, unit_price_cents, discount_cents);
+--   CREATE INDEX idx_products_category_id ON shop.products (category_id);
+WITH product_revenue AS (
+  SELECT
+    oi.product_id,
+    SUM((oi.quantity * oi.unit_price_cents) - oi.discount_cents) AS gross_cents
+  FROM shop.order_items oi
+  GROUP BY oi.product_id
+)
+SELECT c.id AS category_id, top.product_id, p.name, top.gross_cents
+FROM shop.categories c
+CROSS JOIN LATERAL (
+  SELECT pr.product_id, pr.gross_cents
+  FROM product_revenue pr
+  JOIN shop.products p2 ON p2.id = pr.product_id
+  WHERE p2.category_id = c.id
+  ORDER BY pr.gross_cents DESC
+  LIMIT 5
+) top
+JOIN shop.products p ON p.id = top.product_id
+ORDER BY c.id, top.gross_cents DESC;