feat(library): working load directly from files

This reverts commit ba8c87d6ba.

deps.edn

@@ -170,6 +170,7 @@
   org.eclipse.jetty/jetty-unixdomain-server {:mvn/version "12.0.25"}
   org.eclipse.jetty.ee9/jetty-ee9-servlet   {:mvn/version "12.0.24"}
   org.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server {:mvn/version "12.0.25"}
+  org.eclipse.jgit/org.eclipse.jgit        {:mvn/version "7.3.0.202506031305-r"}; Java implementation of Git version control system
   org.flatland/ordered                      {:mvn/version "1.15.12"}            ; ordered maps & sets
   org.graalvm.polyglot/js-community         {:mvn/version "24.2.2" :extension "pom"} ; JavaScript engine
   org.graalvm.polyglot/polyglot             {:mvn/version "24.2.2"}             ; GraalVM platform, required by JS engine

enterprise/backend/src/metabase_enterprise/api/routes.clj

@@ -22,6 +22,7 @@
    [metabase-enterprise.documents.api]
    [metabase-enterprise.email.api]
    [metabase-enterprise.gsheets.api :as gsheets.api]
+   [metabase-enterprise.library.api]
    [metabase-enterprise.llm.api]
    [metabase-enterprise.metabot-v3.api]
    [metabase-enterprise.metabot-v3.tools.api]
@@ -100,6 +101,7 @@
    "/database-routing"             (premium-handler metabase-enterprise.database-routing.api/routes :database-routing)
    "/document"                     (premium-handler metabase-enterprise.documents.api/routes :documents)
    "/email"                        (premium-handler metabase-enterprise.email.api/routes :cloud-custom-smtp)
+   "/library"                      (premium-handler metabase-enterprise.library.api/routes :serialization)
    "/gsheets"                      (-> gsheets.api/routes ;; gsheets requires both features.
                                        (premium-handler :attached-dwh)
                                        (premium-handler :etl-connections))

enterprise/backend/src/metabase_enterprise/core/init.clj

@@ -13,6 +13,7 @@
    [metabase-enterprise.database-replication.init]
    [metabase-enterprise.documents.init]
    [metabase-enterprise.gsheets.init]
+   [metabase-enterprise.library.init]
    [metabase-enterprise.llm.init]
    [metabase-enterprise.metabot-v3.init]
    [metabase-enterprise.scim.init]

enterprise/backend/src/metabase_enterprise/library/api.clj

@@ -0,0 +1,214 @@
+(ns metabase-enterprise.library.api
+  (:require
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [metabase-enterprise.library.sources :as sources]
+   [metabase-enterprise.mbml.core :as mbml]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.util.log :as log]
+   [metabase-enterprise.library.core :as library])
+  (:import (java.io File)
+           (java.nio.file Files)
+           (java.nio.file.attribute FileAttribute)))
+
+(set! *warn-on-reflection* true)
+
+(defn cleanup-temp-directory!
+  "Clean up temporary directory and all its contents."
+  [temp-dir]
+  (when temp-dir
+    (try
+      (let [dir-file (io/file temp-dir)]
+        (when (.exists ^File dir-file)
+          (log/debugf "Cleaning up temporary directory: %s" temp-dir)
+
+          (letfn [(delete-file [file]
+                    (when (.isDirectory file)
+                      (doseq [child (.listFiles file)]
+                        (delete-file child)))
+                    (.delete file))]
+            (delete-file dir-file))
+
+          (log/debugf "Successfully cleaned up temporary directory: %s" temp-dir)))
+      (catch Exception e
+        (log/warnf e "Failed to clean up temporary directory: %s" temp-dir)))))
+
+(defmacro with-temp-directory
+  "Makes a temp dir and then deletes it outside the form."
+  [binding & body]
+  `(let [~binding (.toFile (Files/createTempDirectory "library-" (into-array FileAttribute [])))]
+     (try ~@body
+          (finally
+            (cleanup-temp-directory! ~binding)))))
+
+(defn- reload-from-git!
+  "Reloads the Metabase entities from the "
+  []
+  (if-let [source (sources/get-source)]
+    (with-temp-directory dir
+      (log/info "Reloading Metabase configuration from source")
+      (try
+        (library/index-library!
+         (->> (.toString dir)
+              (sources/load-source! source)
+              io/file
+              file-seq
+              next
+              (filter (fn [f]
+                        (and (not (.isDirectory f))
+                             (some #(str/ends-with? (.getPath f) %)
+                                   [".sql" ".yaml" ".yml"]))))
+              (map #(.getPath %))))
+        (log/info "Successfully reloaded entities from git repository")
+        {:status :success
+         :message "Successfully reloaded from git repository"}
+
+        (catch Exception e
+          (log/errorf e "Failed to reload from git repository: %s" (.getMessage e))
+          (let [error-msg (cond
+                            (instance? java.net.UnknownHostException e)
+                            "Network error: Unable to reach git repository host"
+
+                            (str/includes? (.getMessage e) "Authentication failed")
+                            "Authentication failed: Please check your git credentials"
+
+                            (str/includes? (.getMessage e) "Repository not found")
+                            "Repository not found: Please check the repository URL"
+
+                            (str/includes? (.getMessage e) "branch")
+                            "Branch error: Please check the specified branch exists"
+
+                            :else
+                            (format "Failed to reload from git repository: %s" (.getMessage e)))]
+            {:status :error
+             :message error-msg
+             :details {:error-type (type e)}}))))
+    {:status :error
+     :message "Library source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."}))
+
+(api.macros/defendpoint :post "/import"
+  "Reload Metabase content from Git repository source of truth.
+
+  This endpoint will:
+  1. Fetch the latest changes from the configured git repository
+  2. Load the updated content using the serialization/deserialization system
+
+  Requires superuser permissions."
+  []
+  (api/check-superuser)
+  (let [result (reload-from-git!)]
+    (case (:status result)
+      :success
+      "Success"
+
+      :error
+      {:status 400
+       :body {:status "error"
+              :message (:message result)}}
+
+      ;; Fallback for unexpected result format
+      {:status 500
+       :body {:status "error"
+              :message "Unexpected error occurred during reload"}})))
+
+(api.macros/defendpoint :get "/source"
+  "List items in the source repository"
+  []
+  (if-let [source (sources/get-source)]
+    (with-temp-directory dir
+      (log/info "Reloading Metabase configuration from source")
+      (try
+        (let [root-dir (io/file (sources/load-source! source (.toString dir)))
+              files (next (file-seq root-dir))
+              build-tree (fn build-tree [files parent-path]
+                           (->> files
+                                (filter #(not= (.getName %) ".git"))
+                                (filter #(not= (.getName %) ".github"))
+                                (filter #(not= (.getName %) ".gitignore"))
+                                (filter #(.startsWith (.toPath %) (.toPath parent-path)))
+                                (filter #(= (.getParent (.toPath %)) (.toPath parent-path)))
+                                (map (fn [file]
+                                       (let [rel-path (->> file .toPath (.relativize (.toPath root-dir)) .toString)]
+                                         (if (.isDirectory file)
+                                           {:id rel-path
+                                            :name (str "/" (.getName file))
+                                            :type "folder"
+                                            :children (build-tree files file)}
+                                           {:id rel-path
+                                            :name (str "/" (.getName file))
+                                            :type "file"}))))
+                                (sort-by :name)))]
+          {:id "root"
+           :name "metabase-"
+           :type "folder"
+           :children (build-tree files root-dir)})
+
+        (catch Exception e
+          (log/errorf e "Failed to reload from git repository: %s" (.getMessage e))
+          (let [error-msg (cond
+                            (instance? java.net.UnknownHostException e)
+                            "Network error: Unable to reach git repository host"
+
+                            (str/includes? (.getMessage e) "Authentication failed")
+                            "Authentication failed: Please check your git credentials"
+
+                            (str/includes? (.getMessage e) "Repository not found")
+                            "Repository not found: Please check the repository URL"
+
+                            (str/includes? (.getMessage e) "branch")
+                            "Branch error: Please check the specified branch exists"
+
+                            :else
+                            (format "Failed to reload from git repository: %s" (.getMessage e)))]
+            {:status  :error
+             :message error-msg
+             :details {:error-type (type e)}}))))
+    {:status  :error
+     :message "Library source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."}))
+
+(def ^:private entity-type->fe-type
+  {"model/Transform:v1" "transform"})
+
+(api.macros/defendpoint :get "/source/:path"
+  "List item in the "
+  [{:keys [path]}]
+  (if-let [source (sources/get-source)]
+    (with-temp-directory dir
+      (log/info "Reloading Metabase configuration from source")
+      (try
+        (let [root-dir (io/file (sources/load-source! source (.toString dir)))
+              file (io/file root-dir path)]
+          (api/check-404 (when (.exists file)
+                           (let [[entity-type _ model] (mbml/mbml-file->unsaved-model (str (io/file root-dir path)))]
+                             {:path    path
+                              :entityType (entity-type->fe-type entity-type)
+                              :entity model
+                              :content (slurp file)}))))
+        (catch Exception e
+          (log/errorf e "Failed to reload from git repository: %s" (.getMessage e))
+          (let [error-msg (cond
+                            (instance? java.net.UnknownHostException e)
+                            "Network error: Unable to reach git repository host"
+
+                            (str/includes? (.getMessage e) "Authentication failed")
+                            "Authentication failed: Please check your git credentials"
+
+                            (str/includes? (.getMessage e) "Repository not found")
+                            "Repository not found: Please check the repository URL"
+
+                            (str/includes? (.getMessage e) "branch")
+                            "Branch error: Please check the specified branch exists"
+
+                            :else
+                            (format "Failed to reload from git repository: %s" (.getMessage e)))]
+            {:status  :error
+             :message error-msg
+             :details {:error-type (type e)}}))))
+    {:status  :error
+     :message "Library source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."}))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/library` routes."
+  (api.macros/ns-handler *ns* +auth))

enterprise/backend/src/metabase_enterprise/library/core.clj

@@ -0,0 +1,185 @@
+(ns metabase-enterprise.library.core
+  "The Library is a set of core models provided by a git-backed filesystem.
+
+  Library models can be queried by their filename as well as via specifically defined
+  attributes.  "
+  (:require
+   [clojure.set :as set]
+   [metabase-enterprise.git-source-of-truth.settings :as git-source-of-truth.settings]
+   [metabase-enterprise.mbml.core :as mbml.core]
+   [metabase-enterprise.serialization.cmd :as serdes-cmd]
+   [metabase.cloud-migration.core :as cloud-migration]
+   [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
+   [metabase.lib.core :as lib]
+   [metabase.models.serialization :as serdes]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [methodical.core :as methodical]
+   [potemkin :as p]
+   [toucan2.core :as t2]))
+
+(def ^:dynamic *current-branch* nil)
+
+(derive :model/Transform:v1 :metabase/library-model)
+
+(def ^:private branch+filename->model
+  "Index of the library of filename and branch to a specific model
+
+  TODO(edpaget): use some kind of cache eviction mechanism."
+  (atom {}))
+
+(def ^:private branch+entity+index->filename
+  "Lookup filename by indexed values
+
+   TODO(edpaget): use a btree?"
+  (atom {}))
+
+(methodical/defmulti load
+  "Accepts the parsed result of an mbml file and loads it as a library model applying any validation
+  or transformations required from the file-representation.
+
+  Args:
+    filename: the file that was loaded
+    parsed-contents: the parsed contents from mbml. It must have an :entity keyword for dispatch.
+
+  Returns:
+    the loaded library model"
+  {:arglists '([filename parsed-contents])}
+  (fn [_filename {:keys [entity]}] (keyword entity)))
+
+(methodical/defmethod load :metabase/library-model
+  "Associate the file with an model-kw and a `id` attribute that can be used when it needs to
+  interoperate with toucan models. This is separate from the human readable :identifier."
+  [filename {:keys [identifier] :as parsed-file}]
+  (let [with-id (assoc parsed-file
+                       :filename filename
+                       :id identifier)]
+    (cond->> with-id
+      next-method (next-method filename))))
+
+(defn- update-source-query
+  [{:keys [source body] :as model} database-id]
+  (let [raw-query (or body source)
+        metadata-provider (lib.metadata.jvm/application-database-metadata-provider database-id)
+        resolved-query (cond
+                         (string? raw-query) (lib/native-query metadata-provider raw-query)
+                         (map? raw-query) (serdes/import-mbql raw-query))]
+    (assoc model :source {:type "query"
+                          :query resolved-query})))
+
+(methodical/defmethod load :model/Transform:v1
+  "Validate the transform.
+
+  TODO(edpaget): merge this with the validation in transforms.core"
+  [filename {:keys [tags database] :as mbml-map}]
+  (let [tag-ids (t2/select-pks-vec :model/TransformTag :name [:in tags])
+        database-id (t2/select-one-pk :model/Database :name database)]
+    (when-not (= (count tag-ids) (count (set tags)))
+      (throw (ex-info "Missing Tags" {:type :model/Transform :data mbml-map :file filename})))
+    (when-not database-id
+      (throw (ex-info "Missing Database" {:type :model/Transform :data mbml-map :file filename})))
+    (cond->> (-> mbml-map
+                 (update-source-query database-id)
+                 (dissoc :body)
+                 (assoc :database_id database-id :tag_ids tag-ids))
+      next-method (next-method filename))))
+
+(methodical/defmulti index
+  "Create a map of indexes by entity type used for lookups of items in the library
+
+  Args:
+    model: library model to index
+
+  Returns:
+    sequence of vector paths indidcating where the entity should be indexed."
+  {:arglists '([model])}
+  (fn [{:keys [entity]}] (keyword entity)))
+
+(methodical/defmethod index :metabase/library-model
+  "All models are indexed by their identifier"
+  [{:keys [identifier] :as model}]
+  (cond->> [[:identifier identifier]]
+    next-method (concat (next-method model))))
+
+(methodical/defmethod index :model/Transform:v1
+  "Index transforms by their tags and tag_ids"
+  [{:keys [tags tag_ids] :as model}]
+  (cond->> (concat (map #(vector :tag %1) tags) (map #(vector :tag_id %1) tag_ids))
+    next-method (concat (next-method model))))
+
+(defn current-branch
+  []
+  (or *current-branch* (git-source-of-truth.settings/git-sync-import-branch)))
+
+(def ^:private entity-type->model
+  {"model/Transform:v1" :model/Transform})
+
+(defn index-library!
+  "Given a list of file paths load models and build an index of the library."
+  [paths]
+  (let [branch (current-branch)]
+    (swap! branch+filename->model
+           assoc branch
+           (reduce (fn [accum path]
+                     ;; Turn into t2 instances so hydrate works (hopefully!)
+                     (let [{:keys [entity] :as model} (load path (mbml.core/parse-mbml-file path))]
+                       (assoc accum path (t2/instance (entity-type->model entity) model))))
+                   {} paths))
+    (swap! branch+entity+index->filename
+           assoc branch
+           (reduce-kv (fn [accum filename {:keys [entity] :as model}]
+                        (reduce (fn [accum* index]
+                                  (let [path (concat [(entity-type->model entity)] index)]
+                                    (if (get-in accum* path)
+                                      (update-in accum* path conj filename)
+                                      (assoc-in accum* path #{filename}))))
+                                accum
+                                (index model)))
+                      {} (get @branch+filename->model branch)))))
+
+(defn select
+  "Retrive a model by entity and identifier
+
+  Args:
+    model: the keyword of the model to lookup
+    identifier: the identifier of the model
+
+  Returns:
+    the library model"
+  [model identifier]
+  (when-let [filename (get-in @branch+entity+index->filename [(current-branch) model :identifier identifier])]
+    (when (seq filename)
+      (get-in @branch+filename->model [(current-branch) (first filename)]))))
+
+(defn find-by
+  "Find models give one or more indexes
+
+  Args:
+    model: the keyword of the model to lookup
+    index: the index to search on
+    value: a single value or series of values to lookup from
+
+  Return:
+    sequence of models found."
+  [model index value]
+  (->> (map #(get-in @branch+entity+index->filename [(current-branch) model index %1]) value)
+       (reduce set/union #{})
+       (map #(get-in @branch+filename->model [(current-branch) %1]))))
+
+(defn select-all
+  "Retrive a model by entity and identifier
+
+  Args:
+    model: the keyword of the model to lookup
+
+  Returns:
+    a sequence of the library models"
+  [model]
+  (when-let [filenames (->> (get-in @branch+entity+index->filename [(current-branch) model :identifier])
+                            vals
+                            (reduce set/union #{}))]
+    (get-in @branch+entity+index->filename [(current-branch) model :identifier])
+    (prn filenames)
+    (prn (map #(get-in @branch+filename->model [(current-branch) %1]) filenames))
+    (prn (get-in @branch+filename->model [(current-branch)]))
+    (map #(get-in @branch+filename->model [(current-branch) %1]) filenames)))

enterprise/backend/src/metabase_enterprise/library/git.clj

@@ -0,0 +1,120 @@
+(ns metabase-enterprise.library.git
+  (:require
+   [clojure.java.io :as io]
+   [metabase.util.log :as log])
+  (:import
+   (java.io File)
+   (org.eclipse.jgit.api Git)
+   (org.eclipse.jgit.api.errors GitAPIException InvalidRemoteException TransportException)
+   (org.eclipse.jgit.transport UsernamePasswordCredentialsProvider
+                               RefSpec)))
+
+(set! *warn-on-reflection* true)
+
+(defn clone-repository!
+  "Clone git repository to a temporary directory using jgit. Returns the path to the cloned repository."
+  [url branch dir token]
+  (try
+    (log/info "Cloning repository" {:url url :branch branch :dir dir})
+    (-> (Git/cloneRepository)
+        (.setURI url)
+        (.setBranch branch)
+        (.setDirectory (io/file dir))
+        (.setCloneAllBranches false)
+        (.setCredentialsProvider (when token
+                                   (UsernamePasswordCredentialsProvider. "x-access-token" token)))
+        (.call)
+        (.close))
+    (log/info "Successfully cloned repository" {:dir dir})
+    (catch Exception e
+      (throw (ex-info (format "Failed to clone git repository: %s" (.getMessage e))
+                      {:url url
+                       :branch branch
+                       :dir dir
+                       :error (.getMessage e)})))))
+
+(defn push-branch!
+  "Oh this is terrible, not just pushing a branch but ... sticking the current content into a branch based on
+  `source-branch`, then force pushing it to the `dest-branch`."
+  [{:keys [source-dir url source-branch dest-branch token]}]
+  (let [branch-name (str (random-uuid))
+        repo-dir (io/file source-dir)]
+    (try
+      ;; Initialize or open existing repository
+      (log/info "Initializing git repository" {:dir source-dir})
+      (let [git-repo (if (.exists (io/file source-dir ".git"))
+                       (Git/open repo-dir)
+                       (-> (Git/init)
+                           (.setDirectory repo-dir)
+                           (.call)))
+            repo (.getRepository git-repo)]
+        (try
+          ;; Add origin remote if it doesn't exist
+          (log/info "Setting up remote origin" {:url url})
+          (-> (.remoteAdd git-repo)
+              (.setName "origin")
+              (.setUri (org.eclipse.jgit.transport.URIish. url))
+              (.call))
+
+          ;; Fetch from origin
+          (log/info "Fetching from origin")
+          (-> (.fetch git-repo)
+              (.setRemote "origin")
+              (.setCredentialsProvider (when token
+                                         (UsernamePasswordCredentialsProvider. "x-access-token" token)))
+              (.call))
+
+          ;; Create and checkout new branch
+          (log/info "Creating new branch" {:branch branch-name})
+          (-> (.checkout git-repo)
+              (.setOrphan true)
+              (.setName branch-name)
+              (.call))
+
+          (when-let [origin-ref (.findRef repo (str "origin/" source-branch))]
+            (-> (.reset git-repo)
+                (.setMode org.eclipse.jgit.api.ResetCommand$ResetType/MIXED)
+                (.setRef (str "origin/" source-branch))
+                (.call)))
+
+          ;; Add all changes
+          (log/info "Adding all changes")
+          (-> (.add git-repo)
+              (.addFilepattern ".")
+              (.call))
+
+          ;; Commit changes
+          (log/info "Committing changes")
+          (-> (.commit git-repo)
+              (.setMessage "Update content")
+              (.call))
+
+          ;; Push to destination branch
+          (log/info "Pushing to destination branch" {:dest-branch dest-branch})
+          (-> (.push git-repo)
+              (.setRemote "origin")
+              (.setRefSpecs [(RefSpec. (str "refs/heads/" branch-name ":refs/heads/" dest-branch))])
+              (.setForce true)
+              (.setCredentialsProvider
+               (when token
+                 (UsernamePasswordCredentialsProvider. "x-access-token" token)))
+              (.call))
+
+          (finally
+            (.close git-repo))))
+      (catch Exception e
+        (log/error e "Git push failed")
+        (throw e)))))
+
+(defn can-access-branch-in-repository? [git-url branch token]
+  (try
+    (let [credentials-provider (when token (UsernamePasswordCredentialsProvider. "x-access-token" token))
+          ls-remote-command (-> (Git/lsRemoteRepository)
+                                (.setRemote git-url)
+                                (.setCredentialsProvider credentials-provider))
+          refs (.call ls-remote-command)
+          branch-ref-name (str "refs/heads/" branch)]
+      (boolean (some #(= (.getName %) branch-ref-name) refs)))
+    (catch InvalidRemoteException e false)
+    (catch TransportException e false)
+    (catch GitAPIException e false)))

enterprise/backend/src/metabase_enterprise/library/init.clj

@@ -0,0 +1,3 @@
+(ns metabase-enterprise.library.init
+  (:require
+   [metabase-enterprise.library.settings]))

enterprise/backend/src/metabase_enterprise/library/settings.clj

@@ -0,0 +1,74 @@
+(ns metabase-enterprise.library.settings
+  (:require
+   [metabase-enterprise.library.git :as git]
+   [metabase.settings.core :as setting :refer [defsetting]]
+   [metabase.util.i18n :refer [deferred-tru]]))
+
+(defsetting git-sync-configured
+  (deferred-tru "Whether git sync is configured.")
+  :type       :boolean
+  :visibility :authenticated
+  :export?    false
+  :encryption :no
+  :default    false)
+
+(defn- set-repo-with-verification!-fn [original-key]
+  (fn [new-value]
+    (setting/set-value-of-type! :string original-key new-value)
+    (if (git/can-access-branch-in-repository? (setting/get :git-sync-url)
+                                              (setting/get :git-sync-import-branch)
+                                              (setting/get :git-sync-token))
+      (git-sync-configured! true)
+      (git-sync-configured! false))))
+
+(defsetting git-sync-import-branch
+  (deferred-tru "The git branch to pull from, e.g. `main`")
+  :type :string
+  :visibility :admin
+  :encryption :no
+  :export? false
+  :default "main"
+  :setter (set-repo-with-verification!-fn :git-sync-import-branch))
+
+(defsetting git-sync-token
+  (deferred-tru "A GH token")
+  :type :string
+  :visibility :admin
+  :doc true
+  :export? false
+  :encryption :when-encryption-key-set
+  :audit :getter
+  :setter (set-repo-with-verification!-fn :git-sync-token))
+
+(defsetting git-sync-url
+  (deferred-tru "The location of your git repository, e.g. https://github.com/acme-inco/metabase.git")
+  :type       :string
+  :visibility :admin
+  :encryption :no
+  :export?    false
+  :setter (set-repo-with-verification!-fn :git-sync-url))
+
+;; TODO actually use this
+(defsetting git-sync-key
+  (deferred-tru "An RSA key with write permissions to the git repository.")
+  :type       :string
+  :visibility :admin
+  :doc        true
+  :export?    false
+  :encryption :when-encryption-key-set
+  :audit      :getter)
+
+(defsetting git-sync-export-branch
+  (deferred-tru "The git branch to push to, e.g. `new-changes`")
+  :type :string
+  :visibility :admin
+  :encryption :no
+  :export? false
+  :default "main")
+
+(defsetting git-sync-read-only
+  (deferred-tru "Whether the instance is synced to git and read-only")
+  :type :boolean
+  :visibility :admin
+  :export? false
+  :default false)

enterprise/backend/src/metabase_enterprise/library/sources.clj

@@ -0,0 +1,62 @@
+(ns metabase-enterprise.library.sources
+  (:require
+   [clojure.java.io :as io]
+   [metabase-enterprise.library.git :as git]
+   [metabase-enterprise.library.settings :as settings]
+   [metabase.util.log :as log])
+  (:import
+   (java.io File)
+   (org.eclipse.jgit.api Git)
+   (org.eclipse.jgit.transport UsernamePasswordCredentialsProvider
+                               RefSpec)))
+
+(set! *warn-on-reflection* true)
+
+(defprotocol ISource
+  (load-source! [this dir]
+    "Loads the source, taking a directory and returning an absolute path to the source directory that can be loaded
+    with serdes.
+    TODO: working with directories (taking a directory and modifying it and returning a path) isn't really the best
+    API but it'll do for now."))
+
+(defprotocol IDestination
+  (push-branch! [this source-dir]
+    "Pushes the contentents of `source-dir` to the branch specified."))
+
+(defn get-serdes-path
+  "Get the full path to the serdes files within the repository."
+  [repo-dir path]
+  (let [full-path (if (= path ".")
+                    repo-dir
+                    (str repo-dir File/separator path))]
+
+    (when-not (.exists (io/file full-path))
+      (throw (ex-info (format "Serdes path does not exist: %s" full-path)
+                      {:repo-dir repo-dir
+                       :config-path path
+                       :full-path full-path})))
+
+    full-path))
+
+(defrecord GitSource [url source-branch token dest-branch]
+  ISource
+  (load-source! [_this dir]
+    (git/clone-repository! url source-branch dir token)
+    dir)
+
+  IDestination
+  (push-branch! [_this source-dir]
+    (git/push-branch! {:source-dir source-dir
+                       :url url
+                       :source-branch source-branch
+                       :dest-branch dest-branch
+                       :token token})))
+
+(defn get-source
+  "Returns the source that we should actually use. A source implements the `ISource` protocol."
+  []
+  (when (settings/git-sync-url)
+    (->GitSource (settings/git-sync-url)
+                 (settings/git-sync-import-branch)
+                 (settings/git-sync-token)
+                 (settings/git-sync-export-branch))))

enterprise/backend/src/metabase_enterprise/mbml/core.clj

@@ -0,0 +1,286 @@
+(ns metabase-enterprise.mbml.core
+  "mbml is a file format for creating human readable files that can be loaded into metabase
+  allowing users to manage important metabase entities using version control independently
+  from specific metabase instances.
+
+  mbml is written either as a YAML file or as front-matter to Python or SQL files all of the
+  following are equivalent ways to declare the same transform.
+
+  ```yaml
+  entity: model/Transform:v1
+  name: My Coole Transform
+  identifier: my-coole-transform
+  description: Only making the most legit changes
+  tags:
+  - hourly
+  database: db-name
+  target: my_taget_table
+  source: | #sql
+    SELECT * FROM source_table_name WHERE type = 'gadget';
+  ```
+
+  ```sql
+  -- METABSE_BEGIN
+  -- entity: model/Transform:v1
+  -- name: My Coole Transform
+  -- identifier: my-coole-transform
+  -- description: Only making the most legit changes
+  -- tags:
+  -- - hourly
+  -- database: db-name
+  -- target: my_taget_table
+  -- METABASE_END
+
+  SELECT * FROM source_table_name WHERE type = 'gadget';
+  ```
+
+  ```python
+  # METABASE_BEGIN
+  # entity: model/Transform:v1
+  # name: My Coole Transform
+  # identifier: my-coole-transform
+  # description: Only making the most legit changes
+  # tags:
+  # - hourly
+  # database: db-name
+  # target: my_taget_table
+  # METABASE_END
+  import pandas as pd
+
+  df = pd.read_csv(\"source.csv\")
+  df[df[\"type\"] == 'gadget']
+  ```
+  "
+  (:require
+   [babashka.fs :as fs]
+   [clojure.string :as str]
+   [metabase-enterprise.mbml.errors :as mbml.errors]
+   [metabase-enterprise.mbml.parser :as mbml.parser]
+   [metabase-enterprise.transforms.core :as transforms]
+   [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
+   [metabase.models.serialization :as serdes]
+   [metabase.lib.core :as lib]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2]))
+
+ ;;; ------------------------------------------ Main Parser API -----------------------------------------------
+
+(mu/defn parse-mbml-string :- :map
+  "Parse MBML content from a string with optional file type hint.
+
+  Takes content string and optional file type hint, parses and validates the MBML.
+  This is the main public API function for parsing MBML content directly from strings.
+
+  Args:
+    content: MBML content as string
+    file-type-hint: Optional keyword file type (:yaml, :sql, :python) to override detection
+
+  Returns:
+    Validated MBML entity map or throws exception on error
+
+  Example:
+    (parse-mbml-string yaml-content :yaml)
+    (parse-mbml-string sql-with-frontmatter :sql)"
+  [content :- ms/NonBlankString
+   file-type-hint :- [:maybe mbml.parser/FileType]]
+  (let [file-type (or file-type-hint :yaml)
+        {:keys [metadata body]} (mbml.parser/extract-content content file-type)]
+    (if metadata
+      (let [parsed-data (mbml.parser/parse-yaml metadata)]
+        (mbml.parser/validate-mbml parsed-data body))
+      (throw (ex-info "No MBML metadata found in content"
+                      {:type :no-metadata-error
+                       :content-preview (subs content 0 (min 200 (count content)))
+                       :file-type file-type})))))
+
+(mu/defn parse-mbml-file :- :map
+  "Parse MBML file from disk, detecting type, parsing and validating.
+
+  Reads file from disk, detects file type from extension, extracts and parses MBML content,
+  and validates against schema. This is the main public API function for parsing MBML files.
+
+  Args:
+    file-path: Path to MBML file (string)
+
+  Returns:
+    Validated MBML entity map or throws exception on error
+
+  Throws :
+    Exception for file reading errors, parse errors, or validation errors
+
+  Example:
+    (parse-mbml-file \"/path/to/transform.yaml\")
+    (parse-mbml-file \"/path/to/transform.sql\")"
+  [file-path :- ms/NonBlankString]
+  (try
+    (when-not (fs/exists? file-path)
+      (throw (mbml.errors/format-file-error :file-not-found file-path nil)))
+
+    (let [content (slurp file-path)
+          file-type (mbml.parser/detect-file-type file-path)]
+      (when (= :unknown file-type)
+        (throw (mbml.errors/format-unsupported-file-error file-path)))
+      (when (str/blank? content)
+        (throw (mbml.errors/format-file-error :empty-file-error file-path nil)))
+
+      (binding [mbml.parser/*file* file-path]
+        (parse-mbml-string content file-type)))
+
+    (catch java.io.FileNotFoundException e
+      (throw (mbml.errors/format-file-error :file-not-found file-path e)))
+    (catch java.io.IOException e
+      (throw (mbml.errors/format-file-error :file-io-oerror file-path e)))))
+
+(def ^:private entity-type->model
+  {"model/Transform:v1" :model/Transform})
+
+(defmulti save-model!
+  "Impl for multi-entity dispatch for mbml-file->model. Implement to support versioned
+  deserialization of a model from an mbml-parsed entity map."
+  {:arglists '([entity model existing-entity])}
+  (fn [entity model _existing-entity] (keyword entity)))
+
+(defmulti mbml-file->unsaved-model*
+  "Transformer for mbml-files into content that can be saved as a metabase model
+
+  Args:
+    mbml-map: parsed and validated mbml map
+
+  Returns:
+    map of the model that can be inserted or updated into the appdb
+
+  Raises:
+    ex-info objects when given data that cannot be transformed."
+  {:arglists '([mbml-map])}
+  (fn [{:keys [entity]}] (keyword entity)))
+
+(defmethod mbml-file->unsaved-model* :default
+  [{:keys [identifier] :as mbml-map}]
+  (-> mbml-map
+      (assoc :library_identifer identifier)
+      (dissoc :identifier :entity)))
+
+(defmethod save-model! :default
+  [entity model {existing-entity-id :id}]
+  (let [model-kw (entity-type->model entity)]
+    (if existing-entity-id
+      (do (t2/update! model-kw :id existing-entity-id model)
+          (t2/select-one model-kw))
+      (t2/insert-returning-instance! model-kw model))))
+
+(defn- update-source-query
+  [{:keys [source body] :as model} database-id]
+  (let [raw-query (or body source)
+        metadata-provider (lib.metadata.jvm/application-database-metadata-provider database-id)
+        resolved-query (cond
+                         (string? raw-query) (serdes/import-mbql raw-query)
+                         (map? raw-query) (lib/query metadata-provider raw-query))]
+    (assoc model :source {:type "query"
+                          :query resolved-query})))
+
+(defmethod mbml-file->unsaved-model* :model/Transform:v1
+  [{:keys [tags database identifier] :as mbml-map}]
+  (let [tag-ids (t2/select-pks-vec :model/TransformTag :name [:in tags])
+        database-id (t2/select-one-pk :model/Database :name database)]
+    (when-not (= (count tag-ids) (count (set tags)))
+      (throw (mbml.errors/format-model-transformation-error :missing-tags :model/Transform mbml-map mbml.parser/*file*)))
+    (when-not database-id
+      (throw (mbml.errors/format-model-transformation-error :database-id :model/Transform mbml-map mbml.parser/*file*)))
+    (-> mbml-map
+        (assoc :library_identifier identifier)
+        (update-source-query database-id)
+        (dissoc :entity :tags :body :identifier :database)
+        (assoc :tag_ids tag-ids))))
+
+;; TODO(edpaget): Probably belongs in the transform module
+(defmethod save-model! :model/Transform:v1
+  [_ model {existing-entity-id :id}]
+  (if existing-entity-id
+    (transforms/update-transform! existing-entity-id model)
+    (transforms/create-transform! model)))
+
+(defn mbml-file->unsaved-model
+  "Take an mbml file and create a model that could be saved to the database
+
+  Implment the mbml-file->unsaved-model* multimethod to have it support a new entity-type.
+
+  Args:
+    file-path: Path to MBML file (string)
+
+  Returns:
+    tuple of entity, identifier, and map compatible with a saved toucan entity
+
+  Raises:
+    Exception for file reading errors, parse errors, or validation errors
+
+  Example:
+    (mbml-file->model \"/path/to/transform.yaml\")
+    (mbml-file->model \"/path/to/transform.sql\")"
+  [file-path]
+  (binding [mbml.parser/*file* file-path]
+    (let [{:keys [identifier entity] :as mbml-map} (parse-mbml-file file-path)]
+      [entity identifier (mbml-file->unsaved-model* mbml-map)])))
+
+(defn mbml-file->model
+  "Take an mbml file and write it to the appdb.
+
+  Implment the mbml-file->model* multimethod to have it support a new entity-type.
+
+  Args:
+    file-path: Path to MBML file (string)
+
+  Returns:
+    A toucan instance
+
+  Throws :
+    Exception for file reading errors, parse errors, or validation errors
+
+  Example:
+    (mbml-file->model \"/path/to/transform.yaml\")
+    (mbml-file->model \"/path/to/transform.sql\")"
+  [file-path]
+  (binding [mbml.parser/*file* file-path]
+    (let [[entity identifier model] (mbml-file->unsaved-model file-path)
+          existing-entity (t2/select-one (entity-type->model entity) :library_identifier identifier)]
+      (try
+        (save-model! entity model existing-entity)
+        (catch Exception e
+          (throw (mbml.errors/format-model-transformation-error :default :model/Transform model mbml.parser/*file* e)))))))
+
+(mu/defn mbml-files->models :- [:sequential :map]
+  "Load multiple MBML files transactionally and clean up orphaned models.
+
+  Processes all files within a single transaction, creating or updating models
+  based on their library_identifier. After loading, removes any models with
+  library_identifier values that don't match the loaded files.
+
+  Args:
+    file-paths: Collection of paths to MBML files
+
+  Returns:
+    Collection of created/updated model instances
+
+  Raises:
+    Exception if any file fails to load (entire transaction rolls back)
+
+  Example:
+    (mbml-files->models [\"transform1.yaml\" \"transform2.sql\"])"
+  [file-paths :- [:sequential ms/NonBlankString]]
+  (t2/with-transaction []
+    (let [models (mapv mbml-file->model file-paths)
+          loaded-identifiers (into #{}
+                                   (keep :library_identifier)
+                                   models)]
+
+      (doseq [model-type (vals entity-type->model)]
+        (when (seq loaded-identifiers)
+          ;; Delete models with library_identifier not in loaded set
+          ;; Models without library_identifier (nil) are preserved
+          (t2/delete! model-type
+                      {:where [:and
+                               [:not-in :library_identifier loaded-identifiers]
+                               [:not= :library_identifier nil]]})))
+
+      ;; Return the loaded models
+      models)))

enterprise/backend/src/metabase_enterprise/mbml/errors.clj

@@ -0,0 +1,242 @@
+(ns metabase-enterprise.mbml.errors
+  "Comprehensive error handling utilities for MBML parser.
+
+  Provides structured error formatting, internationalized messages using deferred-tru,
+  and integration with Metabase's error handling patterns including mu/with-api-error-message
+  and metabase.util.malli.humanize."
+  (:require
+   [clojure.string :as str]
+   [java-time.api :as t]
+   [metabase.util.i18n :as i18n :refer [deferred-tru]]))
+
+;;; ------------------------------------------ Error Type Definitions ------------------------------------------
+
+(def ^:const error-types
+  "Map of error type keywords to human-readable descriptions."
+  {:file-not-found (deferred-tru "File not found")
+   :empty-file-error (deferred-tru "File is empty")
+   :unsupported-format (deferred-tru "Unsupported file format")
+   :yaml-parse-error (deferred-tru "YAML parsing error")
+   :schema-validation (deferred-tru "Schema validation error")
+   :frontmatter-error (deferred-tru "Front-matter extraction error")
+   :model-transformation-error (deferred-tru "Failed to create or update model")
+   :io-error (deferred-tru "File I/O error")})
+
+;;; ------------------------------------------ Error Context Helpers -------------------------------------------
+
+(defn error-context
+  "Create error context map with common fields for MBML errors.
+
+  Args:
+    type: Keyword error type from error-types map
+    message: Human-readable error message (string or deferred-tru)
+    opts: Optional map with additional context (:file, :line, :cause, :data, etc.)
+
+  Returns:
+    ex-info object following Metabase error patterns"
+  [type message & [opts]]
+  (let [opts-map (or opts {})
+        msg-str (if (string? message) message (str message))
+        ; Also convert context and suggestions to strings
+        context-str (when-let [ctx (:context opts-map)]
+                      (if (string? ctx) ctx (str ctx)))
+        suggestions-str (when-let [suggs (:suggestions opts-map)]
+                          (mapv #(if (string? %) % (str %)) suggs))]
+    (ex-info msg-str
+             (merge
+              {:type type
+               :message msg-str
+               :timestamp (t/instant)}
+              (select-keys opts-map [:file :line :data])
+              (when context-str {:context context-str})
+              (when suggestions-str {:suggestions suggestions-str}))
+             (:cause opts-map))))
+
+(defn format-file-context
+  "Format file context information for error messages.
+
+  Args:
+    file: File path or filename
+    line: Optional line number
+
+  Returns:
+    Formatted string with file context"
+  [file & [line]]
+  (if line
+    (i18n/tru "in file {0} at line {1}" file line)
+    (i18n/tru "in file {0}" file)))
+
+;;; ------------------------------------------ YAML Error Handling ----------------------------------------------
+
+(defn format-yaml-error
+  "Format YAML parsing errors with user-friendly context.
+
+  Args:
+    yaml-content: Original YAML content that failed to parse
+    cause: Original exception/error message
+    file: Optional file context
+
+  Returns:
+    ex-info object with actionable suggestions"
+  [yaml-content cause & [file]]
+  (let [lines (str/split-lines (or yaml-content ""))
+        line-count (count lines)]
+    (error-context
+     :yaml-parse-error
+     (if file
+       (deferred-tru "Failed to parse YAML content {0}" (format-file-context file))
+       (deferred-tru "Failed to parse YAML content"))
+     {:cause cause
+      :file file
+      :data {:yaml-content yaml-content
+             :line-count line-count}
+      :context (when (> line-count 10)
+                 (deferred-tru "YAML content has {0} lines - check for indentation or syntax errors" line-count))
+      :suggestions [(deferred-tru "Check YAML syntax - ensure proper indentation with spaces (not tabs)")
+                    (deferred-tru "Verify that strings with special characters are properly quoted")
+                    (deferred-tru "Make sure lists and maps use consistent formatting")]})))
+
+;;; ---------------------------------------- Schema Validation Errors -------------------------------------------
+
+(defn format-schema-validation-error
+  "Format Malli schema validation errors using metabase.util.malli.humanize.
+
+  Args:
+    exception: Malli exception
+    data: data that failed to validate
+
+  Returns:
+    ex-info object with humanized validation messages"
+  [exception data & [file]]
+  (error-context
+   :schema-validation
+   (if file
+     (deferred-tru "MBML validation failed {0}" (format-file-context file))
+     (deferred-tru "MBML validation failed"))
+   {:file file
+    :cause exception
+    :data (merge {:failed-data data}
+                 (-> exception ex-data :error))
+    :context (deferred-tru "The MBML entity structure does not match required schema")
+    :suggestions [(deferred-tru "Ensure all required fields are present: entity, name, identifier, database, target")
+                  (deferred-tru "Check that entity type is a supported value (e.g., '''model/Transform:v1''')")
+                  (deferred-tru "Verify field values are the correct type (strings, arrays, etc.)")]}))
+
+;;; ---------------------------------------- Front-matter Error Handling ----------------------------------------
+
+(defn format-frontmatter-error
+  "Format front-matter extraction errors with context and suggestions.
+
+  Args:
+    error-type: Specific front-matter error (:missing-markers, :malformed-markers, etc.)
+    content: File content that failed processing
+    file: Optional file context
+
+  Returns:
+    ex-info object with file format specific guidance"
+  [error-type content & [file]]
+  (let [file-ext (when file
+                   (last (str/split file #"\.")))
+        suggestions [(deferred-tru "Add METABASE_BEGIN and METABASE_END markers to your file")
+                     (if (= file-ext "sql")
+                       (deferred-tru "For SQL files, use: -- METABASE_BEGIN and -- METABASE_END")
+                       (deferred-tru "For Python files, use: # METABASE_BEGIN and # METABASE_END"))]]
+    (error-context
+     :frontmatter-error
+     (if file
+       (deferred-tru "Front-matter extraction failed {0}" (format-file-context file))
+       (deferred-tru "Front-matter extraction failed"))
+     {:file file
+      :data {:content-length (count content)
+             :file-extension file-ext
+             :error-type error-type}
+      :context (deferred-tru "Unable to extract YAML metadata from {0} file" (or file-ext "source"))
+      :suggestions suggestions})))
+
+;;; ----------------------------------------- File System Errors --------------------------------------------
+
+(defn format-file-error
+  "Format file system related errors (not found, permission, I/O).
+
+  Args:
+    error-type: File error type (:file-not-found, :io-error, :permission-denied)
+    file: File path that caused the error
+    cause: Original exception or error message
+
+  Returns:
+    ex-info object with file system context"
+  [error-type file cause]
+  (let [suggestions (case error-type
+                      :file-not-found
+                      [(deferred-tru "Check that the file path is correct")
+                       (deferred-tru "Verify the file exists and is readable")
+                       (deferred-tru "Ensure you have permission to access the file")]
+
+                      :permission-denied
+                      [(deferred-tru "Check file permissions - ensure the file is readable")
+                       (deferred-tru "Verify you have access to the parent directory")]
+
+                      :io-error
+                      [(deferred-tru "Check that the file is not locked by another process")
+                       (deferred-tru "Ensure sufficient disk space and system resources")]
+
+                      [(deferred-tru "Check file accessibility and permissions")])]
+    (error-context
+     error-type
+     (deferred-tru "{0} for file: {1}" (get error-types error-type) file)
+     {:cause cause
+      :file file
+      :suggestions suggestions})))
+
+;;; ----------------------------------------- Format Detection Errors -------------------------------------
+
+(defn format-unsupported-file-error
+  "Format errors for unsupported file types.
+
+  Args:
+    file: File path with unsupported extension
+    detected-type: The file type that was detected
+
+  Returns:
+    ex-info obj explaining supported formats"
+  [file]
+  (error-context
+   :unsupported-format
+   (deferred-tru "Unsupported file format {0}" (format-file-context file))
+   {:file file
+    :context (deferred-tru "MBML files must use supported file extensions")
+    :suggestions [(deferred-tru "Use .yaml or .yml for pure YAML files")
+                  (deferred-tru "Use .sql for SQL files with YAML front-matter")
+                  (deferred-tru "Use .py for Python files with YAML front-matter")]}))
+
+;;; ----------------------------------------- Model Transformation Errors ---------------------------------
+
+(defn format-model-transformation-error
+  "Format errors when we fail to transform the result of a parse into a metabase model.
+
+  Args:
+    error-type: the type fof the transformation failure
+    model-type: the model type that error during construction
+    data: the map of data that was used to construct the model
+    file: the file data was read from
+    cause (optional): the original exception this is wrapping
+
+  Returns:
+    ex-info obj explaining the failure"
+  [error-type model-type data file & [cause]]
+  (error-context
+   :model-transformation-error
+   (case error-type
+     :missing-tags
+     (deferred-tru "Supplied tags do not exist on this instance: {0}" (->> data :tags (str/join ", ")))
+
+     :database-id
+     (deferred-tru "The database does not exist on this instance: {0}" (:database data))
+
+     (deferred-tru "Failed to create or update model: {0}" model-type))
+   {:file file
+    :data {:model-data data
+           :error-type error-type
+           :model-type model-type}
+    :cause cause
+    :context (deferred-tru "Failure during validation or construction of the metabase model.")}))

enterprise/backend/src/metabase_enterprise/mbml/parser.clj

@@ -0,0 +1,165 @@
+(ns metabase-enterprise.mbml.parser
+  "Front-matter extraction and YAML parsing functions for MBML files.
+
+  Supports extraction of YAML metadata from SQL and Python files using
+  METABASE_BEGIN/METABASE_END comment markers, plus YAML parsing and validation."
+  (:require
+   [clojure.string :as str]
+   [metabase-enterprise.mbml.errors :as mbml.errors]
+   [metabase-enterprise.mbml.schema :as mbml.schema]
+   [metabase.util :as u]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.schema :as ms]
+   [metabase.util.yaml :as yaml]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *file*
+  "Binding for the name of the current file being parsed for error reporting."
+  nil)
+
+(def ^:private comment-styles
+  {:sql ["--" #"^--\s?"]
+   :python ["#" #"^#\s?"]})
+
+(def ^:private extract-regex
+  {:sql    #"(?s)^\s*--\s*METABASE_BEGIN\s*\n(.*?)\n\s*--\s*METABASE_END\s*\n?(.*)"
+   :python #"(?s)^\s*#\s*METABASE_BEGIN\s*\n(.*?)\n\s*#\s*METABASE_END\s*\n?(.*)"})
+
+(defn- strip-comments
+  [comment-type]
+  (let [[comment regex] (comment-styles comment-type)]
+    (fn [line]
+      (if (str/starts-with? (str/triml line) comment)
+        (str/replace-first (str/triml line) regex "")
+        line))))
+
+(mu/defn extract-frontmatter :- [:map [:metadata [:maybe :string]]
+                                 [:body [:maybe :string]]]
+  "Extract YAML front-matter from file content.
+
+  Looks for content between 'METABASE_BEGIN' and 'METABASE_END' markers, in either
+  sql-style or python-style comments. It strips the comment prefix and returns both
+  the metadata and the remaining source code.
+
+  Args:
+    content: file content as string
+
+  Returns:
+    Map with :metadata (YAML string) and :body (remaining Python code).
+    Returns {:metadata nil :body content} if no front-matter markers found.
+
+  Raises:
+    ex-info when unable to make a match"
+  [content :- :string
+   content-type :- [:enum :sql :python]]
+  (if-let [match (re-find (extract-regex content-type) content)]
+    (let [[_ frontmatter-raw body] match
+          metadata (->> frontmatter-raw
+                        str/split-lines
+                        (map (strip-comments content-type))
+                        (str/join "\n"))]
+      {:metadata (when-not (str/blank? metadata) metadata)
+       :body     (str/triml body)})
+    (throw (mbml.errors/format-frontmatter-error :missing-markers content *file*))))
+
+ ;; Define the file type schema
+(def FileType
+  "File types we can operate on"
+  [:enum :yaml :sql :python :unknown])
+
+(mu/defn detect-file-type :- FileType
+  "Determine file type based on extension or content.
+
+   Takes filename/path and returns :yaml, :sql, :python, or :unknown
+
+   Args:
+     filename: String filename or path
+
+   Returns:
+     Keyword representing the file type"
+  [filename :- [:maybe :string]]
+  (if filename
+    (let [lower-name (u/lower-case-en filename)]
+      (cond
+        (or (str/ends-with? lower-name ".yml")
+            (str/ends-with? lower-name ".yaml")) :yaml
+        (str/ends-with? lower-name ".sql") :sql
+        (str/ends-with? lower-name ".py") :python
+        :else :unknown))
+    :unknown))
+
+(mu/defn extract-content :- [:map
+                             [:metadata [:maybe :string]]
+                             [:body [:maybe :string]]]
+  "Extract content from file based on file type.
+
+   Takes file content and type, returns {:metadata yaml-string :body code-string}
+   - For YAML files: entire content is metadata, source is nil
+   - For SQL/Python: use the extraction functions from Step 2
+   - For unknown: return error or nil metadata
+
+   Args:
+     content: File content as string
+     file-type: Keyword representing file type (:yaml, :sql, :python, :unknown)
+
+   Returns:
+     Map with :metadata and :body keys"
+  [content :- ms/NonBlankString
+   file-type :- FileType]
+  (case file-type
+    :yaml {:metadata content :body nil}
+    :sql (extract-frontmatter content :sql)
+    :python (extract-frontmatter content :python)
+    :unknown {:metadata nil :body content}))
+
+ ;;; ------------------------------------------ YAML Parsing ---------------------------------------------------
+
+(mu/defn parse-yaml :- [:maybe :map]
+  "Parse YAML string into Clojure data structure.
+
+  Uses metabase.util.yaml to parse YAML content with error handling.
+  Returns nil for empty/blank input or throws exception for malformed YAML.
+
+  Args:
+    yaml-content: YAML string content to parse
+
+  Returns:
+    Parsed Clojure map or nil if content is blank
+
+  Throws:
+    Exception with structured error details for malformed YAML"
+  [yaml-content :- [:maybe :string]]
+  (when-not (str/blank? yaml-content)
+    (try
+      (yaml/parse-string yaml-content)
+      (catch Exception e
+        ;; Use structured error from errors namespace for better user experience
+        (throw (mbml.errors/format-yaml-error yaml-content e *file*))))))
+
+;;; ---------------------------------------- MBML Validation -------------------------------------------------
+
+(mu/defn validate-mbml :- :map
+  "Validate parsed MBML data against schema and integrate source code.
+
+  Takes parsed YAML data and optional source code, validates against MBML schemas,
+  and returns validated map with source field populated if provided.
+
+  Args:
+    parsed-data: Clojure map from YAML parsing
+    source-code: Optional source code string to add to validated entity
+
+  Returns:
+    Validated MBML entity map with source code integrated
+
+  Throws:
+    Exception with validation errors if data doesn't match MBML schema"
+  [parsed-data :- :map
+   source-code :- [:maybe :string]]
+  (try
+    (let [validated-data (mu/validate-throw ::mbml.schema/mbml-entity parsed-data)]
+      (if source-code
+        (assoc validated-data :body source-code)
+        validated-data))
+    (catch Exception e
+      (throw (mbml.errors/format-schema-validation-error e parsed-data *file*)))))

enterprise/backend/src/metabase_enterprise/mbml/schema.clj

@@ -0,0 +1,91 @@
+(ns metabase-enterprise.mbml.schema
+  "Malli schema definitions for MBML (Metabase Markup Language) entities.
+  
+  Defines schemas for validating MBML entities like Transform:v1 using Metabase's
+  established Malli patterns with mr/def registry and user-friendly error messages."
+  (:require
+   [metabase.util.i18n :refer [deferred-tru]]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]))
+
+;;; ------------------------------------------ Entity Type Schema -----------------------------------------------
+
+(mr/def ::entity-type
+  "Schema for valid MBML entity types. Currently supports Transform:v1."
+  (mu/with-api-error-message
+   [:enum "model/Transform:v1"]
+   (deferred-tru "entity must be a valid MBML entity type (e.g. \"model/Transform:v1\")")))
+
+;;; ------------------------------------------ Common Field Schemas ----------------------------------------------
+
+(mr/def ::identifier
+  "Schema for MBML entity identifiers. Must be a non-blank string."
+  (mu/with-api-error-message
+   ms/NonBlankString
+   (deferred-tru "identifier must be a non-blank string")))
+
+(mr/def ::name
+  "Schema for MBML entity names. Must be a non-blank string."
+  (mu/with-api-error-message
+   ms/NonBlankString
+   (deferred-tru "name must be a non-blank string")))
+
+(mr/def ::database
+  "Schema for database references. Must be a non-blank string."
+  (mu/with-api-error-message
+   ms/NonBlankString
+   (deferred-tru "database must be a non-blank string")))
+
+(mr/def ::target
+  "Schema for target table/view names."
+  [:map
+   [:type [:enum "table"]]
+   [:schema {:optional true} ms/NonBlankString]
+   [:name ms/NonBlankString]])
+
+(mr/def ::description
+  "Schema for entity descriptions. Optional string field."
+  [:maybe :string])
+
+(mr/def ::tags
+  "Schema for entity tags. Optional array of strings."
+  [:maybe [:sequential :string]])
+
+(mr/def ::source
+  "Schema for source code content (SQL, Python, or MBQL).
+
+  Optionally a string for SQL or Python or a map for MBQL.
+
+  It does not seem like incoming MBQL queries are validating in other imports (ex POST /api/card),
+  so this also does not attempt to validate the query."
+  [:maybe [:or :string :map]])
+
+;;; ------------------------------------------ Transform:v1 Schema -----------------------------------------------
+
+(mr/def ::transform-v1
+  "Schema for Transform:v1 MBML entities. Validates all required and optional fields
+  according to the MBML specification."
+  (mu/with-api-error-message
+   [:map
+    {:closed true}
+    [:entity ::entity-type]
+    [:name ::name]
+    [:identifier ::identifier]
+    [:database ::database]
+    [:target ::target]
+    [:description {:optional true} ::description]
+    [:tags {:optional true} ::tags]
+    [:source {:optional true} ::source]]
+   (deferred-tru "must be a valid Transform:v1 entity with all required fields")))
+
+;;; ------------------------------------------ General MBML Entity Schema ------------------------------------
+
+(mr/def ::mbml-entity
+  "General schema for any MBML entity. Currently only supports Transform:v1
+  but can be extended to support additional entity types in the future."
+  (mu/with-api-error-message
+   [:multi {:dispatch :entity
+            :error/message "invalid entity type"}
+    ["model/Transform:v1" ::transform-v1]]
+   (deferred-tru "must be a valid MBML entity")))

enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj

@@ -32,7 +32,10 @@
     (into serdes.models/data-model)
 
     (not (:no-settings opts))
-    (conj "Setting")))
+    (conj "Setting")
+
+    (not (:no-transforms opts))
+    (conj "Transform" "TransformTag" "TransformJob")))
 
 (defn targets-of-type
   "Returns target seq filtered on given model name"

enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj

@@ -16,6 +16,13 @@
 
 (declare load-one!)
 
+(defn- filter-by-type-whitelist
+  "Filter a list of entity paths, keeping only those whose model type is in the whitelist."
+  [entity-paths type-whitelist]
+  (filter (fn [path]
+            (contains? type-whitelist (-> path peek :model)))
+          entity-paths))
+
 (def ^:private model->circular-dependency-keys
   "Sometimes models have circular dependencies. For example, a card for a Dashboard Question has a `dashboard_id`
   pointing to the dashboard it's in. But when we try to load that dashboard, we'll create all its dashcards, and one
@@ -189,3 +196,41 @@
     ;;       ideally, we would delay the indexing somehow, or only reindex what we've loaded.
     ;;       while we're figuring that out, here's a crude stopgap.
     (search/reindex! {:async? false})))
+
+(defn load-selective!
+  "Loads entities from an ingestion source, but only those whose types are in the whitelist.
+
+   Parameters:
+   - ingestion: Any Ingestable instance (e.g., YamlIngestion)
+   - type-whitelist: Set of model names to load (e.g., #{\"Transform\"})
+   - opts: Options map (same as load-metabase!)"
+  [ingestion type-whitelist & {:keys [backfill? continue-on-error]
+                               :or   {backfill?         true
+                                      continue-on-error false}}]
+  (u/prog1
+    (t2/with-transaction [_tx]
+      (when backfill?
+        (serdes.backfill/backfill-ids!))
+
+      (let [all-contents (serdes.ingest/ingest-list ingestion)
+            ;; Filter to only load whitelisted types
+            filtered-contents (filter-by-type-whitelist all-contents type-whitelist)
+            ctx {:expanding #{}
+                 :seen      #{}
+                 :circular  #{}
+                 :ingestion ingestion
+                 :from-ids  (m/index-by :id filtered-contents)
+                 :errors    []}]
+        (log/infof "Starting selective deserialization for types %s, loading %s of %s total documents"
+                   type-whitelist (count filtered-contents) (count all-contents))
+        (reduce (fn [ctx item]
+                  (try
+                    (load-one! ctx item)
+                    (catch Exception e
+                      (when-not continue-on-error
+                        (throw e))
+                      (log/warnf (u/strip-error e "Skipping deserialization error"))
+                      (update ctx :errors conj e))))
+                ctx
+                filtered-contents)))
+    (search/reindex! {:async? false})))

enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj

@@ -26,7 +26,12 @@
           content
           ["FieldValues"
            "Metabot"
-           "Setting"]))
+           "Setting"
+           "Transform"
+           "TransformTag"
+           "TransformJob"
+           "TransformJobTransformTag"
+           "TransformTransformTag"]))
 
 (def inlined-models
   "An additional list of models which are inlined into parent entities for serialization.
@@ -92,14 +97,9 @@
    "Secret"
    "Session"
    "TaskHistory"
-   "Transform"
-   "TransformJob"
    "TransformJobRun"
-   "TransformJobTransformTag"
    "TransformRun"
    "TransformRunCancelation"
-   "TransformTag"
-   "TransformTransformTag"
    "Undo"
    "User"
    "UserKeyValue"

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -8,6 +8,7 @@
    [metabase-enterprise.transforms.models.transform-run-cancelation :as transform-run-cancelation]
    [metabase-enterprise.transforms.ordering :as transforms.ordering]
    [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase-enterprise.library.core :as library]
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
    [metabase.api.routes.common :refer [+auth]]
@@ -76,11 +77,11 @@
   [_route-params
    _query-params]
   (api/check-superuser)
-  (-> (t2/select :model/Transform)
-      (t2/hydrate :last_run :transform_tag_ids)
+  (-> (library/select-all :model/Transform)
+      (t2/hydrate :last_run)
       (->> (map #(update % :last_run transforms.util/localize-run-timestamps)))))
 
-(api.macros/defendpoint :post "/"
+#_(api.macros/defendpoint :post "/"
   "Create a new transform."
   [_route-params
    _query-params
@@ -109,18 +110,19 @@
 (api.macros/defendpoint :get "/:id"
   "Get a specific transform."
   [{:keys [id]} :- [:map
-                    [:id ms/PositiveInt]]]
+                    [:id ms/NonBlankString]]]
+  (prn "GET_TRANSFORM" id)
   (log/info "get transform" id)
   (api/check-superuser)
-  (let [{:keys [target] :as transform} (api/check-404 (t2/select-one :model/Transform id))
+  (let [{:keys [target] :as transform} (api/check-404 (library/select :model/Transform id))
         database-id (source-database-id transform)
         target-table (transforms.util/target-table database-id target :active true)]
     (-> transform
-        (t2/hydrate :last_run :transform_tag_ids)
+        (t2/hydrate :last_run)
         (u/update-some :last_run transforms.util/localize-run-timestamps)
         (assoc :table target-table))))
 
-(api.macros/defendpoint :get "/:id/dependencies"
+#_(api.macros/defendpoint :get "/:id/dependencies"
   "Get the dependencies of a specific transform."
   [{:keys [id]} :- [:map
                     [:id ms/PositiveInt]]]
@@ -152,39 +154,39 @@
                                        :limit  (request/limit)))
       (update :data #(map transforms.util/localize-run-timestamps %))))
 
-(api.macros/defendpoint :put "/:id"
-  "Update a transform."
-  [{:keys [id]} :- [:map
-                    [:id ms/PositiveInt]]
-   _query-params
-   body :- [:map
-            [:name {:optional true} :string]
-            [:description {:optional true} [:maybe :string]]
-            [:source {:optional true} ::transform-source]
-            [:target {:optional true} ::transform-target]
-            [:run_trigger {:optional true} ::run-trigger]
-            [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
-  (log/info "put transform" id)
-  (api/check-superuser)
-  (t2/with-transaction [_]
+#_(api.macros/defendpoint :put "/:id"
+    "Update a transform."
+    [{:keys [id]} :- [:map
+                      [:id ms/PositiveInt]]
+     _query-params
+     body :- [:map
+              [:name {:optional true} :string]
+              [:description {:optional true} [:maybe :string]]
+              [:source {:optional true} ::transform-source]
+              [:target {:optional true} ::transform-target]
+              [:run_trigger {:optional true} ::run-trigger]
+              [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
+    (log/info "put transform" id)
+    (api/check-superuser)
+    (t2/with-transaction [_]
     ;; Cycle detection should occur within the transaction to avoid race
-    (let [old (t2/select-one :model/Transform id)
-          new (merge old body)
-          target-fields #(-> % :target (select-keys [:schema :name]))]
-      (check-database-feature new)
-      (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
-        (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
-                        {:status-code 400})))
-      (api/check (not (and (not= (target-fields old) (target-fields new))
-                           (transforms.util/target-table-exists? new)))
-                 403
-                 (deferred-tru "A table with that name already exists.")))
+      (let [old (t2/select-one :model/Transform id)
+            new (merge old body)
+            target-fields #(-> % :target (select-keys [:schema :name]))]
+        (check-database-feature new)
+        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                          {:status-code 400})))
+        (api/check (not (and (not= (target-fields old) (target-fields new))
+                             (transforms.util/target-table-exists? new)))
+                   403
+                   (deferred-tru "A table with that name already exists.")))
 
-    (t2/update! :model/Transform id (dissoc body :tag_ids))
+      (t2/update! :model/Transform id (dissoc body :tag_ids))
     ;; Update tag associations if provided
-    (when (contains? body :tag_ids)
-      (transform.model/update-transform-tags! id (:tag_ids body)))
-    (t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids)))
+      (when (contains? body :tag_ids)
+        (transform.model/update-transform-tags! id (:tag_ids body)))
+      (t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids)))
 
 (api.macros/defendpoint :delete "/:id"
   "Delete a transform."
@@ -217,10 +219,10 @@
 (api.macros/defendpoint :post "/:id/run"
   "Run a transform."
   [{:keys [id]} :- [:map
-                    [:id ms/PositiveInt]]]
+                    [:id ms/NonBlankString]]]
   (log/info "run transform" id)
   (api/check-superuser)
-  (let [transform (api/check-404 (t2/select-one :model/Transform id))
+  (let [transform (api/check-404 (library/select :model/Transform id))
         start-promise (promise)]
     (u.jvm/in-virtual-thread*
      (transforms.execute/run-mbql-transform! transform {:start-promise start-promise

enterprise/backend/src/metabase_enterprise/transforms/core.clj

@@ -1,8 +1,111 @@
 (ns metabase-enterprise.transforms.core
   "API namespace for the `metabase-enterprise.transform` module."
   (:require
+   [metabase-enterprise.transforms.models.transform :as transform.model]
+   [metabase-enterprise.transforms.ordering :as transforms.ordering]
+   [metabase-enterprise.transforms.schema :as t.schema]
    [metabase-enterprise.transforms.settings]
-   [potemkin :as p]))
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver.util :as driver.u]
+   [metabase.lib.core :as lib]
+   [metabase.util.i18n :refer [deferred-tru]]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.schema :as ms]
+   [potemkin :as p]
+   [toucan2.core :as t2]))
+
+(defn- source-database-id
+  [transform]
+  (-> transform :source :query lib/database-id))
+
+(defn error-or-obj
+  [obj-or-falsey message]
+  (when-not obj-or-falsey
+    (throw (ex-info (str message) {:type :validate-transform})))
+  obj-or-falsey)
+
+(defn- check-database-feature
+  [transform]
+  (let [database (error-or-obj (t2/select-one :model/Database (source-database-id transform))
+                               (deferred-tru "The source database cannot be found."))
+        feature (transforms.util/required-database-feature transform)]
+    (error-or-obj (not (:is_sample database))
+                  (deferred-tru "Cannot run transforms on the sample database."))
+    (error-or-obj (not (:is_audit database))
+                  (deferred-tru "Cannot run transforms on audit databases."))
+    (error-or-obj (driver.u/supports? (:engine database) feature database)
+                  (deferred-tru "The database does not support the requested transform target type."))))
+
+(mu/defn create-transform!
+  "Create a transform from input and associate it with tags.
+
+  Args:
+    input: a map describing the transform model
+
+  Returns:
+    a transform model
+
+  Raises:
+    an ex-info object if we cannot validate the transform model"
+  [input :- [:map
+             [:name :string]
+             [:description {:optional true} [:maybe :string]]
+             [:library_identifier {:optional true} :string]
+             [:source :any]
+             [:target ::t.schema/transform-target]
+             [:run_trigger {:optional true} ::t.schema/run-trigger]
+             [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
+  (check-database-feature input)
+  (error-or-obj (not (transforms.util/target-table-exists? input))
+                (deferred-tru "A table with that name already exists."))
+  (t2/with-transaction [_]
+    (let [tag-ids (:tag_ids input)
+          transform (t2/insert-returning-instance!
+                     :model/Transform (select-keys input [:name :description :source :target :run_trigger :library_identifier]))]
+      ;; Add tag associations if provided
+      (when (seq tag-ids)
+        (transform.model/update-transform-tags! (:id transform) tag-ids))
+      ;; Return with hydrated tag_ids
+      (t2/hydrate transform :transform_tag_ids))))
+
+(mu/defn update-transform!
+  "Update a transform from input
+
+  Args:
+    input: a map describing the transform model
+
+  Returns:
+    a transform model
+
+  Raises:
+    an ex-info object if we cannot validate the transform model"
+  [id :- ms/PositiveInt
+   input :- [:map
+             [:name {:optional true} :string]
+             [:description {:optional true} [:maybe :string]]
+             [:library_identifier {:optional true} :string]
+             [:source {:optional true} :any]
+             [:target {:optional true} ::t.schema/transform-target]
+             [:run_trigger {:optional true} ::t.schema/run-trigger]
+             [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
+  (t2/with-transaction [_]
+    ;; Cycle detection should occur within the transaction to avoid race
+    (let [old (t2/select-one :model/Transform id)
+          new (merge old input)
+          target-fields #(-> % :target (select-keys [:schema :name]))]
+      (check-database-feature new)
+      (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+        (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                        {:status-code 400})))
+      (error-or-obj (not (and (not= (target-fields old) (target-fields new))
+                              (transforms.util/target-table-exists? new)))
+                    (deferred-tru "A table with that name already exists.")))
+
+    (t2/update! :model/Transform id (dissoc input :tag_ids))
+    ;; Update tag associations if provided
+    (when (contains? input :tag_ids)
+      (transform.model/update-transform-tags! id (:tag_ids input)))
+    (t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids)))
 
 (p/import-vars
  [metabase-enterprise.transforms.settings

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -11,7 +11,8 @@
    [metabase.query-processor.pipeline :as qp.pipeline]
    [metabase.util.log :as log]
    [metabase.util.malli.registry :as mr]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2]
+   [metabase-enterprise.library.core :as library]))
 
 (set! *warn-on-reflection* true)
 
@@ -28,7 +29,7 @@
 
 (defn- sync-target!
   ([transform-id run-id]
-   (let [{:keys [source target]} (t2/select-one :model/Transform transform-id)
+   (let [{:keys [source target]} (library/select :model/Transform transform-id)
          db (get-in source [:query :database])
          database (t2/select-one :model/Database db)]
      (sync-target! target database run-id)))

enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj

@@ -3,7 +3,9 @@
    [clojure.set :as set]
    [medley.core :as m]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
+   [metabase.api.common :as api]
    [metabase.models.interface :as mi]
+   [metabase.models.serialization :as serdes]
    [methodical.core :as methodical]
    [toucan2.core :as t2]))
 
@@ -11,7 +13,7 @@
 
 (methodical/defmethod t2/table-name :model/Transform [_model] :transform)
 
-(doseq [trait [:metabase/model :hook/entity-id :hook/timestamped?]]
+(doseq [trait [:metabase/model :hook/entity-id :hook/timestamped? :hook/git-sync-protected]]
   (derive :model/Transform trait))
 
 (t2/deftransforms :model/Transform
@@ -41,24 +43,6 @@
       (for [transform transforms]
         (assoc transform :last_run (get last-runs (:id transform)))))))
 
-(mi/define-batched-hydration-method transform-tag-ids
-  :transform_tag_ids
-  "Add tag_ids to a transform, preserving the order defined by position"
-  [transforms]
-  (if-not (seq transforms)
-    transforms
-    (let [transform-ids         (into #{} (map :id) transforms)
-          tag-associations      (when (seq transform-ids)
-                                  (t2/select [:model/TransformTransformTag :transform_id :tag_id :position]
-                                             :transform_id [:in transform-ids]
-                                             {:order-by [[:position :asc]]}))
-          transform-id->tag-ids (reduce (fn [acc {:keys [transform_id tag_id]}]
-                                          (update acc transform_id (fnil conj []) tag_id))
-                                        {}
-                                        tag-associations)]
-      (for [transform transforms]
-        (assoc transform :tag_ids (vec (get transform-id->tag-ids (:id transform) [])))))))
-
 (defn update-transform-tags!
   "Update the tags associated with a transform using smart diff logic.
    Only modifies what has changed: deletes removed tags, updates positions for moved tags,
@@ -109,3 +93,61 @@
                         {:transform_id transform-id
                          :tag_id       tag-id
                          :position     (get new-positions tag-id)})))))))
+
+;;; ------------------------------------------------- Serialization --------------------------------------------------
+
+(defmethod serdes/hash-fields :model/Transform
+  [_transform]
+  [:name :created_at])
+
+(defn- extract-database-ids-from-target
+  "Extract database IDs from target configuration."
+  [target]
+  (when (map? target)
+    (cond
+      ;; Direct database reference
+      (:database target) #{(:database target)}
+      ;; Nested database references  
+      (:databases target) (set (:databases target))
+      ;; Other potential patterns - extend as needed
+      :else #{})))
+
+(defn- transform-target-export
+  "Transform target JSON for export, replacing database IDs with portable names."
+  [target]
+  (when target
+    (let [export-db-fk (:export (serdes/fk :model/Database :name))]
+      (cond-> target
+        (:database target) (update :database export-db-fk)
+        (:databases target) (update :databases #(mapv export-db-fk %))))))
+
+(defn- transform-target-import
+  "Transform target JSON for import, replacing portable names with database IDs."
+  [target]
+  (when target
+    (let [import-db-fk (:import (serdes/fk :model/Database :name))]
+      (cond-> target
+        (:database target) (update :database import-db-fk)
+        (:databases target) (update :databases #(mapv import-db-fk %))))))
+
+(defmethod serdes/make-spec "Transform"
+  [_model-name opts]
+  {:copy [:name :description :entity_id]
+   :skip [:synced_to_source_of_truth]
+   :transform
+   {:created_at (serdes/date)
+    :updated_at (serdes/date)
+    :source     {:export serdes/export-mbql :import serdes/import-mbql}
+    :target     {:export transform-target-export
+                 :import transform-target-import}
+    :tag_associations (serdes/nested :model/TransformTransformTag :transform_id opts)}})
+
+(defmethod serdes/dependencies "Transform"
+  [{:keys [source target]}]
+  (set (serdes/mbql-deps source)))
+
+(defmethod serdes/descendants "Transform"
+  [_model-name id]
+  ;; Include the junction table records that link this transform to its tags
+  (into {} (for [junction-id (t2/select-pks-set :model/TransformTransformTag :transform_id id)]
+             {["TransformTransformTag" junction-id] {"Transform" id}})))

enterprise/backend/src/metabase_enterprise/transforms/models/transform_job.clj

@@ -4,6 +4,7 @@
    [medley.core :as m]
    [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
    [metabase.models.interface :as mi]
+   [metabase.models.serialization :as serdes]
    [metabase.util.i18n :as i18n]
    [methodical.core :as methodical]
    [toucan2.core :as t2]))
@@ -122,3 +123,28 @@
                (t2/changes job))                     ;; user edits
         (update :name        str) ;; convert deferred to strings
         (update :description str))))
+
+;;; ------------------------------------------------- Serialization --------------------------------------------------
+
+(defmethod serdes/hash-fields :model/TransformJob
+  [_transform-job]
+  [:name :created_at])
+
+(defmethod serdes/make-spec "TransformJob"
+  [_model-name opts]
+  {:copy [:name :description :schedule :entity_id :built_in_type]
+   :skip []
+   :transform {:created_at (serdes/date)
+               :updated_at (serdes/date)
+               :tag_associations (serdes/nested :model/TransformJobTransformTag :job_id opts)}})
+
+(defmethod serdes/dependencies "TransformJob"
+  [_ingested]
+  ;; No direct dependencies - TransformTags will be loaded via the junction table dependencies
+  #{})
+
+(defmethod serdes/descendants "TransformJob"
+  [_model-name id]
+  ;; Include the junction table records that link this job to its tags
+  (into {} (for [junction-id (t2/select-pks-set :model/TransformJobTransformTag :job_id id)]
+             {["TransformJobTransformTag" junction-id] {"TransformJob" id}})))

enterprise/backend/src/metabase_enterprise/transforms/models/transform_job_transform_tag.clj

@@ -1,5 +1,6 @@
 (ns metabase-enterprise.transforms.models.transform-job-transform-tag
   (:require
+   [metabase.models.serialization :as serdes]
    [methodical.core :as methodical]
    [toucan2.core :as t2]))
 
@@ -10,3 +11,32 @@
 (doto :model/TransformJobTransformTag
   (derive :metabase/model)
   (derive :hook/entity-id))
+
+;;; ------------------------------------------------- Serialization --------------------------------------------------
+
+(defmethod serdes/hash-fields :model/TransformJobTransformTag
+  [_junction]
+  [:job_id :tag_id :position])
+
+(defmethod serdes/generate-path "TransformJobTransformTag"
+  [_ _junction]
+  ;; Junction tables are inlined into their parent TransformJob, so no independent path
+  nil)
+
+(defmethod serdes/make-spec "TransformJobTransformTag"
+  [_model-name _opts]
+  {:copy [:entity_id :position]
+   :skip []
+   :transform {:job_id (serdes/parent-ref)
+               :tag_id (serdes/fk :model/TransformTag)}})
+
+(defmethod serdes/dependencies "TransformJobTransformTag"
+  [{:keys [tag_id]}]
+  ;; Depends on the TransformTag being loaded first
+  (when tag_id
+    #{[{:model "TransformTag" :id tag_id}]}))
+
+(defmethod serdes/descendants "TransformJobTransformTag"
+  [_model-name _id]
+  ;; Junction tables don't contain other entities
+  {})

enterprise/backend/src/metabase_enterprise/transforms/models/transform_tag.clj

@@ -1,5 +1,6 @@
 (ns metabase-enterprise.transforms.models.transform-tag
   (:require
+   [metabase.models.serialization :as serdes]
    [metabase.util.i18n :as i18n]
    [methodical.core :as methodical]
    [toucan2.core :as t2]))
@@ -41,3 +42,23 @@
                {:built_in_type nil} ;; never translate this again
                (t2/changes tag))    ;; include user edits
         (update :name str)))) ;; convert deferred to string
+;;; ------------------------------------------------- Serialization --------------------------------------------------
+
+(defmethod serdes/hash-fields :model/TransformTag
+  [_transform-tag]
+  [:name :created_at])
+
+(defmethod serdes/make-spec "TransformTag"
+  [_model-name _opts]
+  {:copy [:name :entity_id :built_in_type]
+   :skip []
+   :transform {:created_at (serdes/date)
+               :updated_at (serdes/date)}})
+
+(defmethod serdes/dependencies "TransformTag"
+  [_ingested]
+  #{})
+
+(defmethod serdes/descendants "TransformTag"
+  [_model-name _id]
+  {})

enterprise/backend/src/metabase_enterprise/transforms/models/transform_transform_tag.clj

@@ -1,5 +1,6 @@
 (ns metabase-enterprise.transforms.models.transform-transform-tag
   (:require
+   [metabase.models.serialization :as serdes]
    [methodical.core :as methodical]
    [toucan2.core :as t2]))
 
@@ -10,3 +11,32 @@
 (doto :model/TransformTransformTag
   (derive :metabase/model)
   (derive :hook/entity-id))
+
+;;; ------------------------------------------------- Serialization --------------------------------------------------
+
+(defmethod serdes/hash-fields :model/TransformTransformTag
+  [_junction]
+  [:transform_id :tag_id :position])
+
+(defmethod serdes/generate-path "TransformTransformTag"
+  [_ _junction]
+  ;; Junction tables are inlined into their parent Transform, so no independent path
+  nil)
+
+(defmethod serdes/make-spec "TransformTransformTag"
+  [_model-name _opts]
+  {:copy [:entity_id :position]
+   :skip []
+   :transform {:transform_id (serdes/parent-ref)
+               :tag_id (serdes/fk :model/TransformTag)}})
+
+(defmethod serdes/dependencies "TransformTransformTag"
+  [{:keys [tag_id]}]
+  ;; Depends on the TransformTag being loaded first
+  (when tag_id
+    #{[{:model "TransformTag" :id tag_id}]}))
+
+(defmethod serdes/descendants "TransformTransformTag"
+  [_model-name _id]
+  ;; Junction tables don't contain other entities
+  {})

enterprise/backend/src/metabase_enterprise/transforms/schema.clj

@@ -0,0 +1,18 @@
+(ns metabase-enterprise.transforms.schema
+  (:require
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]))
+
+(mr/def ::transform-source
+  [:map
+   [:type [:= "query"]]
+   [:query [:map [:database :int]]]])
+
+(mr/def ::transform-target
+  [:map
+   [:type [:enum "table"]]
+   [:schema {:optional true} [:or ms/NonBlankString :nil]]
+   [:name :string]])
+
+(mr/def ::run-trigger
+  [:enum "none" "global-schedule"])

enterprise/backend/test/metabase_enterprise/library/core_test.clj

@@ -0,0 +1,160 @@
+(ns metabase-enterprise.library.core-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.git-source-of-truth.settings :as git-source-of-truth.settings]
+   [metabase-enterprise.library.core :as library]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase.test :as mt]))
+
+(def transform-yaml-1-for-index
+  "entity: model/Transform:v1
+name: First Batch Transform
+identifier: batch-transform-1
+description: First transform for batch testing
+tags:
+  - test
+  - batch
+database: test-data (postgres)
+target:
+  type: table
+  name: batch_output_1
+source: |
+  SELECT * FROM table1 WHERE active = true")
+
+(def transform-yaml-2-for-index
+  "entity: model/Transform:v1
+name: Second Batch Transform
+identifier: batch-transform-2
+description: Second transform for batch testing
+tags:
+  - test
+database: test-data (postgres)
+target:
+  type: table
+  name: batch_output_2
+source: |
+  SELECT * FROM table2 WHERE status = 'complete'")
+
+(deftest index-library-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (mt/dataset transforms-dataset/transforms-test
+      (mt/with-model-cleanup [:model/Transform]
+        (testing "Successfully load multiple MBML files"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-batch :id} {:name "batch"}]
+            (mt/with-temp-file [temp-file-1 "transform1.yaml"
+                                temp-file-2 "transform2.yaml"]
+              (spit temp-file-1 transform-yaml-1-for-index)
+              (spit temp-file-2 transform-yaml-2-for-index)
+              (let [result-models (atom {})
+                    result-index (atom {})]
+                (mt/with-temporary-setting-values [git-source-of-truth.settings/git-sync-import-branch "test-branch"]
+                  (with-redefs [library/branch+filename->model result-models
+                                library/branch+entity+index->filename result-index]
+                    (library/index-library! [temp-file-1 temp-file-2])
+                    (is (=? {"test-branch" {temp-file-1 {:name "First Batch Transform"
+                                                         :identifier "batch-transform-1"}
+                                            temp-file-2 {:name "Second Batch Transform"
+                                                         :identifier "batch-transform-2"}}}
+                            @result-models))
+                    (is (= {"test-branch" {:model/Transform {:tag {"test" #{temp-file-2 temp-file-1}
+                                                                   "batch" #{temp-file-1}}
+                                                             :tag_id {tag-id-test #{temp-file-2 temp-file-1}
+                                                                      tag-id-batch #{temp-file-1}}
+                                                             :identifier {"batch-transform-1" #{temp-file-1}
+                                                                          "batch-transform-2" #{temp-file-2}}}}}
+                           @result-index)))))))))
+
+      (testing "Empty file list returns empty result"
+        (let [result-models (atom {})
+              result-index (atom {})]
+          (mt/with-temporary-setting-values [git-source-of-truth.settings/git-sync-import-branch "test-branch"]
+            (with-redefs [library/branch+filename->model result-models
+                          library/branch+entity+index->filename result-index]
+              (library/index-library! [])
+              (is (= {"test-branch" {}} @result-models))
+              (is (= {"test-branch" {}} @result-index)))))))))
+
+(deftest select-test
+  (testing "select retrieves a model by entity and identifier"
+    (mt/with-temporary-setting-values [git-source-of-truth.settings/git-sync-import-branch "test-branch"]
+      (let [test-model {:entity "model/Transform:v1"
+                        :identifier "test-transform"
+                        :name "Test Transform"}]
+        (with-redefs [library/branch+filename->model (atom {"test-branch" {"test-file.yaml" test-model}})
+                      library/branch+entity+index->filename (atom {"test-branch" {:model/Transform {:identifier {"test-transform" #{"test-file.yaml"}}}}})]
+          (is (= test-model (library/select :model/Transform "test-transform")))))))
+
+  (testing "select returns nil when model doesn't exist"
+    (mt/with-temporary-setting-values [git-source-of-truth.settings/git-sync-import-branch "test-branch"]
+      (with-redefs [library/branch+filename->model (atom {"test-branch" {}})
+                    library/branch+entity+index->filename (atom {"test-branch" {}})]
+        (is (nil? (library/select :model/Transform "non-existent"))))))
+
+  (testing "select with wrong entity type returns nil"
+    (mt/with-temporary-setting-values [git-source-of-truth.settings/git-sync-import-branch "test-branch"]
+      (let [test-model {:entity "model/Transform:v1"
+                        :identifier "test-transform"
+                        :name "Test Transform"}]
+        (with-redefs [library/branch+filename->model (atom {"test-branch" {"test-file.yaml" test-model}})
+                      library/branch+entity+index->filename (atom {"test-branch" {:model/Transform {:identifier {"test-transform" #{"test-file.yaml"}}}}})]
+          (is (nil? (library/select :model/SomeOtherType "test-transform")))))))
+
+  (testing "select uses dynamic branch binding when set"
+    (let [test-model {:entity "model/Transform:v1"
+                      :identifier "test-transform"
+                      :name "Test Transform"}]
+      (with-redefs [library/branch+filename->model (atom {"custom-branch" {"test-file.yaml" test-model}})
+                    library/branch+entity+index->filename (atom {"custom-branch" {:model/Transform {:identifier {"test-transform" #{"test-file.yaml"}}}}})]
+        (binding [library/*current-branch* "custom-branch"]
+          (is (= test-model (library/select :model/Transform "test-transform"))))))))
+
+(deftest find-by-test
+  (testing "find-by retrieves models by index"
+    (mt/with-temporary-setting-values [git-source-of-truth.settings/git-sync-import-branch "test-branch"]
+      (let [model1 {:entity "model/Transform:v1"
+                    :identifier "transform-1"
+                    :tags ["test" "batch"]}
+            model2 {:entity "model/Transform:v1"
+                    :identifier "transform-2"
+                    :tags ["test"]}]
+        (with-redefs [library/branch+filename->model (atom {"test-branch" {"file1.yaml" model1
+                                                                           "file2.yaml" model2}})
+                      library/branch+entity+index->filename (atom {"test-branch" {:model/Transform {:tag {"test" #{"file1.yaml" "file2.yaml"}
+                                                                                                          "batch" #{"file1.yaml"}}}}})]
+          (is (= #{model1 model2} (set (library/find-by :model/Transform :tag ["test"]))))
+          (is (= [model1] (library/find-by :model/Transform :tag ["batch"])))))))
+
+  (testing "find-by with multiple values"
+    (mt/with-temporary-setting-values [git-source-of-truth.settings/git-sync-import-branch "test-branch"]
+      (let [model1 {:entity "model/Transform:v1"
+                    :identifier "transform-1"
+                    :tags ["test"]}
+            model2 {:entity "model/Transform:v1"
+                    :identifier "transform-2"
+                    :tags ["batch"]}
+            model3 {:entity "model/Transform:v1"
+                    :identifier "transform-3"
+                    :tags ["prod"]}]
+        (with-redefs [library/branch+filename->model (atom {"test-branch" {"file1.yaml" model1
+                                                                           "file2.yaml" model2
+                                                                           "file3.yaml" model3}})
+                      library/branch+entity+index->filename (atom {"test-branch" {:model/Transform {:tag {"test" #{"file1.yaml"}
+                                                                                                          "batch" #{"file2.yaml"}
+                                                                                                          "prod" #{"file3.yaml"}}}}})]
+          (is (= #{model1 model2} (set (library/find-by :model/Transform :tag ["test" "batch"]))))))))
+
+  (testing "find-by returns empty sequence when no matches"
+    (mt/with-temporary-setting-values [git-source-of-truth.settings/git-sync-import-branch "test-branch"]
+      (with-redefs [library/branch+filename->model (atom {"test-branch" {}})
+                    library/branch+entity+index->filename (atom {"test-branch" {:model/Transform {:tag {}}}})]
+        (is (empty? (library/find-by :model/Transform :tag ["non-existent"]))))))
+
+  (testing "find-by uses dynamic branch binding when set"
+    (let [test-model {:entity "model/Transform:v1"
+                      :identifier "test-transform"
+                      :tags ["test"]}]
+      (with-redefs [library/branch+filename->model (atom {"custom-branch" {"test-file.yaml" test-model}})
+                    library/branch+entity+index->filename (atom {"custom-branch" {:model/Transform {:tag {"test" #{"test-file.yaml"}}}}})]
+        (binding [library/*current-branch* "custom-branch"]
+          (is (= [test-model] (library/find-by :model/Transform :tag ["test"]))))))))

enterprise/backend/test/metabase_enterprise/mbml/core_test.clj

@@ -0,0 +1,606 @@
+(ns metabase-enterprise.mbml.core-test
+  "Comprehensive unit tests for the main MBML API functions.
+
+  Tests cover the two main entry points:
+  - parse-mbml-file: Parse MBML from files on disk
+  - parse-mbml-string: Parse MBML from string content
+  - mbml-file->model: Create Metabase models from MBML files
+
+  All test data is inline to ensure self-contained, reproducible tests."
+  (:require
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [metabase-enterprise.mbml.core :as mbml.core]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+
+;;; ---------------------------------------- Test Data ------------------------------------------------
+
+(def valid-yaml-content
+  "entity: model/Transform:v1
+name: Customer Analysis Transform
+identifier: customer-analysis
+description: Transform for analyzing customer data patterns
+tags:
+  - analytics
+  - customer
+database: analytics-db
+target:
+  type: table
+  name: customer_analysis_results
+source: |
+  SELECT customer_id, total_orders, avg_order_value
+  FROM customer_metrics
+  WHERE last_order_date >= '2024-01-01'")
+
+(def valid-sql-with-frontmatter
+  "-- METABASE_BEGIN
+-- entity: model/Transform:v1
+-- name: Sales Report Transform
+-- identifier: sales-report
+-- description: Generate sales reports for management
+-- tags:
+--   - reporting
+--   - sales
+-- database: sales-db
+-- target:
+--   type: table
+--   name: monthly_sales_report
+-- METABASE_END
+
+SELECT
+  DATE_TRUNC('month', order_date) as month,
+  SUM(order_total) as total_sales,
+  COUNT(*) as total_orders
+FROM orders
+WHERE order_date >= CURRENT_DATE - INTERVAL '12 months'
+GROUP BY DATE_TRUNC('month', order_date)
+ORDER BY month;")
+
+(def valid-python-with-frontmatter
+  "# METABASE_BEGIN
+# entity: model/Transform:v1
+# name: Data Processing Transform
+# identifier: data-processing
+# description: Process raw data for analytics
+# tags:
+#   - etl
+#   - processing
+# database: warehouse-db
+# target:
+#   type: table
+#   name: processed_data
+# METABASE_END
+
+import pandas as pd
+import numpy as np
+
+def process_customer_data(df):
+    df['customer_lifetime_value'] = df['total_orders'] * df['avg_order_value']
+    return df.fillna(0)
+
+if __name__ == '__main__':
+    data = pd.read_csv('customer_data.csv')
+    result = process_customer_data(data)
+    result.to_csv('processed_data.csv', index=False)")
+
+(def invalid-yaml-missing-required-fields
+  "entity: model/Transform:v1
+name: Incomplete Transform
+description: Missing required fields")
+
+(def invalid-yaml-wrong-entity-type
+  "entity: model/InvalidType:v1
+name: Wrong Entity Type
+identifier: wrong-type
+database: test-db
+target: test_table")
+
+(def malformed-yaml-content
+  "entity: model/Transform:v1
+name: Malformed YAML
+identifier: malformed
+  invalid_indentation: true
+database: test-db
+target: test_table")
+
+(def sql-without-frontmatter
+  "SELECT * FROM customers WHERE active = true;")
+
+(def sql-with-incomplete-frontmatter
+  "-- METABASE_BEGIN
+-- entity: model/Transform:v1
+-- name: Incomplete Front-matter
+-- No METABASE_END marker
+
+SELECT * FROM incomplete;")
+
+(def empty-content "")
+
+(def whitespace-only-content "   \n  \t  \n   ")
+
+;;; ---------------------------------------- parse-mbml-file Tests ------------------------------------
+
+(deftest ^:parallel parse-mbml-file-yaml-test
+  (testing "Parse valid YAML file"
+    (mt/with-temp-file [temp-file "transform.yaml"]
+      (spit temp-file valid-yaml-content)
+      (let [result (mbml.core/parse-mbml-file temp-file)]
+        (is (=? {:entity "model/Transform:v1"
+                 :name "Customer Analysis Transform"
+                 :identifier "customer-analysis"
+                 :database "analytics-db"
+                 :target {:name "customer_analysis_results"}
+                 :tags ["analytics" "customer"]
+                 :source string?}
+                result)))))
+
+  (testing "Parse valid SQL file with front-matter"
+    (mt/with-temp-file [temp-file "transform.sql"]
+      (spit temp-file valid-sql-with-frontmatter)
+      (let [result (mbml.core/parse-mbml-file temp-file)]
+        (is (=? {:entity "model/Transform:v1"
+                 :name "Sales Report Transform"
+                 :identifier "sales-report"
+                 :database "sales-db"
+                 :target {:name "monthly_sales_report"}
+                 :body #(and (string? %)
+                             (str/includes? % "SELECT")
+                             (str/includes? % "DATE_TRUNC"))}
+                result)))))
+
+  (testing "Parse valid Python file with front-matter"
+    (mt/with-temp-file [temp-file "transform.py"]
+      (spit temp-file valid-python-with-frontmatter)
+      (let [result (mbml.core/parse-mbml-file temp-file)]
+        (is (=? {:entity "model/Transform:v1"
+                 :name "Data Processing Transform"
+                 :identifier "data-processing"
+                 :database "warehouse-db"
+                 :target {:name "processed_data"}
+                 :body #(and (string? %)
+                             (str/includes? % "import pandas")
+                             (str/includes? % "def process_customer_data"))}
+                result))))))
+
+(deftest ^:parallel parse-mbml-file-error-handling-test
+  (testing "File not found error"
+    (is (thrown-with-msg?
+         clojure.lang.ExceptionInfo
+         #"File not found"
+         (mbml.core/parse-mbml-file "/nonexistent/path/file.yaml"))))
+
+  (testing "Empty file error"
+    (mt/with-temp-file [temp-file "empty.yaml"]
+      (spit temp-file empty-content)
+      (is (thrown-with-msg?
+           clojure.lang.ExceptionInfo
+           #"File is empty"
+           (mbml.core/parse-mbml-file temp-file)))))
+
+  (testing "Whitespace-only file error"
+    (mt/with-temp-file [temp-file "whitespace.yaml"]
+      (spit temp-file whitespace-only-content)
+      (is (thrown-with-msg?
+           clojure.lang.ExceptionInfo
+           #"File is empty"
+           (mbml.core/parse-mbml-file temp-file)))))
+
+  (testing "Schema validation error"
+    (mt/with-temp-file [temp-file "invalid.yaml"]
+      (spit temp-file invalid-yaml-missing-required-fields)
+      (is (thrown-with-msg?
+           clojure.lang.ExceptionInfo
+           #"MBML validation failed"
+           (mbml.core/parse-mbml-file temp-file)))))
+
+  (testing "Malformed YAML error"
+    (mt/with-temp-file [temp-file "malformed.yaml"]
+      (spit temp-file malformed-yaml-content)
+      (is (thrown-with-msg?
+           clojure.lang.ExceptionInfo
+           #"YAML|parse"
+           (mbml.core/parse-mbml-file temp-file)))))
+
+  (testing "SQL file without front-matter metadata"
+    (mt/with-temp-file [temp-file "no-metadata.sql"]
+      (spit temp-file sql-without-frontmatter)
+      (is (thrown-with-msg?
+           clojure.lang.ExceptionInfo
+           #"Front-matter extraction failed"
+           (mbml.core/parse-mbml-file temp-file)))))
+
+  (testing "SQL file with incomplete front-matter"
+    (mt/with-temp-file [temp-file "incomplete.sql"]
+      (spit temp-file sql-with-incomplete-frontmatter)
+      (is (thrown-with-msg?
+           clojure.lang.ExceptionInfo
+           #"Front-matter extraction failed"
+           (mbml.core/parse-mbml-file temp-file))))))
+
+;;; ---------------------------------------- parse-mbml-string Tests ----------------------------------
+
+(deftest ^:parallel parse-mbml-string-yaml-test
+  (testing "Parse valid YAML content with explicit file type"
+    (let [result (mbml.core/parse-mbml-string valid-yaml-content :yaml)]
+      (is (=? {:entity "model/Transform:v1"
+               :name "Customer Analysis Transform"
+               :identifier "customer-analysis"
+               :source string?}
+              result))))
+
+  (testing "Parse valid YAML content without file type hint (defaults to :yaml)"
+    (let [result (mbml.core/parse-mbml-string valid-yaml-content nil)]
+      (is (=? {:entity "model/Transform:v1"
+               :name "Customer Analysis Transform"}
+              result)))))
+
+(deftest ^:parallel parse-mbml-string-sql-test
+  (testing "Parse valid SQL content with front-matter"
+    (let [result (mbml.core/parse-mbml-string valid-sql-with-frontmatter :sql)]
+      (is (=? {:entity "model/Transform:v1"
+               :name "Sales Report Transform"
+               :identifier "sales-report"
+               :body #(and (string? %)
+                           (str/includes? % "SELECT")
+                           (not (str/includes? % "METABASE_BEGIN")))}
+              result))))
+
+  (testing "Parse SQL content without front-matter"
+    (is (thrown-with-msg?
+         clojure.lang.ExceptionInfo
+         #"Front-matter extraction failed"
+         (mbml.core/parse-mbml-string sql-without-frontmatter :sql)))))
+
+(deftest ^:parallel parse-mbml-string-python-test
+  (testing "Parse valid Python content with front-matter"
+    (let [result (mbml.core/parse-mbml-string valid-python-with-frontmatter :python)]
+      (is (=? {:entity "model/Transform:v1"
+               :name "Data Processing Transform"
+               :identifier "data-processing"
+               :body #(and (string? %)
+                           (str/includes? % "import pandas")
+                           (not (str/includes? % "METABASE_BEGIN")))}
+              result)))))
+
+(deftest ^:parallel parse-mbml-string-validation-test
+  (testing "Schema validation with missing required fields"
+    (is (thrown-with-msg?
+         clojure.lang.ExceptionInfo
+         #"MBML validation failed"
+         (mbml.core/parse-mbml-string invalid-yaml-missing-required-fields :yaml))))
+
+  (testing "Schema validation with invalid entity type"
+    (is (thrown-with-msg?
+         clojure.lang.ExceptionInfo
+         #"MBML validation failed"
+         (mbml.core/parse-mbml-string invalid-yaml-wrong-entity-type :yaml))))
+
+  (testing "YAML parsing error with malformed content"
+    (is (thrown-with-msg?
+         clojure.lang.ExceptionInfo
+         #"YAML|parse"
+         (mbml.core/parse-mbml-string malformed-yaml-content :yaml)))))
+
+(deftest ^:parallel parse-mbml-string-edge-cases-test
+  (testing "Empty content"
+    (is (thrown-with-msg?
+         clojure.lang.ExceptionInfo
+         #"Invalid input|non-blank string"
+         (mbml.core/parse-mbml-string empty-content :yaml))))
+
+  (testing "Whitespace-only content"
+    (is (thrown-with-msg?
+         clojure.lang.ExceptionInfo
+         #"Invalid input|non-blank string"
+         (mbml.core/parse-mbml-string whitespace-only-content :yaml))))
+
+  (testing "Valid content with different file type hints"
+    ;; SQL content parsed as YAML should fail
+    (is (thrown-with-msg?
+         clojure.lang.ExceptionInfo
+         #"YAML|parse|metadata"
+         (mbml.core/parse-mbml-string valid-sql-with-frontmatter :yaml)))
+
+    ;; YAML content parsed as SQL should fail (no front-matter)
+    (is (thrown-with-msg?
+         clojure.lang.ExceptionInfo
+         #"Front-matter extraction failed"
+         (mbml.core/parse-mbml-string valid-yaml-content :sql)))))
+
+;;; ---------------------------------------- API Function Edge Cases ----------------------------------
+
+(deftest ^:parallel api-function-parameter-validation-test
+  (testing "parse-mbml-file with invalid parameters"
+    ;; This should be caught by mu/defn parameter validation
+    (is (thrown?
+         clojure.lang.ExceptionInfo
+         (mbml.core/parse-mbml-file nil)))
+
+    (is (thrown?
+         clojure.lang.ExceptionInfo
+         (mbml.core/parse-mbml-file ""))))
+
+  (testing "parse-mbml-string with invalid parameters"
+    ;; This should be caught by mu/defn parameter validation
+    (is (thrown?
+         clojure.lang.ExceptionInfo
+         (mbml.core/parse-mbml-string nil :yaml)))
+
+    (is (thrown?
+         clojure.lang.ExceptionInfo
+         (mbml.core/parse-mbml-string "" :yaml)))))
+
+(deftest ^:parallel file-type-detection-integration-test
+  (testing "File type detection integration with different extensions"
+    (mt/with-temp-file [yaml-file "test.yml"]
+      (spit yaml-file valid-yaml-content)
+      (let [result (mbml.core/parse-mbml-file yaml-file)]
+        (is (=? {:name "Customer Analysis Transform"} result))))
+
+    (mt/with-temp-file [yaml-file "test.YAML"]
+      (spit yaml-file valid-yaml-content)
+      (let [result (mbml.core/parse-mbml-file yaml-file)]
+        (is (=? {:name "Customer Analysis Transform"} result))))
+
+    (mt/with-temp-file [sql-file "test.SQL"]
+      (spit sql-file valid-sql-with-frontmatter)
+      (let [result (mbml.core/parse-mbml-file sql-file)]
+        (is (=? {:name "Sales Report Transform"} result))))
+
+    (mt/with-temp-file [py-file "test.PY"]
+      (spit py-file valid-python-with-frontmatter)
+      (let [result (mbml.core/parse-mbml-file py-file)]
+        (is (=? {:name "Data Processing Transform"} result))))))
+
+(deftest ^:parallel error-context-preservation-test
+  (testing "Error context includes relevant debugging information"
+    (try
+      (mbml.core/parse-mbml-file "/nonexistent/file.yaml")
+      (catch clojure.lang.ExceptionInfo e
+        (let [data (ex-data e)]
+          (is (=? {:type :file-not-found
+                   :file "/nonexistent/file.yaml"}
+                  data)))))
+
+    (mt/with-temp-file [temp-file "malformed.yaml"]
+      (spit temp-file malformed-yaml-content)
+      (try
+        (mbml.core/parse-mbml-file temp-file)
+        (catch clojure.lang.ExceptionInfo e
+          (let [data (ex-data e)]
+            (is (=? {:type some?} data))
+            (is (or (str/includes? (str data) "YAML")
+                    (str/includes? (str data) "parse")))))))))
+
+(deftest ^:parallel comprehensive-integration-test
+  (testing "Complete parse workflow with all supported formats"
+    ;; Test the complete workflow from file creation to validated result
+    (let [test-cases [{:extension "yaml" :content valid-yaml-content :expected-name "Customer Analysis Transform"}
+                      {:extension "sql" :content valid-sql-with-frontmatter :expected-name "Sales Report Transform"}
+                      {:extension "py" :content valid-python-with-frontmatter :expected-name "Data Processing Transform"}]]
+
+      (doseq [{:keys [extension content expected-name]} test-cases]
+        (mt/with-temp-file [temp-file (str "test." extension)]
+          (spit temp-file content)
+          (let [result (mbml.core/parse-mbml-file temp-file)]
+            (is (=? {:entity "model/Transform:v1"
+                     :name expected-name
+                     :identifier string?
+                     :database string?
+                     :target map?}
+                    result))
+            (when (not= extension "yaml")
+              (is (contains? result :body)
+                  (str "Source code should be present for " extension " files")))))))))
+
+;;; ---------------------------------------- mbml-file->model Tests -----------------------------------
+
+(def transform-yaml-no-db-for-model-test
+  "entity: model/Transform:v1
+name: Test Transform for Model Creation
+identifier: test-transform-model
+description: Transform used for testing model creation
+tags:
+  - test
+  - hourly
+database: test-data
+target:
+  type: table
+  name: test_transform_output
+source: |
+  SELECT id, name, created_at
+  FROM test_table
+  WHERE active = true")
+
+(def transform-yaml-for-model-test
+  "entity: model/Transform:v1
+name: Test Transform for Model Creation
+identifier: test-transform-model
+description: Transform used for testing model creation
+tags:
+  - test
+  - hourly
+database: test-data (postgres)
+target:
+  type: table
+  name: test_transform_output
+source: |
+  SELECT id, name, created_at
+  FROM test_table
+  WHERE active = true")
+
+(def transform-sql-for-model-test
+  "-- METABASE_BEGIN
+-- entity: model/Transform:v1
+-- name: SQL Transform for Model Creation
+-- identifier: sql-transform-model
+-- description: SQL transform used for testing
+-- tags:
+--   - test
+--   - daily
+-- database: test-data (postgres)
+-- target:
+--   type: table
+--   name: sql_transform_output
+-- METABASE_END
+
+SELECT
+  customer_id,
+  COUNT(*) as order_count,
+  SUM(total) as total_value
+FROM orders
+WHERE status = 'completed'
+GROUP BY customer_id;")
+
+(deftest mbml-file->model-transform-creation-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (mt/dataset transforms-dataset/transforms-test
+      (testing "Create Transform model from YAML file"
+        (mt/with-model-cleanup [:model/Transform]
+          (mt/with-temp [:model/TransformTag _ {:name "test"}]
+            (mt/with-temp-file [temp-file "transform.yaml"]
+              (spit temp-file transform-yaml-for-model-test)
+              (let [result (mbml.core/mbml-file->model temp-file)]
+                (is (=? {:id int?
+                         :name "Test Transform for Model Creation"
+                         :description "Transform used for testing model creation"
+                         :source map?
+                         :target map?
+                         :entity_id string?
+                         :created_at some?
+                         :updated_at some?}
+                        result))
+
+                ;; Verify the transform was actually created in the database
+                (let [saved-transform (t2/select-one :model/Transform :id (:id result))]
+                  (is (=? {:name "Test Transform for Model Creation"
+                           :description "Transform used for testing model creation"}
+                          saved-transform))))))))
+
+      (testing "Create Transform model from SQL file"
+        (mt/with-model-cleanup [:model/Transform]
+          (mt/with-temp [:model/TransformTag _ {:name "test"}]
+            (mt/with-temp-file [temp-file "transform.sql"]
+              (spit temp-file transform-sql-for-model-test)
+              (let [result (mbml.core/mbml-file->model temp-file)]
+                (is (=? {:id int?
+                         :name "SQL Transform for Model Creation"
+                         :description "SQL transform used for testing"
+                         :source map?
+                         :target map?}
+                        result))
+
+                ;; Verify the source came from the SQL body, not the original source field
+                (let [saved-transform (t2/select-one :model/Transform :id (:id result))]
+                  (is (=? {:name "SQL Transform for Model Creation"}
+                          saved-transform))))))))
+
+      (testing "Test update functionality"
+        (mt/with-model-cleanup [:model/Transform]
+          (mt/with-temp [:model/TransformTag {tag-id-1 :id} {:name "test"}
+                         :model/Transform {transform-id :id} {:name "Existing Transform"
+                                                              :library_identifier "test-transform-model"
+                                                              :description "Old description"
+                                                              :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT 1\"}}"
+                                                              :target "{\"type\":\"table\",\"name\":\"old_table\"}"}
+                         :model/TransformTransformTag _ {:tag_id tag-id-1 :transform_id transform-id :position 0}]
+            (mt/with-temp-file [temp-file "transform.yaml"]
+              (spit temp-file transform-yaml-for-model-test)
+              (let [result (mbml.core/mbml-file->model temp-file)]
+                (is (=? {:id transform-id
+                         :name "Test Transform for Model Creation"
+                         :description "Transform used for testing model creation"}
+                        result))
+
+                ;; Verify the transform was actually updated in the database
+                (let [updated-transform (t2/select-one :model/Transform :id transform-id)]
+                  (is (=? {:name "Test Transform for Model Creation"
+                           :description "Transform used for testing model creation"}
+                          updated-transform)))))))))))
+
+(deftest ^:parallel mbml-file->model-error-handling-test
+  (testing "Error when database doesn't exist"
+    (mt/with-temp [:model/TransformTag _ {:name "test"}]
+      (mt/with-temp-file [temp-file "transform.yaml"]
+        (spit temp-file transform-yaml-no-db-for-model-test)
+        (is (thrown-with-msg?
+             clojure.lang.ExceptionInfo
+             #"The database does not exist.*test-data"
+             (mbml.core/mbml-file->model temp-file))))))
+
+  (testing "Error when tags don't exist"
+    (mt/with-temp-file [temp-file "transform.yaml"]
+      (spit temp-file transform-yaml-for-model-test)
+      (is (thrown-with-msg?
+           clojure.lang.ExceptionInfo
+           #"Supplied tags do not exist.*test"
+           (mbml.core/mbml-file->model temp-file)))))
+
+  (testing "Error context includes file information"
+    (mt/with-temp-file [temp-file "bad-transform.yaml"]
+      (spit temp-file transform-yaml-for-model-test)
+      (try
+        (mbml.core/mbml-file->model temp-file)
+        (catch clojure.lang.ExceptionInfo e
+          (let [data (ex-data e)]
+            (is (=? {:type :model-transformation-error
+                     :file string?}
+                    data))
+            (is (str/includes? (:file data) "bad-transform.yaml"))))))))
+
+(def transform-yaml-1-for-batch-test
+  "entity: model/Transform:v1
+name: First Batch Transform
+identifier: batch-transform-1
+description: First transform for batch testing
+tags:
+  - test
+  - batch
+database: test-data (postgres)
+target:
+  type: table
+  name: batch_output_1
+source: |
+  SELECT * FROM table1 WHERE active = true")
+
+(def transform-yaml-2-for-batch-test
+  "entity: model/Transform:v1
+name: Second Batch Transform
+identifier: batch-transform-2
+description: Second transform for batch testing
+tags:
+  - test
+  - batch
+database: test-data (postgres)
+target:
+  type: table
+  name: batch_output_2
+source: |
+  SELECT * FROM table2 WHERE status = 'complete'")
+
+(deftest mbml-files->models-successful-batch-loading-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (mt/dataset transforms-dataset/transforms-test
+      (mt/with-model-cleanup [:model/Transform]
+        (testing "Successfully load multiple MBML files"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-batch :id} {:name "batch"}]
+            (mt/with-temp-file [temp-file-1 "transform1.yaml"
+                                temp-file-2 "transform2.yaml"]
+              (spit temp-file-1 transform-yaml-1-for-batch-test)
+              (spit temp-file-2 transform-yaml-2-for-batch-test)
+
+              (let [results (mbml.core/mbml-files->models [temp-file-1 temp-file-2])]
+                (is (= 2 (count results)))
+                (is (=? [{:name "First Batch Transform"
+                          :library_identifier "batch-transform-1"}
+                         {:name "Second Batch Transform"
+                          :library_identifier "batch-transform-2"}]
+                        results)))))))
+
+      (testing "Empty file list returns empty result"
+        (let [results (mbml.core/mbml-files->models [])]
+          (is (= [] results)))))))

enterprise/backend/test/metabase_enterprise/mbml/errors_test.clj

@@ -0,0 +1,329 @@
+(ns metabase-enterprise.mbml.errors-test
+  "Comprehensive unit tests for MBML error handling utilities.
+
+  Tests all error formatting functions with inline test data, edge cases,
+  and validation of internationalized error messages."
+  (:require
+   [clojure.string :as str]
+   [clojure.test :refer [deftest ^:parallel is testing]]
+   [metabase-enterprise.mbml.errors :as mbml.errors]
+   [metabase.util.i18n :as i18n]))
+
+;;; ------------------------------------------ Error Context Tests ------------------------------------------
+
+(deftest ^:parallel error-context-test
+  (testing "error-context creates structured error maps"
+    (let [error (mbml.errors/error-context
+                 :yaml-parse-error
+                 "Test error message"
+                 {:file "test.yaml"
+                  :line 42
+                  :data {:key "value"}
+                  :context "Additional context"
+                  :suggestions ["Fix this" "Try that"]})]
+
+      (is (instance? clojure.lang.ExceptionInfo error))
+      (is (= "Test error message" (ex-message error)))
+      (is (=? {:type :yaml-parse-error
+               :message "Test error message"
+               :file "test.yaml"
+               :line 42
+               :data {:key "value"}
+               :context "Additional context"
+               :suggestions ["Fix this" "Try that"]
+               :timestamp some?}
+              (ex-data error)))))
+
+  (testing "error-context handles minimal inputs"
+    (let [error (mbml.errors/error-context :file-not-found "File missing")]
+      (is (instance? clojure.lang.ExceptionInfo error))
+      (is (= "File missing" (ex-message error)))
+      (is (=? {:type :file-not-found
+               :message "File missing"
+               :timestamp some?}
+              (ex-data error)))))
+
+  (testing "error-context handles cause exceptions"
+    (let [root-cause (Exception. "Root problem")
+          error (mbml.errors/error-context
+                 :io-error
+                 "IO failed"
+                 {:cause root-cause
+                  :file "data.yaml"})]
+
+      (is (instance? clojure.lang.ExceptionInfo error))
+      (is (= root-cause (ex-cause error)))
+      (is (=? {:type :io-error
+               :file "data.yaml"
+               :message "IO failed"
+               :timestamp some?}
+              (ex-data error))))))
+
+;;; ---------------------------------------- File Context Tests ------------------------------------------
+
+(deftest ^:parallel format-file-context-test
+  (testing "format-file-context with line numbers"
+    (is (= "in file test.yaml at line 42"
+           (mbml.errors/format-file-context "test.yaml" 42)))
+    (is (= "in file /path/to/data.sql at line 1"
+           (mbml.errors/format-file-context "/path/to/data.sql" 1))))
+
+  (testing "format-file-context without line numbers"
+    (is (= "in file test.yaml"
+           (mbml.errors/format-file-context "test.yaml")))
+    (is (= "in file /absolute/path/script.py"
+           (mbml.errors/format-file-context "/absolute/path/script.py"))))
+
+  (testing "format-file-context edge cases"
+    (is (= "in file  at line 0"
+           (mbml.errors/format-file-context "" 0)))
+    (is (= "in file filename.txt"
+           (mbml.errors/format-file-context "filename.txt" nil)))))
+
+;;; ------------------------------------------ YAML Error Tests ------------------------------------------
+
+(deftest ^:parallel format-yaml-error-test
+  (testing "format-yaml-error with file context"
+    (let [yaml-content "invalid: yaml: content:\n  - missing\n    quote"
+          cause (Exception. "YAML parsing failed")
+          error (mbml.errors/format-yaml-error yaml-content cause "test.yaml")]
+
+      (is (instance? clojure.lang.ExceptionInfo error))
+      (is (= cause (ex-cause error)))
+      (is (=? {:type :yaml-parse-error
+               :file "test.yaml"
+               :data {:yaml-content yaml-content
+                      :line-count 3}
+               :suggestions vector?
+               :message string?
+               :timestamp some?}
+              (ex-data error)))
+      (is (some #(re-find #"indentation" %) (:suggestions (ex-data error))))))
+
+  (testing "format-yaml-error without file context"
+    (let [yaml-content "bad:\nyaml"
+          cause (Exception. "Parse error")
+          error (mbml.errors/format-yaml-error yaml-content cause)]
+
+      (is (=? {:type :yaml-parse-error
+               :file nil
+               :data {:yaml-content yaml-content
+                      :line-count 2}
+               :message string?
+               :timestamp some?}
+              (ex-data error)))))
+
+  (testing "format-yaml-error with empty content"
+    (let [error (mbml.errors/format-yaml-error "" (Exception. "Empty"))]
+      (is (=? {:data {:yaml-content ""
+                      :line-count 1}}
+              (ex-data error)))))
+
+  (testing "format-yaml-error with large content provides context"
+    (let [large-yaml (str/join "\n" (repeat 50 "line: value"))
+          error (mbml.errors/format-yaml-error large-yaml (Exception. "Large file error"))]
+      (is (=? {:data {:line-count 50}
+               :context #".*50 lines.*"
+               :type :yaml-parse-error}
+              (ex-data error))))))
+
+;;; ------------------------------------- Schema Validation Error Tests -------------------------------------
+
+(deftest ^:parallel format-schema-validation-error-test
+  (testing "format-schema-validation-error with file context"
+    (let [mock-exception (ex-info "Validation failed"
+                                  {:error {:errors [{:path [:name] :message "missing required field"}
+                                                    {:path [:entity] :message "invalid entity type"}]}})
+          invalid-data {:entity "invalid" :database "test"}
+          error (mbml.errors/format-schema-validation-error mock-exception invalid-data "transform.yaml")]
+
+      (is (instance? clojure.lang.ExceptionInfo error))
+      (is (=? {:type :schema-validation
+               :file "transform.yaml"
+               :data {:failed-data invalid-data
+                      :errors [{:path [:name] :message "missing required field"}
+                               {:path [:entity] :message "invalid entity type"}]}
+               :suggestions vector?
+               :message string?
+               :timestamp some?}
+              (ex-data error)))
+      (is (some #(re-find #"required fields" %) (:suggestions (ex-data error))))))
+
+  (testing "format-schema-validation-error without file context"
+    (let [mock-exception (ex-info "Validation failed"
+                                  {:error {:errors [{:path [:identifier] :message "blank identifier"}]}})
+          invalid-data {:name "test"}
+          error (mbml.errors/format-schema-validation-error mock-exception invalid-data)]
+
+      (is (=? {:type :schema-validation
+               :file nil
+               :data {:failed-data invalid-data
+                      :errors [{:path [:identifier] :message "blank identifier"}]}
+               :message string?
+               :timestamp some?}
+              (ex-data error)))))
+
+  (testing "format-schema-validation-error suggestions are comprehensive"
+    (let [mock-exception (ex-info "Validation failed" {:error {:errors []}})
+          error (mbml.errors/format-schema-validation-error mock-exception {})
+          suggestions (:suggestions (ex-data error))]
+      (is (>= (count suggestions) 3))
+      (is (some #(re-find #"entity.*identifier.*database.*target" %) suggestions))
+      (is (some #(re-find #"Transform:v1" %) suggestions)))))
+
+;;; --------------------------------------- Front-matter Error Tests ---------------------------------------
+
+(deftest ^:parallel format-frontmatter-error-test
+  (testing "format-frontmatter-error for missing markers"
+    (let [content "SELECT * FROM table -- no METABASE markers"
+          error (mbml.errors/format-frontmatter-error :missing-markers content "query.sql")]
+
+      (is (=? {:type :frontmatter-error
+               :file "query.sql"
+               :data {:error-type :missing-markers
+                      :content-length (count content)
+                      :file-extension "sql"}
+               :suggestions vector?
+               :message string?
+               :timestamp some?}
+              (ex-data error)))
+      (is (some #(re-find #"METABASE_BEGIN.*METABASE_END" %) (:suggestions (ex-data error))))
+      (is (some #(re-find #"-- METABASE_BEGIN" %) (:suggestions (ex-data error))))))
+
+  (testing "format-frontmatter-error without file extension"
+    (let [error (mbml.errors/format-frontmatter-error :missing-markers "content" nil)]
+      (is (=? {:data {:file-extension nil}
+               :file nil
+               :suggestions vector?}
+              (ex-data error)))
+      (is (some #(re-find #"METABASE_BEGIN.*METABASE_END" %) (:suggestions (ex-data error)))))))
+
+;;; ---------------------------------------- File System Error Tests ----------------------------------------
+
+(deftest ^:parallel format-file-error-test
+  (testing "format-file-error for file not found"
+    (let [cause (Exception. "File does not exist")
+          error (mbml.errors/format-file-error :file-not-found "missing.yaml" cause)]
+
+      (is (instance? clojure.lang.ExceptionInfo error))
+      (is (= cause (.getCause error)))
+      (is (=? {:type :file-not-found
+               :file "missing.yaml"
+               :suggestions vector?
+               :message string?
+               :timestamp some?}
+              (ex-data error)))
+      (is (some #(re-find #"file path.*correct" %) (:suggestions (ex-data error))))
+      (is (some #(re-find #"permission.*access" %) (:suggestions (ex-data error))))))
+
+  (testing "format-file-error for permission denied"
+    (let [cause (Exception. "Access denied")
+          error (mbml.errors/format-file-error :permission-denied "secure.yaml" cause)]
+
+      (is (=? {:type :permission-denied
+               :file "secure.yaml"
+               :suggestions vector?}
+              (ex-data error)))
+      (is (some #(re-find #"file permissions.*readable" %) (:suggestions (ex-data error))))
+      (is (some #(re-find #"parent directory" %) (:suggestions (ex-data error))))))
+
+  (testing "format-file-error for I/O errors"
+    (let [cause (Exception. "Disk full")
+          error (mbml.errors/format-file-error :io-error "data.yaml" cause)]
+
+      (is (=? {:type :io-error
+               :suggestions vector?}
+              (ex-data error)))
+      (is (some #(re-find #"locked.*another process" %) (:suggestions (ex-data error))))
+      (is (some #(re-find #"disk space.*resources" %) (:suggestions (ex-data error))))))
+
+  (testing "format-file-error for unknown error types"
+    (let [error (mbml.errors/format-file-error :unknown-error "file.txt" (Exception. "Unknown"))
+          suggestions (:suggestions (ex-data error))]
+      (is (= 1 (count suggestions)))
+      (is (re-find #"file accessibility.*permissions" (first suggestions)))))
+
+  (testing "format-file-error message includes error type description"
+    (let [error (mbml.errors/format-file-error :file-not-found "test.yaml" (Exception.))]
+      (is (re-find #"File not found.*test\.yaml" (.getMessage error))))))
+
+;;; ------------------------------------- Unsupported File Error Tests ------------------------------------
+
+(deftest ^:parallel format-unsupported-file-error-test
+  (testing "format-unsupported-file-error basic functionality"
+    (let [error (mbml.errors/format-unsupported-file-error "document.txt")]
+      (is (=? {:type :unsupported-format
+               :file "document.txt"
+               :suggestions vector?
+               :message string?
+               :timestamp some?}
+              (ex-data error)))
+      (is (some #(re-find #"\.yaml.*\.yml" %) (:suggestions (ex-data error))))
+      (is (some #(re-find #"\.sql.*SQL.*front-matter" %) (:suggestions (ex-data error))))
+      (is (some #(re-find #"\.py.*Python.*front-matter" %) (:suggestions (ex-data error))))))
+
+  (testing "format-unsupported-file-error message format"
+    (let [error (mbml.errors/format-unsupported-file-error "bad.exe")]
+      (is (re-find #"Unsupported file format.*bad\.exe" (.getMessage error)))))
+
+  (testing "format-unsupported-file-error with various file types"
+    (let [error (mbml.errors/format-unsupported-file-error "unknown")]
+      (is (=? {:file "unknown"
+               :type :unsupported-format}
+              (ex-data error))))))
+
+;;; -------------------------------------- Internationalization Tests ------------------------------------
+
+(deftest ^:parallel internationalization-test
+  (testing "error messages use deferred-tru"
+    ; Test that error type definitions use deferred-tru
+    (is (every? i18n/localized-string? (vals mbml.errors/error-types))))
+
+  (testing "error functions produce localized strings"
+    (let [error (mbml.errors/format-file-error :file-not-found "test.yaml" (Exception.))]
+      ; Message should be a string (deferred-tru gets resolved)
+      (is (string? (.getMessage error))))
+
+    (let [error (mbml.errors/format-yaml-error "bad yaml" (Exception.) "file.yaml")]
+      (is (string? (.getMessage error))))
+
+    (let [error (mbml.errors/format-frontmatter-error :missing-markers "content" "file.sql")]
+      (is (string? (.getMessage error)))))
+
+  (testing "suggestions are localized"
+    (let [error (mbml.errors/format-schema-validation-error (ex-info "test message" {:error {}}) "file.yaml")
+          suggestions (:suggestions (ex-data error))]
+      (is (every? string? suggestions))
+      (is (every? #(> (count %) 0) suggestions))))
+
+  (testing "context messages are localized"
+    (let [error (mbml.errors/format-yaml-error
+                 (str/join "\n" (repeat 20 "line"))
+                 (Exception.)
+                 "large.yaml")
+          context (:context (ex-data error))]
+      (is (string? context))
+      (is (> (count context) 0)))))
+
+;;; ---------------------------------------- Error Types Tests ----------------------------------------
+
+(deftest ^:parallel error-types-test
+  (testing "error-types map contains expected keys"
+    (let [expected-types #{:file-not-found
+                           :unsupported-format
+                           :yaml-parse-error
+                           :empty-file-error
+                           :schema-validation
+                           :frontmatter-error
+                           :io-error}]
+      (is (= expected-types (set (keys mbml.errors/error-types))))))
+
+  (testing "error-types values are localized strings"
+    (doseq [[type desc] mbml.errors/error-types]
+      (is (keyword? type))
+      (is (i18n/localized-string? desc))))
+
+  (testing "error-types can be resolved to strings"
+    (doseq [[_ desc] mbml.errors/error-types]
+      (is (string? (str desc)))
+      (is (> (count (str desc)) 0)))))

enterprise/backend/test/metabase_enterprise/mbml/files_to_models_test.clj

@@ -0,0 +1,392 @@
+(ns metabase-enterprise.mbml.files-to-models-test
+  "Unit tests for the mbml-files->models batch loading function.
+
+  Tests comprehensive functionality of loading multiple MBML files transactionally,
+  including success scenarios, error handling, and orphaned model cleanup."
+  (:require
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [metabase-enterprise.mbml.core :as mbml.core]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+;;; ---------------------------------------- Test Data ------------------------------------------------
+
+(def transform-yaml-1-for-batch-test
+  "entity: model/Transform:v1
+name: First Batch Transform
+identifier: batch-transform-1
+description: First transform for batch testing
+tags:
+  - test
+  - batch
+database: test-data (postgres)
+target:
+  type: table
+  name: batch_output_1
+source: |
+  SELECT * FROM table1 WHERE active = true")
+
+(def transform-yaml-2-for-batch-test
+  "entity: model/Transform:v1
+name: Second Batch Transform
+identifier: batch-transform-2
+description: Second transform for batch testing
+tags:
+  - test
+  - batch
+database: test-data (postgres)
+target:
+  type: table
+  name: batch_output_2
+source: |
+  SELECT * FROM table2 WHERE status = 'complete'")
+
+(def transform-yaml-3-for-batch-test
+  "entity: model/Transform:v1
+name: Third Batch Transform
+identifier: batch-transform-3
+description: Third transform for batch testing
+tags:
+  - test
+database: test-data (postgres)
+target:
+  type: table
+  name: batch_output_3
+source: |
+  SELECT * FROM table3 WHERE created_at > '2024-01-01'")
+
+(def transform-yaml-invalid-for-batch-test
+  "entity: model/Transform:v1
+name: Invalid Transform
+identifier: invalid-transform
+description: Invalid transform that will cause failure
+tags:
+  - nonexistent-tag
+database: test-data (postgres)
+target:
+  type: table
+  name: invalid_output
+source: |
+  SELECT * FROM invalid_table")
+
+(def malformed-yaml-content
+  "entity: model/Transform:v1
+name: Malformed YAML
+identifier: malformed
+  invalid_indentation: true
+database: test-db
+target: test_table")
+
+(def transform-sql-for-model-test
+  "-- METABASE_BEGIN
+-- entity: model/Transform:v1
+-- name: SQL Transform for Model Creation
+-- identifier: sql-transform-model
+-- description: SQL transform used for testing
+-- tags:
+--   - test
+--   - daily
+-- database: test-data (postgres)
+-- target:
+--   type: table
+--   name: sql_transform_output
+-- METABASE_END
+
+SELECT
+  customer_id,
+  COUNT(*) as order_count,
+  SUM(total) as total_value
+FROM orders
+WHERE status = 'completed'
+GROUP BY customer_id;")
+
+;;; ---------------------------------------- Successful Batch Loading Tests ---------------------------
+
+(deftest mbml-files->models-successful-batch-loading-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (mt/dataset transforms-dataset/transforms-test
+      (testing "Successfully load multiple MBML files"
+        (mt/with-model-cleanup [:model/Transform]
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-batch :id} {:name "batch"}]
+            (mt/with-temp-file [temp-file-1 "transform1.yaml"
+                                temp-file-2 "transform2.yaml"
+                                temp-file-3 "transform3.yaml"]
+              (spit temp-file-1 transform-yaml-1-for-batch-test)
+              (spit temp-file-2 transform-yaml-2-for-batch-test)
+              (spit temp-file-3 transform-yaml-3-for-batch-test)
+
+              (let [results (mbml.core/mbml-files->models [temp-file-1 temp-file-2 temp-file-3])]
+                ;; Check we got 3 results
+                (is (= 3 (count results)))
+
+                ;; Check each transform was created with correct properties
+                (is (=? [{:name "First Batch Transform"
+                          :library_identifier "batch-transform-1"
+                          :description "First transform for batch testing"}
+                         {:name "Second Batch Transform"
+                          :library_identifier "batch-transform-2"
+                          :description "Second transform for batch testing"}
+                         {:name "Third Batch Transform"
+                          :library_identifier "batch-transform-3"
+                          :description "Third transform for batch testing"}]
+                        results))
+
+                ;; Verify all transforms exist in database
+                (let [saved-transforms (t2/select :model/Transform
+                                                  {:where [:in :library_identifier
+                                                           ["batch-transform-1"
+                                                            "batch-transform-2"
+                                                            "batch-transform-3"]]})]
+                  (is (= 3 (count saved-transforms)))))))))
+
+      (testing "Empty file list returns empty result"
+        (let [results (mbml.core/mbml-files->models [])]
+          (is (= [] results)))))))
+
+;;; ---------------------------------------- Transactional Rollback Tests -----------------------------
+
+(deftest mbml-files->models-transactional-rollback-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (mt/dataset transforms-dataset/transforms-test
+      (mt/with-model-cleanup [:model/Transform]
+        (testing "Transaction rolls back when one file fails"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-batch :id} {:name "batch"}]
+            (mt/with-temp-file [temp-file-1 "transform1.yaml"
+                                temp-file-2 "transform2.yaml"
+                                temp-file-invalid "invalid.yaml"]
+              (spit temp-file-1 transform-yaml-1-for-batch-test)
+              (spit temp-file-2 transform-yaml-2-for-batch-test)
+              (spit temp-file-invalid transform-yaml-invalid-for-batch-test)
+
+              ;; This should fail because "nonexistent-tag" doesn't exist
+              (is (thrown-with-msg?
+                   clojure.lang.ExceptionInfo
+                   #"Supplied tags do not exist.*nonexistent-tag"
+                   (mbml.core/mbml-files->models [temp-file-1 temp-file-2 temp-file-invalid])))
+
+              ;; Verify no transforms were created (transaction rolled back)
+              (let [created-transforms (t2/select :model/Transform
+                                                  {:where [:in :library_identifier
+                                                           ["batch-transform-1"
+                                                            "batch-transform-2"
+                                                            "invalid-transform"]]})]
+                (is (= 0 (count created-transforms)))))))
+
+        (testing "Transaction rolls back on malformed MBML"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-batch :id} {:name "batch"}]
+            (mt/with-temp-file [temp-file-1 "transform1.yaml"
+                                temp-file-malformed "malformed.yaml"]
+              (spit temp-file-1 transform-yaml-1-for-batch-test)
+              (spit temp-file-malformed malformed-yaml-content)
+
+              (is (thrown-with-msg?
+                   clojure.lang.ExceptionInfo
+                   #"YAML|parse"
+                   (mbml.core/mbml-files->models [temp-file-1 temp-file-malformed])))
+
+              ;; Verify first transform wasn't created
+              (let [created-transform (t2/select-one :model/Transform
+                                                     :library_identifier "batch-transform-1")]
+                (is (nil? created-transform))))))))))
+
+;;; ---------------------------------------- Orphaned Model Cleanup Tests -----------------------------
+
+(deftest mbml-files->models-orphaned-cleanup-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (mt/dataset transforms-dataset/transforms-test
+      (mt/with-model-cleanup [:model/Transform]
+        (testing "Clean up orphaned models with library_identifier"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-batch :id} {:name "batch"}
+                         ;; Create existing transforms with library_identifiers
+                         :model/Transform {orphan-1-id :id} {:name "Orphan Transform 1"
+                                                             :library_identifier "orphan-1"
+                                                             :description "Will be deleted"
+                                                             :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT 1\"}}"
+                                                             :target "{\"type\":\"table\",\"name\":\"orphan_1\"}"}
+                         :model/Transform {orphan-2-id :id} {:name "Orphan Transform 2"
+                                                             :library_identifier "orphan-2"
+                                                             :description "Will also be deleted"
+                                                             :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT 2\"}}"
+                                                             :target "{\"type\":\"table\",\"name\":\"orphan_2\"}"}
+                         ;; Create a transform without library_identifier (should not be deleted)
+                         :model/Transform {manual-id :id} {:name "Manual Transform"
+                                                           :library_identifier nil
+                                                           :description "Created manually, should persist"
+                                                           :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT 3\"}}"
+                                                           :target "{\"type\":\"table\",\"name\":\"manual\"}"}
+                         ;; Create tags for the orphaned transforms
+                         :model/TransformTransformTag _ {:tag_id tag-id-test :transform_id orphan-1-id :position 0}
+                         :model/TransformTransformTag _ {:tag_id tag-id-test :transform_id orphan-2-id :position 0}
+                         :model/TransformTransformTag _ {:tag_id tag-id-test :transform_id manual-id :position 0}]
+
+            (mt/with-temp-file [temp-file-1 "transform1.yaml"
+                                temp-file-2 "transform2.yaml"]
+              (spit temp-file-1 transform-yaml-1-for-batch-test)
+              (spit temp-file-2 transform-yaml-2-for-batch-test)
+
+              ;; Load new transforms
+              (let [results (mbml.core/mbml-files->models [temp-file-1 temp-file-2])]
+                (is (= 2 (count results)))
+
+                ;; Verify orphaned transforms were deleted
+                (is (nil? (t2/select-one :model/Transform :id orphan-1-id)))
+                (is (nil? (t2/select-one :model/Transform :id orphan-2-id)))
+
+                ;; Verify manual transform (without library_identifier) was preserved
+                (is (some? (t2/select-one :model/Transform :id manual-id)))
+
+                ;; Verify new transforms exist
+                (let [loaded-transforms (t2/select :model/Transform
+                                                   {:where [:in :library_identifier
+                                                            ["batch-transform-1"
+                                                             "batch-transform-2"]]})]
+                  (is (= 2 (count loaded-transforms)))))))))
+
+      (mt/with-model-cleanup [:model/Transform]
+        (testing "Update existing transforms and clean up others"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-test :id} {:name "batch"}
+                         :model/Transform {existing-id :id} {:name "Existing Transform"
+                                                             :library_identifier "batch-transform-1"
+                                                             :description "Old description"
+                                                             :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT old\"}}"
+                                                             :target "{\"type\":\"table\",\"name\":\"old_table\"}"}
+                         :model/Transform {orphan-id :id} {:name "Orphan Transform"
+                                                           :library_identifier "will-be-deleted"
+                                                           :description "Will be deleted"
+                                                           :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT orphan\"}}"
+                                                           :target "{\"type\":\"table\",\"name\":\"orphan\"}"}
+                         :model/TransformTransformTag _ {:tag_id tag-id-test :transform_id existing-id :position 0}
+                         :model/TransformTransformTag _ {:tag_id tag-id-test :transform_id orphan-id :position 0}]
+
+            (mt/with-temp-file [temp-file "transform.yaml"]
+              (spit temp-file transform-yaml-1-for-batch-test)
+
+              (let [results (mbml.core/mbml-files->models [temp-file])]
+                (is (= 1 (count results)))
+
+                ;; Verify existing transform was updated (same ID)
+                (is (= existing-id (:id (first results))))
+                (is (=? {:name "First Batch Transform"
+                         :description "First transform for batch testing"}
+                        (first results)))
+
+                ;; Verify orphan was deleted
+                (is (nil? (t2/select-one :model/Transform :id orphan-id)))))))))))
+
+;;; ---------------------------------------- Edge Cases and Error Handling Tests ----------------------
+
+(deftest mbml-files->models-edge-cases-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (mt/dataset transforms-dataset/transforms-test
+      (mt/with-model-cleanup [:model/Transform]
+        (testing "Handle duplicate identifiers in input"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-test :id} {:name "batch"}]
+            (mt/with-temp-file [temp-file-1 "transform1.yaml"
+                                temp-file-2 "transform2.yaml"]
+              ;; Both files have the same identifier
+              (spit temp-file-1 transform-yaml-1-for-batch-test)
+              (spit temp-file-2 (str/replace transform-yaml-2-for-batch-test
+                                             "batch-transform-2"
+                                             "batch-transform-1"))
+
+              ;; Should create/update only one transform (last one wins)
+              (let [results (mbml.core/mbml-files->models [temp-file-1 temp-file-2])]
+                (is (= 2 (count results)))
+
+                ;; Both results should have the same ID (updated same record)
+                (is (= (:id (first results)) (:id (second results))))
+
+                ;; Final state should be from the second file
+                (let [final-transform (t2/select-one :model/Transform
+                                                     :library_identifier "batch-transform-1")]
+                  (is (=? {:name "Second Batch Transform"
+                           :description "Second transform for batch testing"}
+                          final-transform))))))))
+
+      (mt/with-model-cleanup [:model/Transform]
+        (testing "Mixed file types in batch"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-test :id} {:name "batch"}]
+            (mt/with-temp-file [yaml-file "transform.yaml"
+                                sql-file "transform.sql"]
+              (spit yaml-file transform-yaml-1-for-batch-test)
+              (spit sql-file transform-sql-for-model-test)
+
+              (let [results (mbml.core/mbml-files->models [yaml-file sql-file])]
+                (is (= 2 (count results)))
+
+                (is (=? [{:name "First Batch Transform"}
+                         {:name "SQL Transform for Model Creation"}]
+                        results)))))))
+
+      (mt/with-model-cleanup [:model/Transform]
+        (testing "File not found in batch"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-test :id} {:name "batch"}]
+            (mt/with-temp-file [temp-file-1 "transform1.yaml"]
+              (spit temp-file-1 transform-yaml-1-for-batch-test)
+
+              (is (thrown-with-msg?
+                   clojure.lang.ExceptionInfo
+                   #"File not found"
+                   (mbml.core/mbml-files->models [temp-file-1 "/nonexistent/file.yaml"]))))))))))
+
+(deftest mbml-files->models-cleanup-scope-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (mt/dataset transforms-dataset/transforms-test
+      (mt/with-model-cleanup [:model/Transform]
+        (testing "Cleanup only affects models with library_identifier"
+          (mt/with-temp [:model/TransformTag {tag-id-test :id} {:name "test"}
+                         :model/TransformTag {tag-id-test :id} {:name "batch"}
+                         ;; Create multiple transforms with different scenarios
+                         :model/Transform {lib-transform-1 :id} {:name "Library Transform 1"
+                                                                 :library_identifier "lib-1"
+                                                                 :description "Has library ID, will be deleted"
+                                                                 :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT 1\"}}"
+                                                                 :target "{\"type\":\"table\",\"name\":\"lib1\"}"}
+                         :model/Transform {lib-transform-2 :id} {:name "Library Transform 2"
+                                                                 :library_identifier "lib-2"
+                                                                 :description "Has library ID, will be deleted"
+                                                                 :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT 2\"}}"
+                                                                 :target "{\"type\":\"table\",\"name\":\"lib2\"}"}
+                         :model/Transform {manual-1 :id} {:name "Manual Transform 1"
+                                                          :library_identifier nil
+                                                          :description "No library ID, should persist"
+                                                          :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT 3\"}}"
+                                                          :target "{\"type\":\"table\",\"name\":\"manual1\"}"}
+                         :model/Transform {manual-2 :id} {:name "Manual Transform 2"
+                                                          :library_identifier nil
+                                                          :description "No library ID, should persist"
+                                                          :source "{\"type\":\"native\",\"native\":{\"query\":\"SELECT 4\"}}"
+                                                          :target "{\"type\":\"table\",\"name\":\"manual2\"}"}
+                         :model/TransformTransformTag _ {:tag_id tag-id-test :transform_id lib-transform-1 :position 0}
+                         :model/TransformTransformTag _ {:tag_id tag-id-test :transform_id lib-transform-2 :position 0}
+                         :model/TransformTransformTag _ {:tag_id tag-id-test :transform_id manual-1 :position 0}
+                         :model/TransformTransformTag _ {:tag_id tag-id-test :transform_id manual-2 :position 0}]
+
+            (mt/with-temp-file [temp-file "transform.yaml"]
+              (spit temp-file transform-yaml-1-for-batch-test)
+
+              ;; Load one new transform
+              (let [results (mbml.core/mbml-files->models [temp-file])]
+                (is (= 1 (count results)))
+
+                ;; Verify library-managed transforms were deleted
+                (is (nil? (t2/select-one :model/Transform :id lib-transform-1)))
+                (is (nil? (t2/select-one :model/Transform :id lib-transform-2)))
+
+                ;; Verify manual transforms persist
+                (is (some? (t2/select-one :model/Transform :id manual-1)))
+                (is (some? (t2/select-one :model/Transform :id manual-2)))
+
+                ;; Verify new transform exists
+                (is (some? (t2/select-one :model/Transform
+                                          :library_identifier "batch-transform-1")))))))))))

enterprise/backend/test/metabase_enterprise/mbml/parser_test.clj

@@ -0,0 +1,599 @@
+(ns metabase-enterprise.mbml.parser-test
+  "Comprehensive unit tests for MBML parser engine functions.
+  
+  Tests cover all parser engine functions:
+  - extract-frontmatter: Front-matter extraction for SQL and Python files
+  - detect-file-type: File type detection from extensions
+  - extract-content: Content extraction routing
+  - parse-yaml: YAML parsing with error handling
+  - validate-mbml: Schema validation with Malli
+  
+  All test data is inline to ensure self-contained, reproducible tests."
+  (:require
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [metabase-enterprise.mbml.parser :as mbml.parser]))
+
+;;; ------------------------------------------ Test Data --------------------------------------------------
+
+(def valid-sql-with-frontmatter
+  "-- METABASE_BEGIN
+-- entity: model/Transform:v1
+-- name: Customer Sales Analysis
+-- identifier: customer_sales_transform
+-- description: Transform customer data with sales metrics
+-- database: analytics_db
+-- target:
+--   name: customer_sales_view
+--   type: table
+-- tags:
+--   - sales
+--   - customer
+-- METABASE_END
+
+SELECT 
+  c.customer_id,
+  c.customer_name,
+  SUM(o.total_amount) as total_sales
+FROM customers c
+JOIN orders o ON c.customer_id = o.customer_id
+GROUP BY c.customer_id, c.customer_name;")
+
+(def valid-python-with-frontmatter
+  "# METABASE_BEGIN
+# entity: model/Transform:v1
+# name: Data Processing Pipeline
+# identifier: data_processing_transform
+# description: Python-based data transformation
+# database: warehouse_db
+# target:
+#   name: processed_data
+#   type: table
+# tags:
+#   - etl
+#   - pipeline
+# METABASE_END
+
+import pandas as pd
+import numpy as np
+
+def process_data(df):
+    return df.groupby('category').sum()
+
+if __name__ == '__main__':
+    data = pd.read_csv('input.csv')
+    result = process_data(data)
+    result.to_csv('output.csv')")
+
+(def sql-without-frontmatter
+  "SELECT * FROM customers WHERE status = 'active';")
+
+(def python-without-frontmatter
+  "import pandas as pd
+print('Hello, world!')")
+
+(def sql-with-incomplete-frontmatter
+  "-- METABASE_BEGIN
+-- entity: model/Transform:v1
+-- name: Incomplete Transform
+
+SELECT * FROM table;")
+
+(def python-with-incomplete-frontmatter
+  "# METABASE_BEGIN
+# entity: model/Transform:v1
+# name: Incomplete Transform
+
+print('Missing end marker')")
+
+(def sql-with-indented-markers
+  "    -- METABASE_BEGIN
+    -- entity: model/Transform:v1
+    -- name: Indented Transform
+    -- identifier: indented_transform
+    -- database: test_db
+    -- target:
+    --   type: table
+    --   name: indented_view
+    -- METABASE_END
+
+SELECT * FROM indented_table;")
+
+(def python-with-indented-markers
+  "    # METABASE_BEGIN
+    # entity: model/Transform:v1
+    # name: Indented Transform
+    # identifier: indented_transform
+    # database: test_db
+    # target:
+    #   type: table
+    #   name: indented_view
+    # METABASE_END
+
+print('Indented Python code')")
+
+(def valid-yaml-content
+  "entity: model/Transform:v1
+name: YAML Transform
+identifier: yaml_transform
+description: Pure YAML configuration
+database: yaml_db
+target:
+  type: table
+  name: yaml_view
+tags:
+  - yaml
+  - config")
+
+(def malformed-yaml-content
+  "entity: model/Transform:v1
+name: Malformed YAML
+identifier: malformed_transform
+invalid_yaml: [unclosed_array
+database: test_db")
+
+(def empty-content "")
+
+(def whitespace-only-content "   \n\t  \n  ")
+
+;;; ---------------------------------- extract-frontmatter Tests ------------------------------------
+
+(deftest ^:parallel extract-frontmatter-sql-test
+  (testing "Extracts front-matter from valid SQL file"
+    (let [result (mbml.parser/extract-frontmatter valid-sql-with-frontmatter :sql)]
+      (is (=? {:metadata #(and (string? %)
+                               (str/includes? % "entity: model/Transform:v1")
+                               (str/includes? % "name: Customer Sales Analysis"))
+               :body #(and (string? %)
+                           (str/includes? % "SELECT")
+                           (str/includes? % "customers c"))}
+              result))))
+
+  (testing "Handles SQL without front-matter"
+    (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                          (mbml.parser/extract-frontmatter sql-without-frontmatter :sql))))
+
+  (testing "Handles incomplete SQL front-matter (missing end marker)"
+    (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                          (mbml.parser/extract-frontmatter sql-with-incomplete-frontmatter :sql))))
+
+  (testing "Handles indented markers correctly"
+    (let [result (mbml.parser/extract-frontmatter sql-with-indented-markers :sql)]
+      (is (=? {:metadata #(and (string? %)
+                               (str/includes? % "entity: model/Transform:v1")
+                               (str/includes? % "name: Indented Transform"))
+               :body #(str/includes? % "SELECT * FROM indented_table")}
+              result))))
+
+  (testing "Strips comment prefixes correctly"
+    (let [result (mbml.parser/extract-frontmatter valid-sql-with-frontmatter :sql)]
+      (is (=? {:metadata #(and (not (str/includes? % "--"))
+                               (str/includes? % "entity:")
+                               (str/includes? % "tags:\n  - sales\n  - customer"))}
+              result)))))
+
+(deftest ^:parallel extract-frontmatter-sql-edge-cases-test
+  (testing "Handles empty SQL content"
+    (is (thrown? Exception (mbml.parser/extract-frontmatter empty-content :sql))))
+
+  (testing "Handles whitespace-only SQL content"
+    (is (thrown? Exception (mbml.parser/extract-frontmatter whitespace-only-content :sql))))
+
+  (testing "Handles SQL content with only begin marker"
+    (let [content "-- METABASE_BEGIN\n-- entity: test\nSELECT * FROM test;"]
+      (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                            (mbml.parser/extract-frontmatter content :sql)))))
+
+  (testing "Handles SQL content with only end marker"
+    (let [content "-- entity: test\n-- METABASE_END\nSELECT * FROM test;"]
+      (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                            (mbml.parser/extract-frontmatter content :sql))))))
+
+;;; --------------------------------- extract-frontmatter Python Tests ---------------------------------
+
+(deftest ^:parallel extract-frontmatter-python-test
+  (testing "Extracts front-matter from valid Python file"
+    (let [result (mbml.parser/extract-frontmatter valid-python-with-frontmatter :python)]
+      (is (=? {:metadata #(and (string? %)
+                               (str/includes? % "entity: model/Transform:v1")
+                               (str/includes? % "name: Data Processing Pipeline"))
+               :body #(and (string? %)
+                           (str/includes? % "import pandas as pd")
+                           (str/includes? % "def process_data"))}
+              result))))
+
+  (testing "Handles Python without front-matter"
+    (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                          (mbml.parser/extract-frontmatter python-without-frontmatter :python))))
+
+  (testing "Handles incomplete Python front-matter (missing end marker)"
+    (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                          (mbml.parser/extract-frontmatter python-with-incomplete-frontmatter :python))))
+
+  (testing "Handles indented markers correctly"
+    (let [result (mbml.parser/extract-frontmatter python-with-indented-markers :python)]
+      (is (=? {:metadata #(and (string? %)
+                               (str/includes? % "entity: model/Transform:v1")
+                               (str/includes? % "name: Indented Transform"))
+               :body #(str/includes? % "print('Indented Python code')")}
+              result))))
+
+  (testing "Strips comment prefixes correctly"
+    (let [result (mbml.parser/extract-frontmatter valid-python-with-frontmatter :python)]
+      (is (=? {:metadata #(and (not (str/includes? % "#"))
+                               (str/includes? % "entity:")
+                               (str/includes? % "tags:\n  - etl\n  - pipeline"))}
+              result)))))
+
+(deftest ^:parallel extract-frontmatter-python-edge-cases-test
+  (testing "Handles empty Python content"
+    (is (thrown? Exception (mbml.parser/extract-frontmatter empty-content :python))))
+
+  (testing "Handles whitespace-only Python content"
+    (is (thrown? Exception (mbml.parser/extract-frontmatter whitespace-only-content :python))))
+
+  (testing "Handles Python content with only begin marker"
+    (let [content "# METABASE_BEGIN\n# entity: test\nprint('hello')"]
+      (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                            (mbml.parser/extract-frontmatter content :python)))))
+
+  (testing "Handles Python content with only end marker"
+    (let [content "# entity: test\n# METABASE_END\nprint('hello')"]
+      (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                            (mbml.parser/extract-frontmatter content :python))))))
+
+;;; ------------------------------------- *file* Dynamic Variable Tests ----------------------------------
+
+(deftest ^:parallel file-dynamic-variable-test
+  (testing "extract-frontmatter uses *file* in error messages when bound"
+    (binding [mbml.parser/*file* "test.sql"]
+      (try
+        (mbml.parser/extract-frontmatter sql-without-frontmatter :sql)
+        (is false "Should have thrown exception")
+        (catch Exception e
+          (is (str/includes? (ex-message e) "test.sql"))))))
+
+  (testing "extract-frontmatter works without *file* binding"
+    (try
+      (mbml.parser/extract-frontmatter sql-without-frontmatter :sql)
+      (is false "Should have thrown exception")
+      (catch Exception e
+        (is (not (str/includes? (ex-message e) "test.sql")))))))
+
+;;; ------------------------------------ detect-file-type Tests --------------------------------------
+
+(deftest ^:parallel detect-file-type-test
+  (testing "Detects YAML files correctly"
+    (is (= :yaml (mbml.parser/detect-file-type "config.yaml")))
+    (is (= :yaml (mbml.parser/detect-file-type "data.yml")))
+    (is (= :yaml (mbml.parser/detect-file-type "/path/to/config.yaml")))
+    (is (= :yaml (mbml.parser/detect-file-type "CONFIG.YAML"))) ; case insensitive
+    (is (= :yaml (mbml.parser/detect-file-type "DATA.YML")))) ; case insensitive
+
+  (testing "Detects SQL files correctly"
+    (is (= :sql (mbml.parser/detect-file-type "query.sql")))
+    (is (= :sql (mbml.parser/detect-file-type "/path/to/query.sql")))
+    (is (= :sql (mbml.parser/detect-file-type "QUERY.SQL")))) ; case insensitive
+
+  (testing "Detects Python files correctly"
+    (is (= :python (mbml.parser/detect-file-type "script.py")))
+    (is (= :python (mbml.parser/detect-file-type "/path/to/script.py")))
+    (is (= :python (mbml.parser/detect-file-type "SCRIPT.PY")))) ; case insensitive
+
+  (testing "Returns unknown for unsupported file types"
+    (is (= :unknown (mbml.parser/detect-file-type "document.txt")))
+    (is (= :unknown (mbml.parser/detect-file-type "image.png")))
+    (is (= :unknown (mbml.parser/detect-file-type "data.json")))
+    (is (= :unknown (mbml.parser/detect-file-type "no-extension"))))
+
+  (testing "Handles nil filename"
+    (is (= :unknown (mbml.parser/detect-file-type nil))))
+
+  (testing "Handles empty filename"
+    (is (= :unknown (mbml.parser/detect-file-type "")))))
+
+;;; ------------------------------------- extract-content Tests -------------------------------------
+
+(deftest ^:parallel extract-content-test
+  (testing "Routes YAML files correctly"
+    (let [result (mbml.parser/extract-content valid-yaml-content :yaml)]
+      (is (=? {:metadata valid-yaml-content
+               :body nil}
+              result))))
+
+  (testing "Routes SQL files correctly"
+    (let [result (mbml.parser/extract-content valid-sql-with-frontmatter :sql)]
+      (is (=? {:metadata #(and (string? %)
+                               (str/includes? % "entity: model/Transform:v1"))
+               :body #(and (string? %)
+                           (str/includes? % "SELECT"))}
+              result))))
+
+  (testing "Routes Python files correctly"
+    (let [result (mbml.parser/extract-content valid-python-with-frontmatter :python)]
+      (is (=? {:metadata #(and (string? %)
+                               (str/includes? % "entity: model/Transform:v1"))
+               :body #(and (string? %)
+                           (str/includes? % "import pandas"))}
+              result))))
+
+  (testing "Handles unknown file types"
+    (let [result (mbml.parser/extract-content "some content" :unknown)]
+      (is (=? {:metadata nil
+               :body "some content"}
+              result))))
+
+  (testing "Handles SQL files without front-matter"
+    (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                          (mbml.parser/extract-content sql-without-frontmatter :sql))))
+
+  (testing "Handles Python files without front-matter"
+    (is (thrown-with-msg? Exception #"Front-matter extraction failed"
+                          (mbml.parser/extract-content python-without-frontmatter :python)))))
+
+(deftest ^:parallel extract-content-edge-cases-test
+  (testing "Handles empty content"
+    (is (thrown? Exception (mbml.parser/extract-content empty-content :yaml))))
+
+  (testing "Handles whitespace-only content"
+    (is (thrown? Exception (mbml.parser/extract-content whitespace-only-content :sql)))))
+
+;;; --------------------------------------- parse-yaml Tests ----------------------------------------
+
+(deftest ^:parallel parse-yaml-test
+  (testing "Parses valid YAML content"
+    (let [result (mbml.parser/parse-yaml valid-yaml-content)]
+      (is (=? {:entity "model/Transform:v1"
+               :name "YAML Transform"
+               :identifier "yaml_transform"
+               :tags ["yaml" "config"]}
+              result))))
+
+  (testing "Returns nil for empty content"
+    (is (nil? (mbml.parser/parse-yaml empty-content)))
+    (is (nil? (mbml.parser/parse-yaml whitespace-only-content)))
+    (is (nil? (mbml.parser/parse-yaml nil))))
+
+  (testing "Throws exception for malformed YAML"
+    (is (thrown-with-msg? Exception #"YAML" (mbml.parser/parse-yaml malformed-yaml-content))))
+
+  (testing "Parses YAML with nested structures"
+    (let [nested-yaml "config:\n  database:\n    host: localhost\n    port: 5432\n  features:\n    - auth\n    - analytics"
+          result (mbml.parser/parse-yaml nested-yaml)]
+      (is (=? {:config {:database {:host "localhost"
+                                   :port 5432}
+                        :features ["auth" "analytics"]}}
+              result))))
+
+  (testing "Handles YAML with special characters"
+    (let [special-yaml "name: 'Transform with special chars: @#$%'\ndescription: \"Multi-line\ntext with newlines\""
+          result (mbml.parser/parse-yaml special-yaml)]
+      (is (=? {:name "Transform with special chars: @#$%"
+               :description #(str/includes? % "Multi-line")}
+              result)))))
+
+(deftest ^:parallel parse-yaml-error-handling-test
+  (testing "Provides structured error for YAML parse failure"
+    (try
+      (mbml.parser/parse-yaml malformed-yaml-content)
+      (is false "Should have thrown exception")
+      (catch Exception e
+        (let [data (ex-data e)]
+          (is (=? {:message string?
+                   :type some?}
+                  data))))))
+
+  (testing "Handles various YAML syntax errors"
+    (let [invalid-yamls ["unmatched: [array"
+                         "invalid:\n  - item\nunmatched_indent"
+                         "key: value\n  invalid_indented_key: value"]]
+      (doseq [invalid-yaml invalid-yamls]
+        (is (thrown? Exception (mbml.parser/parse-yaml invalid-yaml))
+            (str "Should throw exception for: " invalid-yaml))))))
+
+;;; ------------------------------------- validate-mbml Tests --------------------------------------
+
+(deftest ^:parallel validate-mbml-test
+  (testing "Validates valid MBML entity without source code"
+    (let [valid-data {:entity "model/Transform:v1"
+                      :name "Test Transform"
+                      :identifier "test_transform"
+                      :database "test_db"
+                      :target {:type "table" :name "test_view"}}
+          result (mbml.parser/validate-mbml valid-data nil)]
+      (is (=? {:entity "model/Transform:v1"
+               :name "Test Transform"
+               :identifier "test_transform"
+               :database "test_db"
+               :target {:type "table" :name "test_view"}}
+              result))
+      (is (not (contains? result :body)))))
+
+  (testing "Validates valid MBML entity with source code"
+    (let [valid-data {:entity "model/Transform:v1"
+                      :name "Test Transform"
+                      :identifier "test_transform"
+                      :database "test_db"
+                      :target {:type "table" :name "test_view"}}
+          source-code "SELECT * FROM test_table;"
+          result (mbml.parser/validate-mbml valid-data source-code)]
+      (is (=? {:body source-code}
+              result))))
+
+  (testing "Validates MBML entity with optional fields"
+    (let [valid-data {:entity "model/Transform:v1"
+                      :name "Complete Transform"
+                      :identifier "complete_transform"
+                      :database "test_db"
+                      :target {:type "table" :name "test_view"}
+                      :description "A complete transform with all fields"
+                      :tags ["test" "complete"]}
+          result (mbml.parser/validate-mbml valid-data nil)]
+      (is (=? {:description "A complete transform with all fields"
+               :tags ["test" "complete"]}
+              result))))
+
+  (testing "Throws exception for missing required fields"
+    (let [invalid-data {:entity "model/Transform:v1"
+                        :name "Incomplete Transform"}] ; missing identifier, database, target
+      (is (thrown? Exception (mbml.parser/validate-mbml invalid-data nil)))))
+
+  (testing "Throws exception for invalid entity type"
+    (let [invalid-data {:entity "invalid/EntityType:v1"
+                        :name "Invalid Transform"
+                        :identifier "invalid_transform"
+                        :database "test_db"
+                        :target {:type "table" :name "test_view"}}]
+      (is (thrown? Exception (mbml.parser/validate-mbml invalid-data nil)))))
+
+  (testing "Throws exception for invalid field types"
+    (let [invalid-data {:entity "model/Transform:v1"
+                        :name 123 ; should be string
+                        :identifier "test_transform"
+                        :database "test_db"
+                        :target {:type "table" :name "test_view"}}]
+      (is (thrown? Exception (mbml.parser/validate-mbml invalid-data nil))))))
+
+(deftest ^:parallel validate-mbml-error-handling-test
+  (testing "Provides structured error for validation failure"
+    (try
+      (mbml.parser/validate-mbml {:entity "invalid"} nil)
+      (is false "Should have thrown exception")
+      (catch Exception e
+        (is (string? (ex-message e)))
+        (is (not (str/blank? (ex-message e)))))))
+
+  (testing "Validates field constraints"
+    (let [test-cases [{:entity "" ; empty entity
+                       :name "Test"
+                       :identifier "test"
+                       :database "db"
+                       :target {:type "table" :name "view"}}
+                      {:entity "model/Transform:v1"
+                       :name "" ; empty name
+                       :identifier "test"
+                       :database "db"
+                       :target {:type "table" :name "view"}}
+                      {:entity "model/Transform:v1"
+                       :name "Test"
+                       :identifier "" ; empty identifier
+                       :database "db"
+                       :target {:type "table" :name "view"}}]]
+      (doseq [invalid-data test-cases]
+        (is (thrown? Exception (mbml.parser/validate-mbml invalid-data nil))
+            (str "Should throw exception for: " invalid-data))))))
+
+;;; ----------------------------------- Frontmatter Edge Cases -----------------------------------
+
+(deftest ^:parallel frontmatter-edge-cases-test
+  (testing "Handles multiple METABASE blocks in SQL (uses first valid one)"
+    (let [content "-- METABASE_BEGIN
+-- entity: model/Transform:v1
+-- name: First Block
+-- identifier: first_block
+-- database: test_db
+-- target: first_view
+-- METABASE_END
+
+SELECT * FROM first_table;
+
+-- METABASE_BEGIN  
+-- entity: model/Transform:v1
+-- name: Second Block
+-- identifier: second_block
+-- database: test_db
+-- target: second_view
+-- METABASE_END
+
+SELECT * FROM second_table;"
+          result (mbml.parser/extract-frontmatter content :sql)]
+      (is (=? {:metadata #(and (str/includes? % "name: First Block")
+                               (not (str/includes? % "name: Second Block")))
+               :body #(and (str/includes? % "SELECT * FROM first_table")
+                           (str/includes? % "SELECT * FROM second_table"))}
+              result))))
+
+  (testing "Handles nested comment structures in SQL"
+    (let [content "-- METABASE_BEGIN
+-- entity: model/Transform:v1
+-- name: Nested Comments Test
+-- identifier: nested_test
+-- database: test_db
+-- target: test_view
+-- description: |
+--   This is a multi-line description
+--   with -- nested comment markers
+--   that should be preserved
+-- METABASE_END
+
+/* This is a block comment
+   with -- line comment inside */
+SELECT * FROM test;"
+          result (mbml.parser/extract-frontmatter content :sql)]
+      (is (=? {:metadata #(and (str/includes? % "description: |")
+                               (str/includes? % "This is a multi-line description"))
+               :body #(str/includes? % "/* This is a block comment")}
+              result))))
+
+  (testing "Handles markers with different spacing"
+    (let [variations ["--METABASE_BEGIN\n-- entity: test\n--METABASE_END"
+                      "--  METABASE_BEGIN\n-- entity: test\n--  METABASE_END"
+                      "-- METABASE_BEGIN \n-- entity: test\n-- METABASE_END "]]
+      (doseq [content variations]
+        (let [result (mbml.parser/extract-frontmatter (str content "\nSELECT 1;") :sql)]
+          (is (=? {:metadata string?}
+                  result)
+              (str "Should extract metadata from: " content)))))))
+
+(deftest ^:parallel unicode-content-test
+  (testing "Handles unicode characters in SQL frontmatter"
+    (let [content "-- METABASE_BEGIN
+-- entity: model/Transform:v1
+-- name: Análisis de Données 数据分析
+-- identifier: unicode_transform
+-- description: Transform with émojis 🚀 and ñoñó characters
+-- database: test_db
+-- target: test_view
+-- METABASE_END
+
+SELECT * FROM test_tëst;"
+          result (mbml.parser/extract-frontmatter content :sql)]
+      (is (=? {:metadata #(and (str/includes? % "Análisis de Données 数据分析")
+                               (str/includes? % "émojis 🚀"))
+               :body #(str/includes? % "test_tëst")}
+              result))))
+
+  (testing "Handles unicode characters in Python frontmatter"
+    (let [content "# METABASE_BEGIN
+# entity: model/Transform:v1
+# name: Pythön Trànsförm
+# identifier: python_unicode
+# description: Unicode test with 中文 characters
+# database: test_db
+# target: test_view
+# METABASE_END
+
+print('Hello 世界')"
+          result (mbml.parser/extract-frontmatter content :python)]
+      (is (=? {:metadata #(and (str/includes? % "Pythön Trànsförm")
+                               (str/includes? % "Unicode test with 中文"))
+               :body #(str/includes? % "Hello 世界")}
+              result))))
+
+  (testing "Handles unicode in YAML parsing"
+    (let [yaml-content "entity: model/Transform:v1
+name: Transformación Ünicøde
+identifier: unicode_yaml_test
+description: Test with 日本語 and Русский
+database: tëst_db
+target: ünicode_view
+tags:
+  - tést
+  - 测试"
+          result (mbml.parser/parse-yaml yaml-content)]
+      (is (=? {:name "Transformación Ünicøde"
+               :description #(and (str/includes? % "日本語")
+                                  (str/includes? % "Русский"))
+               :database "tëst_db"
+               :tags ["tést" "测试"]}
+              result)))))

enterprise/backend/test/metabase_enterprise/mbml/schema_test.clj

@@ -0,0 +1,268 @@
+(ns metabase-enterprise.mbml.schema-test
+  "Comprehensive unit tests for MBML Malli schema definitions.
+
+  Tests all schema validations for Transform:v1 entities including required fields,
+  optional fields, entity type validation, and error message testing with internationalization."
+  (:require
+   [clojure.test :refer [are deftest ^:parallel is testing]]
+   [metabase-enterprise.mbml.schema :as mbml.schema]
+   [metabase.util.malli.registry :as mr]))
+
+;;; ------------------------------------------ Entity Type Schema Tests ------------------------------------------
+
+(deftest ^:parallel entity-type-schema-test
+  (testing "::entity-type schema validation"
+    (testing "valid entity types"
+      (is (true? (mr/validate ::mbml.schema/entity-type "model/Transform:v1"))))
+
+    (testing "invalid entity types"
+      (are [invalid-type] (= false (mr/validate ::mbml.schema/entity-type invalid-type))
+        "invalid-type"
+        "Transform:v1"
+        "model/Transform"
+        "model/Transform:v2"
+        ""
+        nil
+        123
+        []
+        {}))))
+
+;;; ------------------------------------------ Required Field Schema Tests ------------------------------------
+
+(deftest ^:parallel required-field-schemas-test
+  (testing "::identifier schema validation"
+    (testing "valid identifiers"
+      (are [valid-id] (true? (mr/validate ::mbml.schema/identifier valid-id))
+        "my-transform"
+        "transform_with_underscores"
+        "Transform123"
+        "a"
+        "very-long-identifier-name-with-dashes-and-numbers-123"))
+
+    (testing "invalid identifiers"
+      (are [invalid-id] (= false (mr/validate ::mbml.schema/identifier invalid-id))
+        ""
+        nil
+        123
+        []
+        {}
+        "   ")))
+
+  (testing "::name schema validation"
+    (testing "valid names"
+      (are [valid-name] (true? (mr/validate ::mbml.schema/name valid-name))
+        "My Transform"
+        "Transform with Spaces"
+        "1234567890"
+        "Special Characters !@#$%"))
+
+    (testing "invalid names"
+      (are [invalid-name] (= false (mr/validate ::mbml.schema/name invalid-name))
+        ""
+        nil
+        123
+        []
+        {}
+        "   ")))
+
+  (testing "::database schema validation"
+    (testing "valid database references"
+      (are [valid-db] (true? (mr/validate ::mbml.schema/database valid-db))
+        "production"
+        "test-db"
+        "database_123"
+        "db"))
+
+    (testing "invalid database references"
+      (are [invalid-db] (= false (mr/validate ::mbml.schema/database invalid-db))
+        ""
+        nil
+        123
+        []
+        {}
+        "   ")))
+
+  (testing "::target schema validation"
+    (testing "valid target maps"
+      (are [valid-target] (true? (mr/validate ::mbml.schema/target valid-target))
+        {:type "table" :name "output_table"}
+        {:type "table" :name "my-view"}
+        {:type "table" :name "TableName"}
+        {:type "table" :name "t"}))
+
+    (testing "invalid target maps"
+      (are [invalid-target] (= false (mr/validate ::mbml.schema/target invalid-target))
+        ""
+        nil
+        123
+        []
+        {}
+        "   "
+        {:name "table"} ; missing type
+        {:type "table"} ; missing name
+        {:type "view" :name "table"} ; invalid type
+        {:type "table" :name ""} ; empty name
+        {:type "table" :name nil}))))
+
+;;; ------------------------------------------ Optional Field Schema Tests ------------------------------------
+
+(deftest ^:parallel optional-field-schemas-test
+  (testing "::description schema validation"
+    (testing "valid descriptions"
+      (are [valid-desc] (true? (mr/validate ::mbml.schema/description valid-desc))
+        "A description"
+        ""
+        "Multi-line\ndescription"
+        "Description with special chars !@#$%"
+        nil))
+
+    (testing "invalid descriptions"
+      (are [invalid-desc] (= false (mr/validate ::mbml.schema/description invalid-desc))
+        123
+        []
+        {})))
+
+  (testing "::tags schema validation"
+    (testing "valid tags"
+      (are [valid-tags] (true? (mr/validate ::mbml.schema/tags valid-tags))
+        ["tag1" "tag2"]
+        ["single-tag"]
+        []
+        nil))
+
+    (testing "invalid tags"
+      (are [invalid-tags] (= false (mr/validate ::mbml.schema/tags invalid-tags))
+        "not-a-vector"
+        ["valid-tag" 123]
+        [nil "tag"]
+        123
+        {})))
+
+  (testing "::source schema validation"
+    (testing "valid source code"
+      (are [valid-source] (true? (mr/validate ::mbml.schema/source valid-source))
+        "SELECT * FROM table"
+        "# Python code\nprint('hello')"
+        ""
+        nil))
+
+    (testing "invalid source code"
+      (are [invalid-source] (= false (mr/validate ::mbml.schema/source invalid-source))
+        123
+        []
+        {}))))
+
+;;; ------------------------------------------ Transform:v1 Schema Tests --------------------------------------
+
+(deftest ^:parallel transform-v1-schema-test
+  (let [valid-complete-transform
+        {:entity "model/Transform:v1"
+         :name "Complete Transform"
+         :identifier "complete-transform"
+         :database "production"
+         :target {:type "table" :name "output_table"}
+         :description "A complete transform with all fields"
+         :tags ["etl" "production" "analytics"]
+         :source "SELECT id, name, created_at FROM users WHERE active true?"}
+
+        valid-minimal-transform
+        {:entity "model/Transform:v1"
+         :name "Minimal Transform"
+         :identifier "minimal-transform"
+         :database "test-db"
+         :target {:type "table" :name "test_output"}}]
+
+    (testing "valid Transform:v1 entities"
+      (is (true? (mr/validate ::mbml.schema/transform-v1 valid-complete-transform)))
+      (is (true? (mr/validate ::mbml.schema/transform-v1 valid-minimal-transform))))
+
+    (testing "missing required fields"
+      (are [missing-field] (= false (mr/validate ::mbml.schema/transform-v1 (dissoc valid-minimal-transform missing-field)))
+        :entity
+        :name
+        :identifier
+        :database
+        :target))
+
+    (testing "invalid field values"
+      (is (= false (mr/validate ::mbml.schema/transform-v1 (assoc valid-minimal-transform :entity "invalid-entity"))))
+      (is (= false (mr/validate ::mbml.schema/transform-v1 (assoc valid-minimal-transform :name ""))))
+      (is (= false (mr/validate ::mbml.schema/transform-v1 (assoc valid-minimal-transform :identifier nil))))
+      (is (= false (mr/validate ::mbml.schema/transform-v1 (assoc valid-minimal-transform :database 123))))
+      (is (= false (mr/validate ::mbml.schema/transform-v1 (assoc valid-minimal-transform :target [])))))
+
+    (testing "invalid optional field values"
+      (is (= false (mr/validate ::mbml.schema/transform-v1 (assoc valid-minimal-transform :description 123))))
+      (is (= false (mr/validate ::mbml.schema/transform-v1 (assoc valid-minimal-transform :tags "not-a-vector"))))
+      (is (= false (mr/validate ::mbml.schema/transform-v1 (assoc valid-minimal-transform :source {})))))
+
+    (testing "extra fields not allowed (closed schema)"
+      (is (= false (mr/validate ::mbml.schema/transform-v1 (assoc valid-minimal-transform :extra-field "value")))))))
+
+;;; ------------------------------------------ MBML Entity Multi-dispatch Tests ------------------------------
+
+(deftest ^:parallel mbml-entity-schema-test
+  (let [valid-transform
+        {:entity "model/Transform:v1"
+         :name "Test Transform"
+         :identifier "test-transform"
+         :database "test-db"
+         :target {:type "table" :name "test_table"}
+         :description "Test description"
+         :tags ["test"]
+         :source "SELECT 1"}]
+
+    (testing "valid MBML entities"
+      (is (true? (mr/validate ::mbml.schema/mbml-entity valid-transform))))
+
+    (testing "invalid entity dispatch"
+      (is (= false (mr/validate ::mbml.schema/mbml-entity (assoc valid-transform :entity "model/Unknown:v1")))))
+
+    (testing "missing entity field"
+      (is (= false (mr/validate ::mbml.schema/mbml-entity (dissoc valid-transform :entity)))))
+
+    (testing "entity field with wrong type"
+      (is (= false (mr/validate ::mbml.schema/mbml-entity (assoc valid-transform :entity 123)))))))
+
+;;; ------------------------------------------ Error Message Testing ------------------------------------------
+
+(deftest ^:parallel schema-error-messages-test
+  (testing "error explanations are provided"
+    (let [invalid-transform {:entity "invalid" :name "" :identifier nil}
+          errors (mr/explain ::mbml.schema/transform-v1 invalid-transform)]
+      (is (some? errors) "Should provide error explanation for invalid data")))
+
+  (testing "individual schema error explanations"
+    (is (some? (mr/explain ::mbml.schema/entity-type "invalid-type")))
+    (is (some? (mr/explain ::mbml.schema/identifier "")))
+    (is (some? (mr/explain ::mbml.schema/name nil)))
+    (is (some? (mr/explain ::mbml.schema/database 123)))
+    (is (some? (mr/explain ::mbml.schema/target []))))
+
+  (testing "no errors for valid data"
+    (let [valid-transform
+          {:entity "model/Transform:v1"
+           :name "Valid Transform"
+           :identifier "valid-transform"
+           :database "db"
+           :target {:type "table" :name "table"}}]
+      (is (nil? (mr/explain ::mbml.schema/transform-v1 valid-transform))))))
+
+;;; ------------------------------------------ Internationalization Tests ---------------------------------
+
+(deftest ^:parallel internationalization-test
+  (testing "error messages use deferred-tru for i18n"
+    ;; Note: Testing actual i18n behavior requires setting up locale contexts
+    ;; Here we test that the schema definitions include i18n error messages
+    (let [entity-error (mr/explain ::mbml.schema/entity-type "invalid")
+          identifier-error (mr/explain ::mbml.schema/identifier "")
+          name-error (mr/explain ::mbml.schema/name nil)]
+
+      (is (some? entity-error) "Entity type validation should provide error")
+      (is (some? identifier-error) "Identifier validation should provide error")
+      (is (some? name-error) "Name validation should provide error")
+
+      ;; Check that error messages contain meaningful text
+      (is (string? (str entity-error)))
+      (is (string? (str identifier-error)))
+      (is (string? (str name-error))))))

enterprise/frontend/src/metabase-enterprise/api/git-sync.ts

@@ -0,0 +1,59 @@
+import type { Transform } from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+import { tag } from "./tags";
+
+export type GitTreeNode = {
+  id: string;
+  name: string;
+  type: "file" | "folder";
+  children?: GitTreeNode[];
+};
+
+export type EntityType = "transform";
+
+export type GitFileContent = {
+  path: string;
+  content: string;
+  entityType?: EntityType;
+  entity?: Transform;
+};
+
+export const gitSyncApi = EnterpriseApi.injectEndpoints({
+  endpoints: (builder) => ({
+    importGit: builder.mutation({
+      query: () => ({
+        method: "POST",
+        url: "/api/ee/library/import",
+      }),
+      invalidatesTags: [tag("git-tree"), tag("git-file-content")],
+    }),
+    exportGit: builder.mutation({
+      query: () => ({
+        method: "POST",
+        url: "/api/ee/library/export",
+      }),
+    }),
+    getRepositoryTree: builder.query<GitTreeNode, void>({
+      query: () => ({
+        method: "GET",
+        url: "/api/ee/library/source",
+      }),
+      providesTags: [tag("git-tree")],
+    }),
+    getFileContent: builder.query<GitFileContent, string>({
+      query: (path) => ({
+        method: "GET",
+        url: `/api/ee/library/source/${encodeURIComponent(path)}`,
+      }),
+      providesTags: [tag("git-file-content")],
+    }),
+  }),
+});
+
+export const {
+  useImportGitMutation,
+  useExportGitMutation,
+  useGetRepositoryTreeQuery,
+  useGetFileContentQuery,
+} = gitSyncApi;

enterprise/frontend/src/metabase-enterprise/api/index.ts

@@ -6,6 +6,7 @@ export * from "./billing-info";
 export * from "./content-translation";
 export * from "./db-routing";
 export * from "./gdrive";
+export * from "./git-sync";
 export * from "./metabot";
 export * from "./document";
 export * from "./saml";

enterprise/frontend/src/metabase-enterprise/api/tags.ts

@@ -21,6 +21,8 @@ export const ENTERPRISE_TAG_TYPES = [
   "transform-job",
   "transform-job-via-tag",
   "transform-run",
+  "git-tree",
+  "git-file-content",
 ] as const;
 
 export type EnterpriseTagType = (typeof ENTERPRISE_TAG_TYPES)[number];

enterprise/frontend/src/metabase-enterprise/git_sync/FileContentViewer.tsx

@@ -0,0 +1,70 @@
+import { useState } from "react";
+import { t } from "ttag";
+
+import {
+  Box,
+  Code,
+  Flex,
+  ScrollArea,
+  SegmentedControl,
+  Text,
+} from "metabase/ui";
+import type { GitFileContent } from "metabase-enterprise/api";
+import type { Transform } from "metabase-types/api";
+
+import { QueryView } from "../transforms/components/QueryView/QueryView";
+
+interface FileContentViewerProps {
+  content: GitFileContent;
+}
+
+export const FileContentViewer = ({ content }: FileContentViewerProps) => {
+  const [viewMode, setViewMode] = useState<"source" | "preview">(
+    content.entityType ? "preview" : "source",
+  );
+
+  const canShowEntity = content.entityType && content.entity;
+
+  return (
+    <Flex h="100%" direction="column">
+      <Flex
+        p="md"
+        align="center"
+        justify="space-between"
+        style={{
+          borderBottom: "1px solid var(--mb-color-border)",
+        }}
+      >
+        <Text size="md" c="text-medium">
+          {content.path}
+        </Text>
+        {canShowEntity && (
+          <SegmentedControl
+            value={viewMode}
+            onChange={(value) => setViewMode(value as "source" | "preview")}
+            data={[
+              { label: t`Source`, value: "source" },
+              { label: t`Preview`, value: "preview" },
+            ]}
+            size="xs"
+          />
+        )}
+      </Flex>
+      <ScrollArea h="100%">
+        {viewMode === "source" || !canShowEntity ? (
+          <Box maw="900px" mx="auto" p="md">
+            <Code block p="md" style={{ fontSize: "13px", lineHeight: 1.6 }}>
+              {content.content}
+            </Code>
+          </Box>
+        ) : (
+          <Box p="md">
+            {content.entityType === "transform" && (
+              <QueryView query={(content.entity as Transform).source.query} />
+            )}
+          </Box>
+        )}
+      </ScrollArea>
+    </Flex>
+  );
+};

enterprise/frontend/src/metabase-enterprise/git_sync/GitChangeList.tsx

@@ -0,0 +1,40 @@
+import { t } from "ttag";
+
+import {
+  SettingsPageWrapper,
+  SettingsSection,
+} from "metabase/admin/components/SettingsSection";
+import { Table } from "metabase/common/components/Table";
+import { Button } from "metabase/ui";
+
+export const GitChangeList = () => {
+  return (
+    <SettingsPageWrapper title="Git changes">
+      <SettingsSection title="Pending changes">
+        <Table
+          rows={[
+            { id: 1, change: "Added new dashboard", author: "Alice" },
+            { id: 2, change: "Updated metric definition", author: "Bob" },
+            { id: 3, change: "Removed segment", author: "Charlie" },
+          ]}
+          columns={[
+            { key: "author", name: "Author" },
+            { key: "change", name: "Change" },
+            { key: "actions", name: "" },
+          ]}
+          rowRenderer={(row) => (
+            <tr>
+              <td style={{ padding: "8px 16px" }}>{row.author}</td>
+              <td style={{ padding: "8px 16px" }}>{row.change}</td>
+              <td style={{ padding: "8px 16px" }}>
+                <Button variant="filled" color="danger" size="compact-sm">
+                  {t`Reject`}
+                </Button>
+              </td>
+            </tr>
+          )}
+        />
+      </SettingsSection>
+    </SettingsPageWrapper>
+  );
+};

enterprise/frontend/src/metabase-enterprise/git_sync/GitSyncSettings.tsx

@@ -0,0 +1,91 @@
+import { t } from "ttag";
+
+import {
+  SettingsPageWrapper,
+  SettingsSection,
+} from "metabase/admin/components/SettingsSection";
+import { AdminSettingInput } from "metabase/admin/settings/components/widgets/AdminSettingInput";
+import ActionButton from "metabase/common/components/ActionButton";
+import { useSetting } from "metabase/common/hooks";
+import { Box, Flex, Group, Icon, Stack, Text } from "metabase/ui";
+import { useImportGitMutation } from "metabase-enterprise/api";
+
+export const GitSyncSettings = () => {
+  const [importGit, { isLoading: isImporting }] = useImportGitMutation();
+
+  const isLoading = isImporting;
+
+  return (
+    <SettingsPageWrapper title={t`Git sync settings`}>
+      <SettingsSection
+        title={t`Configure git`}
+        description={
+          <Stack gap="xs">
+            <Text>
+              {/* eslint-disable-next-line */}
+              {t`Enable Git sync to back up and version control your Metabase library.`}
+            </Text>
+            <GitSyncStatus />
+          </Stack>
+        }
+      >
+        <AdminSettingInput
+          name="git-sync-url"
+          title={t`Git Url`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="git-sync-token"
+          title={t`Github PAT`}
+          inputType="password"
+        />
+      </SettingsSection>
+      <SettingsSection
+        title={t`Sync control`}
+        description={t`Manually trigger a git sync to push any local changes to the remote repository and pull down any changes from the remote repository.`}
+      >
+        <Flex gap="md" align="end">
+          <AdminSettingInput
+            name="git-sync-import-branch"
+            // eslint-disable-next-line
+            description={t`Metabase will pull in all content from this branch`}
+            title={t`Import branch`}
+            inputType="text"
+            w="20rem"
+          />
+          <ActionButton
+            primary
+            actionFn={() => importGit({}).unwrap()}
+            variant="filled"
+            failedText={t`Sync failed`}
+            activeText={t`Syncing...`}
+            successText={t`Synced`}
+            useLoadingSpinner
+            disabled={isLoading}
+          >
+            <Group align="center" gap="sm">
+              <Icon name="download" />
+              {t`Import`}
+            </Group>
+          </ActionButton>
+        </Flex>
+      </SettingsSection>
+    </SettingsPageWrapper>
+  );
+};
+
+const GitSyncStatus = () => {
+  const syncStatus = useSetting("git-sync-configured");
+
+  const color = syncStatus ? "success" : "error";
+  const message = syncStatus
+    ? t`Git sync is configured.`
+    : t`Git sync is not configured.`;
+
+  return (
+    <Flex align={"center"}>
+      <Box bdrs="50%" h="14" w="14" mr="sm" bg={color} />
+      <Text c="text-medium">{message}</Text>
+    </Flex>
+  );
+};

enterprise/frontend/src/metabase-enterprise/git_sync/LibraryNav.tsx

@@ -0,0 +1,13 @@
+import { t } from "ttag";
+
+import { SettingsNavItem } from "metabase/admin/settings/components/SettingsNav";
+
+export const LibraryNav = () => {
+  return (
+    <SettingsNavItem path="library/sync" label={t`Library`} icon="folder">
+      {/* <SettingsNavItem path="library/sync" label={t`Sync`} /> */}
+      {/* Not used for now so let's hide */}
+      {/* <SettingsNavItem path="library/changes" label={t`Changes`} /> */}
+    </SettingsNavItem>
+  );
+};

enterprise/frontend/src/metabase-enterprise/git_sync/LibraryView.tsx

@@ -0,0 +1,215 @@
+import { useMemo, useState } from "react";
+import { t } from "ttag";
+
+import { Tree } from "metabase/common/components/tree";
+import type { ITreeNodeItem } from "metabase/common/components/tree/types";
+import { useSetting } from "metabase/common/hooks";
+import { useDebouncedValue } from "metabase/common/hooks/use-debounced-value";
+import { useDispatch } from "metabase/lib/redux";
+import { addUndo } from "metabase/redux/undo";
+import {
+  Box,
+  Button,
+  Flex,
+  Icon,
+  Loader,
+  ScrollArea,
+  Stack,
+  Text,
+  TextInput,
+  Title,
+} from "metabase/ui";
+import {
+  type GitTreeNode,
+  useGetFileContentQuery,
+  useGetRepositoryTreeQuery,
+  useImportGitMutation,
+} from "metabase-enterprise/api";
+
+import { FileContentViewer } from "./FileContentViewer";
+
+function convertGitTreeToTreeItems(node: GitTreeNode): ITreeNodeItem {
+  return {
+    id: node.id,
+    name: node.id,
+    icon: node.type === "folder" ? "folder" : "document",
+    children: node.children?.map(convertGitTreeToTreeItems),
+  };
+}
+
+function filterTreeItems(
+  items: ITreeNodeItem[],
+  filter: string,
+): ITreeNodeItem[] {
+  if (!filter) {
+    return items;
+  }
+
+  const lowerFilter = filter.toLowerCase();
+  const result: ITreeNodeItem[] = [];
+
+  for (const item of items) {
+    const nameMatches = item.name.toLowerCase().includes(lowerFilter);
+    const filteredChildren = item.children
+      ? filterTreeItems(item.children, filter)
+      : undefined;
+
+    if (nameMatches || (filteredChildren && filteredChildren.length > 0)) {
+      result.push({
+        ...item,
+        children: filteredChildren,
+      });
+    }
+  }
+
+  return result;
+}
+
+export const LibraryView = () => {
+  const [selectedFilePath, setSelectedFilePath] = useState<string | null>(null);
+  const [filterValue, setFilterValue] = useState("");
+  const debouncedFilter = useDebouncedValue(filterValue, 300);
+  const importBranch = useSetting("git-sync-import-branch");
+  const [importGit, { isLoading: isImporting }] = useImportGitMutation();
+  const dispatch = useDispatch();
+
+  const { data: treeData, isFetching: isTreeFetching } =
+    useGetRepositoryTreeQuery();
+  const {
+    data: fileContent,
+    isLoading: isFileLoading,
+    isFetching: isFileFetching,
+  } = useGetFileContentQuery(selectedFilePath ?? "", {
+    skip: !selectedFilePath,
+  });
+
+  const handleNodeSelect = (item: ITreeNodeItem) => {
+    if (!item.children) {
+      setSelectedFilePath(item.id as string);
+    }
+  };
+
+  const handleImport = async () => {
+    try {
+      await importGit({}).unwrap();
+      dispatch(
+        addUndo({
+          message: importBranch
+            ? t`Successfully imported from ${importBranch}`
+            : t`Successfully imported from git`,
+        }),
+      );
+    } catch (error) {
+      dispatch(
+        addUndo({
+          message: t`Failed to import from git`,
+          error: true,
+        }),
+      );
+    }
+  };
+
+  const treeItems = useMemo(() => {
+    if (!treeData) {
+      return [];
+    }
+    const items = [convertGitTreeToTreeItems(treeData)];
+    return filterTreeItems(items, debouncedFilter);
+  }, [treeData, debouncedFilter]);
+
+  return (
+    <Flex h="100%" gap={0}>
+      <Box
+        w="360px"
+        h="100%"
+        style={{
+          borderRight: "1px solid var(--mb-color-border)",
+        }}
+      >
+        <Flex direction="column" h="100%">
+          <Box px="lg" pt="lg">
+            <Title order={4}>{t`Your library`}</Title>
+          </Box>
+          <Box p="md">
+            <TextInput
+              placeholder={t`Go to file`}
+              value={filterValue}
+              onChange={(e) => setFilterValue(e.currentTarget.value)}
+              leftSection={<Icon name="search" />}
+              size="sm"
+            />
+          </Box>
+          <ScrollArea flex={1}>
+            {isTreeFetching ? (
+              <Flex justify="center" align="center" h="200px">
+                <Loader size="sm" />
+              </Flex>
+            ) : (
+              <Tree
+                initiallyExpanded
+                data={treeItems}
+                selectedId={selectedFilePath ?? undefined}
+                onSelect={handleNodeSelect}
+                emptyState={
+                  <Flex justify="center" align="center" h="200px">
+                    <Stack align="center" gap="md">
+                      <Icon
+                        name="search"
+                        size={32}
+                        color="var(--mb-color-text-light)"
+                      />
+                      <Text c="text-light">{t`No results found`}</Text>
+                    </Stack>
+                  </Flex>
+                }
+              />
+            )}
+          </ScrollArea>
+          <Box
+            p="md"
+            style={{
+              borderTop: "1px solid var(--mb-color-border)",
+            }}
+          >
+            <Button
+              variant="filled"
+              fullWidth
+              leftSection={<Icon name="download" />}
+              onClick={handleImport}
+              loading={isImporting}
+            >
+              {importBranch ? t`Import from ${importBranch}` : t`Import`}
+            </Button>
+          </Box>
+        </Flex>
+      </Box>
+
+      <Box flex={1} h="100%">
+        {selectedFilePath ? (
+          isFileFetching || (isFileLoading && !fileContent) ? (
+            <Flex justify="center" align="center" h="100%">
+              <Loader />
+            </Flex>
+          ) : fileContent && fileContent.path === selectedFilePath ? (
+            <FileContentViewer content={fileContent} />
+          ) : (
+            <Flex justify="center" align="center" h="100%">
+              <Text c="text-light">{t`File not found`}</Text>
+            </Flex>
+          )
+        ) : (
+          <Flex justify="center" align="center" h="100%">
+            <Stack align="center" gap="md">
+              <Icon
+                name="document"
+                size={48}
+                color="var(--mb-color-text-light)"
+              />
+              <Text c="text-light">{t`Select a file to view its contents`}</Text>
+            </Stack>
+          </Flex>
+        )}
+      </Box>
+    </Flex>
+  );
+};

enterprise/frontend/src/metabase-enterprise/git_sync/ViewLibraryButton.tsx

@@ -0,0 +1,44 @@
+import { useState } from "react";
+import { t } from "ttag";
+
+import { useSetting } from "metabase/common/hooks";
+import { useSelector } from "metabase/lib/redux";
+import { getUserIsAdmin } from "metabase/selectors/user";
+import { Button, Icon } from "metabase/ui";
+
+import { ViewLibraryModal } from "./ViewLibraryModal";
+
+export const ViewLibraryButton = () => {
+  const [isOpen, setIsOpen] = useState(false);
+  const isAdmin = useSelector(getUserIsAdmin);
+  const syncConfigured = useSetting("git-sync-configured");
+
+  if (!isAdmin) {
+    return null;
+  }
+
+  const buttonText = syncConfigured ? t`Library` : t`Set up your library`;
+  const ariaLabel = syncConfigured ? t`Browse library` : t`Set up your library`;
+
+  return (
+    <>
+      <Button
+        variant="subtle"
+        fullWidth
+        leftSection={<Icon name="collection" />}
+        onClick={() => {
+          setIsOpen(true);
+        }}
+        aria-label={ariaLabel}
+      >
+        {buttonText}
+      </Button>
+      <ViewLibraryModal
+        isOpen={isOpen}
+        onClose={() => {
+          setIsOpen(false);
+        }}
+      />
+    </>
+  );
+};

enterprise/frontend/src/metabase-enterprise/git_sync/ViewLibraryModal.tsx

@@ -0,0 +1,45 @@
+import { Link } from "react-router";
+import { t } from "ttag";
+
+import { useSetting } from "metabase/common/hooks";
+import { Box, Button, Center, Flex, Icon, Modal } from "metabase/ui";
+
+import { LibraryView } from "./LibraryView";
+
+interface ViewLibraryModalProps {
+  isOpen: boolean;
+  onClose: () => void;
+}
+
+export const ViewLibraryModal = ({
+  isOpen,
+  onClose,
+}: ViewLibraryModalProps) => {
+  const syncConfigured = useSetting("git-sync-configured");
+
+  return (
+    <Modal.Root opened={isOpen} onClose={onClose} size="calc(100vw - 60px)">
+      <Modal.Overlay />
+      <Modal.Content h="100%">
+        <Modal.Body h="100%" p={0}>
+          {!syncConfigured ? (
+            <Box h="100%">
+              <Flex justify="center" align="center" h="100%">
+                <Center>
+                  <Button
+                    leftSection={<Icon name="collection" />}
+                    component={Link}
+                    variant="filled"
+                    to="/admin/settings/library"
+                  >{t`Set up your library`}</Button>
+                </Center>
+              </Flex>
+            </Box>
+          ) : (
+            <LibraryView />
+          )}
+        </Modal.Body>
+      </Modal.Content>
+    </Modal.Root>
+  );
+};

enterprise/frontend/src/metabase-enterprise/git_sync/index.ts

@@ -0,0 +1,14 @@
+import { PLUGIN_GIT_SYNC } from "metabase/plugins";
+
+import { GitChangeList } from "./GitChangeList";
+import { GitSyncSettings } from "./GitSyncSettings";
+import { LibraryNav } from "./LibraryNav";
+import { ViewLibraryButton } from "./ViewLibraryButton";
+
+// eslint-disable-next-line
+if ("git-sync-enabled" > "FIXME add a real token feature") {
+  PLUGIN_GIT_SYNC.GitSyncSettings = GitSyncSettings;
+  PLUGIN_GIT_SYNC.LibraryNav = LibraryNav;
+  PLUGIN_GIT_SYNC.GitChangeList = GitChangeList;
+  PLUGIN_GIT_SYNC.ViewLibraryButton = ViewLibraryButton;
+}

enterprise/frontend/src/metabase-enterprise/plugins.js

@@ -15,6 +15,7 @@ import "./collections";
 import "./content_translation";
 import "./content_verification";
 import "./database_routing";
+import "./git_sync";
 import "./whitelabel";
 import "./embedding";
 import "./embedding-sdk";

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx

@@ -70,7 +70,7 @@ export function getParsedParams({
   transformId,
 }: TransformPageParams): TransformPageParsedParams {
   return {
-    transformId: Urls.extractEntityId(transformId),
+    transformId,
   };
 }
 

frontend/src/metabase-types/api/settings.ts

@@ -621,9 +621,25 @@ export type DatabaseReplicationConnections = Record<
   { connection_id: string }
 >;
 
+export type SyncableEntity =
+  | "transform"
+  | "snippet"
+  | "dataset"
+  | "metric"
+  | "segment"
+  | "dashboard"
+  | "question";
+
 export interface EnterpriseSettings extends Settings {
   "application-colors"?: ColorSettings | null;
   "application-logo-url"?: string;
+  "git-sync-token"?: string | null;
+  "git-sync-url"?: string | null;
+  "git-sync-import-branch"?: string | null; // import branch
+  "git-sync-export-branch"?: string | null;
+  "git-sync-entities"?: Partial<Record<SyncableEntity, boolean>> | null;
+  "git-sync-configured"?: boolean | null;
+  "git-sync-read-only"?: boolean;
   "login-page-illustration"?: IllustrationSettingValue;
   "login-page-illustration-custom"?: string;
   "landing-page-illustration"?: IllustrationSettingValue;

frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx

@@ -10,6 +10,7 @@ import {
 import { UpsellGem } from "metabase/admin/upsells/components/UpsellGem";
 import { useHasTokenFeature } from "metabase/common/hooks";
 import { useSelector } from "metabase/lib/redux";
+import { PLUGIN_GIT_SYNC } from "metabase/plugins";
 import { getLocation } from "metabase/selectors/routing";
 import { Divider, Flex } from "metabase/ui";
 
@@ -46,6 +47,7 @@ export function SettingsNav() {
         {hasSaml && <SettingsNavItem path="authentication/saml" label="SAML" />}
         {hasJwt && <SettingsNavItem path="authentication/jwt" label="JWT" />}
       </SettingsNavItem>
+      <PLUGIN_GIT_SYNC.LibraryNav />
       <NavDivider />
       <SettingsNavItem path="email" label={t`Email`} icon="mail" />
       <SettingsNavItem

frontend/src/metabase/admin/settingsRoutes.tsx

@@ -2,7 +2,11 @@ import { IndexRedirect, Route } from "react-router";
 
 import { AdminSettingsLayout } from "metabase/common/components/AdminLayout/AdminSettingsLayout";
 import { NotFound } from "metabase/common/components/ErrorPages";
-import { PLUGIN_ADMIN_SETTINGS, PLUGIN_AUTH_PROVIDERS } from "metabase/plugins";
+import {
+  PLUGIN_ADMIN_SETTINGS,
+  PLUGIN_AUTH_PROVIDERS,
+  PLUGIN_GIT_SYNC,
+} from "metabase/plugins";
 
 import { GoogleAuthForm } from "./settings/auth/components/GoogleAuthForm";
 import {
@@ -60,6 +64,18 @@ export const getSettingsRoutes = () => (
       path="authentication/jwt"
       component={() => <PLUGIN_AUTH_PROVIDERS.SettingsJWTForm />}
     />
+    <Route
+      path="library"
+      component={() => <PLUGIN_GIT_SYNC.GitSyncSettings />}
+    />
+    <Route
+      path="library/sync"
+      component={() => <PLUGIN_GIT_SYNC.GitSyncSettings />}
+    />
+    <Route
+      path="library/changes"
+      component={() => <PLUGIN_GIT_SYNC.GitChangeList />}
+    />
     <Route path="maps" component={MapsSettingsPage} />
     <Route path="localization" component={LocalizationSettingsPage} />
     <Route path="uploads" component={UploadSettingsPage} />

frontend/src/metabase/common/components/ActionButton/ActionButton.jsx

@@ -125,6 +125,7 @@ export default class ActionButton extends Component {
       failedClassName,
       forceActiveStyle,
       children,
+      disabled,
       ...props
     } = this.props;
     const { active, result } = this.state;
@@ -146,6 +147,7 @@ export default class ActionButton extends Component {
               })
         }
         onClick={this.onClick}
+        disabled={disabled}
       >
         {active ? (
           useLoadingSpinner ? (

frontend/src/metabase/common/components/AppBanner/AppBanner.tsx

@@ -24,7 +24,8 @@ export const AppBanner = () => {
   const isAdmin = useSelector(getUserIsAdmin);
   const isHosted = useSelector(getIsHosted);
   const tokenStatus = useSetting("token-status");
-  const readOnly = useSetting("read-only-mode");
+  const migrateReadOnly = useSetting("read-only-mode");
+  const gitReadOnly = useSetting("git-sync-read-only");
   const isDevMode = useSetting("development-mode?");
 
   const { shouldShowLicenseTokenMissingBanner, dismissBanner } =
@@ -44,7 +45,7 @@ export const AppBanner = () => {
     return isDevMode ? <DevModeBanner /> : null;
   }
 
-  if (readOnly) {
+  if (migrateReadOnly || gitReadOnly) {
     return <ReadOnlyBanner />;
   }
 

frontend/src/metabase/common/components/tree/Tree.tsx

@@ -1,18 +1,18 @@
 import { useCallback, useEffect, useState } from "react";
 import * as React from "react";
 import { usePrevious } from "react-use";
-import _ from "underscore";
 
 import { TreeNode as DefaultTreeNode } from "./TreeNode";
 import { TreeNodeList } from "./TreeNodeList";
 import type { ITreeNodeItem } from "./types";
-import { getInitialExpandedIds } from "./utils";
+import { getAllExpandableIds, getInitialExpandedIds } from "./utils";
 
 interface TreeProps {
   data: ITreeNodeItem[];
   selectedId?: ITreeNodeItem["id"];
   role?: string;
   emptyState?: React.ReactNode;
+  initiallyExpanded?: boolean;
   onSelect?: (item: ITreeNodeItem) => void;
   TreeNode?: any; // This was previously set to TreeNodeComponent, but after upgrading to react 18, the type no longer played nice with forward ref compontents, including styled components
 }
@@ -22,28 +22,46 @@ function BaseTree({
   selectedId,
   role = "menu",
   emptyState = null,
+  initiallyExpanded = false,
   onSelect,
   TreeNode = DefaultTreeNode,
 }: TreeProps) {
-  const [expandedIds, setExpandedIds] = useState(
-    new Set(selectedId != null ? getInitialExpandedIds(selectedId, data) : []),
-  );
+  const [expandedIds, setExpandedIds] = useState(() => {
+    if (initiallyExpanded) {
+      return new Set(getAllExpandableIds(data));
+    }
+    return new Set(
+      selectedId != null ? getInitialExpandedIds(selectedId, data) : [],
+    );
+  });
   const previousSelectedId = usePrevious(selectedId);
   const prevData = usePrevious(data);
 
   useEffect(() => {
+    if (initiallyExpanded && data !== prevData) {
+      setExpandedIds(new Set(getAllExpandableIds(data)));
+      return;
+    }
+
     if (!selectedId) {
       return;
     }
     const selectedItemChanged =
       previousSelectedId !== selectedId && !expandedIds.has(selectedId);
-    if (selectedItemChanged || !_.isEqual(data, prevData)) {
+    if (selectedItemChanged || data !== prevData) {
       setExpandedIds(
         (prev) =>
           new Set([...prev, ...getInitialExpandedIds(selectedId, data)]),
       );
     }
-  }, [prevData, data, selectedId, previousSelectedId, expandedIds]);
+  }, [
+    prevData,
+    data,
+    selectedId,
+    previousSelectedId,
+    expandedIds,
+    initiallyExpanded,
+  ]);
 
   const handleToggleExpand = useCallback(
     (itemId: string | number) => {

frontend/src/metabase/common/components/tree/utils.tsx

@@ -18,3 +18,21 @@ export const getInitialExpandedIds = (
       return [];
     })
     .flat();
+
+export const getAllExpandableIds = (
+  nodes: ITreeNodeItem[],
+): ITreeNodeItem["id"][] => {
+  const ids: ITreeNodeItem["id"][] = [];
+
+  const traverse = (items: ITreeNodeItem[]) => {
+    for (const item of items) {
+      if (item.children && item.children.length > 0) {
+        ids.push(item.id);
+        traverse(item.children);
+      }
+    }
+  };
+
+  traverse(nodes);
+  return ids;
+};

frontend/src/metabase/nav/components/ReadOnlyBanner/ReadOnlyBanner.tsx

@@ -1,14 +1,33 @@
 import { t } from "ttag";
 
 import { Banner } from "metabase/common/components/Banner";
+import { useSetting } from "metabase/common/hooks";
 import { Text } from "metabase/ui";
 
 export const ReadOnlyBanner = () => {
+  const gitSyncConfigured = useSetting("git-sync-read-only");
+
+  if (gitSyncConfigured) {
+    return (
+      <Banner
+        bg="brand"
+        c="white"
+        body={
+          <Text fw="bold" c="white">
+            {/* eslint-disable-next-line no-literal-metabase-strings -- correct usage */}
+            {t`Content in this Metabase instance is managed by serious people and cannot be changed here.`}
+          </Text>
+        }
+        icon="info_filled"
+      />
+    );
+  }
+
   return (
     <Banner
       bg="warning"
       body={
-        <Text fw="bold" color="text-dark">
+        <Text fw="bold" c="text-dark">
           {/* eslint-disable-next-line no-literal-metabase-strings -- correct usage */}
           {t`Metabase is under maintenance and is operating in read-only mode. It should only take up to 30 minutes.`}
         </Text>

frontend/src/metabase/nav/containers/MainNavbar/MainNavbarContainer/MainNavbarView.tsx

@@ -20,6 +20,7 @@ import { isSmallScreen } from "metabase/lib/dom";
 import { useSelector } from "metabase/lib/redux";
 import * as Urls from "metabase/lib/urls";
 import { WhatsNewNotification } from "metabase/nav/components/WhatsNewNotification";
+import { PLUGIN_GIT_SYNC } from "metabase/plugins";
 import {
   ActionIcon,
   Flex,
@@ -243,7 +244,14 @@ export function MainNavbarView({
             </TrashSidebarSection>
           )}
         </div>
-        <WhatsNewNotification />
+        <div>
+          <WhatsNewNotification />
+          <SidebarSection>
+            {PLUGIN_GIT_SYNC.ViewLibraryButton && (
+              <PLUGIN_GIT_SYNC.ViewLibraryButton />
+            )}
+          </SidebarSection>
+        </div>
       </SidebarContentRoot>
 
       <AddDataModal opened={addDataModalOpened} onClose={closeAddDataModal} />

frontend/src/metabase/plugins/index.ts

@@ -849,3 +849,10 @@ export const PLUGIN_TRANSFORMS: TransformsPlugin = {
   getAdminPaths: () => [],
   getAdminRoutes: () => null,
 };
+
+export const PLUGIN_GIT_SYNC = {
+  LibraryNav: PluginPlaceholder,
+  GitSyncSettings: NotFoundPlaceholder,
+  GitChangeList: NotFoundPlaceholder,
+  ViewLibraryButton: PluginPlaceholder,
+};

resources/migrations/057_update_migrations.yaml

@@ -1083,6 +1083,160 @@ databaseChangeLog:
         - customChange:
             class: "metabase.app_db.custom_migrations.MigrateClickHouseDetailsToMultiDB"
 
+  - changeSet:
+      id: v57.2025-08-26T10:51:08
+      author: jds
+      comment: Add synced_to_source_of_truth column to transform table
+      changes:
+        - addColumn:
+            tableName: transform
+            columns:
+              - column:
+                  name: synced_to_source_of_truth
+                  remarks: Whether this transform is synced to the git source of truth
+                  type: ${boolean.type}
+                  defaultValueBoolean: false
+                  constraints:
+                    nullable: false
+
+  - changeSet:
+      id: v57.2025-09-05T10:00:00
+      author: edpaget
+      comment: Drop foreign key constraint on transform_run.transform_id
+      changes:
+        - dropForeignKeyConstraint:
+            baseTableName: transform_run
+            constraintName: fk_transform_run_transform_id
+      rollback:
+        - addForeignKeyConstraint:
+            baseTableName: transform_run
+            baseColumnNames: transform_id
+            referencedTableName: transform
+            referencedColumnNames: id
+            constraintName: fk_transform_run_transform_id
+            onDelete: CASCADE
+
+  - changeSet:
+      id: v57.2025-09-05T10:01:00
+      author: edpaget
+      comment: Drop foreign key constraint on transform_transform_tag.transform_id
+      changes:
+        - dropForeignKeyConstraint:
+            baseTableName: transform_transform_tag
+            constraintName: fk_transform_transform_tag_transform_id
+      rollback:
+        - addForeignKeyConstraint:
+            baseTableName: transform_transform_tag
+            baseColumnNames: transform_id
+            referencedTableName: transform
+            referencedColumnNames: id
+            constraintName: fk_transform_transform_tag_transform_id
+            onDelete: CASCADE
+
+  - changeSet:
+      id: v57.2025-09-05T10:02:00
+      author: edpaget
+      comment: Modify transform_run.transform_id to varchar(254)
+      changes:
+        - modifyDataType:
+            tableName: transform_run
+            columnName: transform_id
+            newDataType: varchar(254)
+      rollback:
+        - modifyDataType:
+            tableName: transform_run
+            columnName: transform_id
+            newDataType: int
+
+  - changeSet:
+      id: v57.2025-09-05T10:03:00
+      author: edpaget
+      comment: Modify transform_transform_tag.transform_id to varchar(254)
+      changes:
+        - modifyDataType:
+            tableName: transform_transform_tag
+            columnName: transform_id
+            newDataType: varchar(254)
+      rollback:
+        - modifyDataType:
+            tableName: transform_transform_tag
+            columnName: transform_id
+            newDataType: int
+
+  - changeSet:
+      id: v57.2025-09-05T10:04:00
+      author: edpaget
+      comment: Drop transform table
+      changes:
+        - dropTable:
+            tableName: transform
+            cascadeConstraints: true
+      rollback:
+        - createTable:
+            tableName: transform
+            remarks: The main table for Transform entities
+            columns:
+              - column:
+                  name: id
+                  type: int
+                  remarks: Unique ID
+                  autoIncrement: true
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: name
+                  remarks: Name
+                  type: ${text.type}
+                  constraints:
+                    nullable: false
+              - column:
+                  name: description
+                  type: ${text.type}
+                  remarks: the description of the transform
+                  constraints:
+                    nullable: true
+              - column:
+                  name: source
+                  remarks: JSON of source
+                  type: ${text.type}
+                  constraints:
+                    nullable: false
+              - column:
+                  name: target
+                  remarks: JSON of target
+                  type: ${text.type}
+                  constraints:
+                    nullable: false
+              - column:
+                  name: entity_id
+                  type: char(21)
+                  remarks: Random NanoID tag for unique identity.
+                  constraints:
+                    nullable: true
+                    unique: true
+              - column:
+                  remarks: The timestamp of when the transform was created
+                  name: created_at
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: The timestamp of when the transform was updated
+                  name: updated_at
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  name: synced_to_source_of_truth
+                  remarks: Whether this transform is synced to the git source of truth
+                  type: ${boolean.type}
+                  defaultValueBoolean: false
+                  constraints:
+                    nullable: false
+
 # >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
 
 ########################################################################################################################

src/metabase/collections/models/collection.clj

@@ -125,6 +125,7 @@
 (doto :model/Collection
   (derive :metabase/model)
   (derive :hook/entity-id)
+  (derive :hook/git-sync-protected)
   (derive ::mi/read-policy.full-perms-for-perms-set)
   (derive ::mi/write-policy.full-perms-for-perms-set))
 

src/metabase/core/library.clj

@@ -0,0 +1,17 @@
+(ns metabase.core.library
+  (:require
+   [metabase.classloader.core :as classloader]
+   [metabase.util.log :as log]))
+
+(defn initialize-from-git-if-available!
+  "Shim for running the git source of truth code, used by [[metabase.core.core]]. The git source of truth code only ships in
+  the Enterprise Edition™ JAR, so this checks whether the namespace exists, and if it does,
+  invokes [[metabase-enterprise.library.core/initialize-from-git!]]; otherwise, this no-ops."
+  []
+  (when (try
+          (classloader/require 'metabase-enterprise.library.core)
+          :ok
+          (catch Throwable _
+            (log/debug "metabase-enterprise.library.core not available; cannot initialize from git.")
+            nil))
+    ((resolve 'metabase-enterprise.library.core/initialize-from-git!))))

src/metabase/dashboards/models/dashboard.clj

@@ -40,7 +40,8 @@
   (derive :metabase/model)
   (derive :perms/use-parent-collection-perms)
   (derive :hook/timestamped?)
-  (derive :hook/entity-id))
+  (derive :hook/entity-id)
+  (derive :hook/git-sync-protected))
 
 (defmethod mi/can-write? :model/Dashboard
   ([instance]

src/metabase/lib/database.cljc

@@ -15,7 +15,8 @@
 
   we will attempt to resolve the correct Database ID by getting metadata for the source Card and returning its
   `:database-id`; if this is not available for one reason or another this will return `nil`."
-  [query :- ::lib.schema/query]
+  ;; TODO(johnswanson) wtf did I have to remove the query schema bit here
+  [query :- :any]
   (when-let [id (:database query)]
     (if (not= id lib.schema.id/saved-questions-virtual-database-id)
       id

src/metabase/models/interface.clj

@@ -32,6 +32,7 @@
    [toucan2.model :as t2.model]
    [toucan2.protocols :as t2.protocols]
    [toucan2.tools.before-insert :as t2.before-insert]
+   [toucan2.tools.before-update :as t2.before-update]
    [toucan2.tools.hydrate :as t2.hydrate]
    [toucan2.tools.identity-query :as t2.identity-query]
    [toucan2.util :as t2.u])
@@ -58,6 +59,10 @@
   on a future deserialization."
   false)
 
+(def ^:dynamic *syncing-source-of-truth-entities*
+  "This is dynamically bound to the set of entities that we are syncing when syncing the source of truth."
+  #{})
+
 (def ^{:arglists '([x & _args])} dispatch-on-model
   "Helper dispatch function for multimethods. Dispatches on the first arg, using [[models.dispatch/model]]."
   ;; make sure model namespace gets loaded e.g. `:model/Database` should load `metabase.model.database` if needed.
@@ -579,9 +584,32 @@
   (-> instance
       add-entity-id))
 
+(defn- prevent-sync-protected-writes
+  "Prevents writes to entities that are synced to source of truth, unless explicitly allowed."
+  [instance]
+  (when (and (try ((requiring-resolve 'metabase-enterprise.library.settings/git-sync-read-only))
+                  (catch Exception _ false))
+             (not *deserializing?*))
+    (throw (ex-info "Cannot modify entities synced to source of truth"
+                    {:entity-id (:entity_id instance)
+                     :model (model instance)})))
+  instance)
+
+(t2/define-before-insert :hook/git-sync-protected
+  [instance]
+  (prevent-sync-protected-writes instance))
+
+(t2/define-before-update :hook/git-sync-protected
+  [instance]
+  (prevent-sync-protected-writes instance))
+
 (methodical/prefer-method! #'t2.before-insert/before-insert :hook/timestamped? :hook/entity-id)
 (methodical/prefer-method! #'t2.before-insert/before-insert :hook/updated-at-timestamped? :hook/entity-id)
 (methodical/prefer-method! #'t2.before-insert/before-insert :hook/created-at-timestamped? :hook/entity-id)
+(methodical/prefer-method! #'t2.before-insert/before-insert :hook/entity-id :hook/git-sync-protected)
+(methodical/prefer-method! #'t2.before-insert/before-insert :hook/timestamped? :hook/git-sync-protected)
+(methodical/prefer-method! #'t2.before-update/before-update :hook/entity-id :hook/git-sync-protected)
+(methodical/prefer-method! #'t2.before-update/before-update :hook/timestamped? :hook/git-sync-protected)
 
 ;; --- helper fns
 (defn changes-with-pk
@@ -648,6 +676,14 @@
   {:arglists '([instance] [model pk])}
   dispatch-on-model)
 
+(defmethod can-write? :hook/git-sync-protected
+  ([instance]
+   (try (false? ((requiring-resolve 'metabase-enterprise.library.settings/git-sync-read-only)))
+        (catch Exception _ true)))
+  ([_model _pk]
+   (try (false? ((requiring-resolve 'metabase-enterprise.library.settings/git-sync-read-only)))
+        (catch Exception _ true))))
+
 #_{:clj-kondo/ignore [:unused-private-var]}
 (define-simple-hydration-method ^:private hydrate-can-write
   :can_write

src/metabase/queries/models/card.clj

@@ -116,7 +116,8 @@
   ;; You can read/write a Card if you can read/write its parent Collection
   (derive :perms/use-parent-collection-perms)
   (derive :hook/timestamped?)
-  (derive :hook/entity-id))
+  (derive :hook/entity-id)
+  (derive :hook/git-sync-protected))
 
 (defmethod mi/can-write? :model/Card
   ([instance]