Kimirock/metabase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Per-PR liquibase changelogs (#70263)

Browse Source
This commit is contained in:
Nathan Voxland
2026-03-10 16:58:26 -05:00
committed by GitHub
parent 5e2e510811
commit b12dfa4fd9
19 changed files with 534 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.gitattributes vendored
View File

@@ -1,7 +1,9 @@
# Custom merge driver for YAML migration files
# Treats each changeset as an atomic unit to avoid conflicts within changesets
# Custom merge driver for per-release YAML migration files.
# Treats each changeset as an atomic unit to avoid conflicts within changesets.
# Only applies to per-release *_update_migrations.yaml files — directory-based
# migrations (e.g. 060/*.yaml) have one file per migration and don't need this.
# Execute this in your shell to enable custom merge driver:
# echo '\n[include]\npath=../.gitconfig' >> .git/config
#
# TODO (Bryan 12/12/25) -- reenable this when we can completely verify correctness
# resources/migrations/*.yaml merge=yaml-migrations
# resources/migrations/*_update_migrations.yaml merge=yaml-migrations

2
.gitconfig
View File

@@ -1,5 +1,5 @@
# execute this in your shell to enable custom merge driver:
# echo '\n[include]\npath=../.gitconfig' >> .git/config
[merge "yaml-migrations"]
name = YAML migration changeset-aware merge
name = Per-release YAML migration changeset-aware merge
driver = bin/mage -merge-yaml-migrations %O %A %B %L %P

9
bb.edn
View File

@@ -162,6 +162,15 @@
(p/shell "bin/pre-lint-migrations-file.sh")
(lint-migrations-file/-main))}
new-migration
{:doc "Create a new directory-based migration file (v60+)"
:requires [[mage.new-migration :as new-migration]]
:examples [["./bin/mage new-migration 60 mq_indexes" "Create resources/migrations/060/20260905_mq_indexes.yaml"]]
:arg-schema [:tuple
[:int {:name "version" :description "Major version number (e.g., 60)"}]
[:string {:name "description" :description "Lowercase alphanumeric/underscore name (e.g., mq_indexes)"}]]
:task (task! (new-migration/new-migration parsed))}
jar-download
{:doc "Download (and optionally run) jar for a metabase version or branch"
:examples [["./bin/mage jar-download 50" "Download the latest enterprise version of release 50 to ~/path/to/my/metabase/jars"]

62
bin/lint-migrations-file/src/change_set/strict.clj
View File

@@ -18,16 +18,9 @@
This format should no longer be used for new migrations, use `id-timestamp-format-re` instead."
#"^v\d{2,}\.\d{2}-\d{3}$")
(s/def ::id (s/and string?
(s/or
;; new ids should conform this format
:id-with-timestamp ;; e.g: v49.2023-12-14T08:54:54
(s/and #(re-matches id-timestamp-format-re %))
:id-with-old-version-format ;; e.g: v49.00-008
(s/and #(re-matches id-number-format-re %)
;; the cut off id for this old format is v49.00-060
;; see #36787 for context
#(neg? (compare % "v49.00-061"))))))
;; Basic type check only — format enforcement is file-aware and happens in
;; `lint-migrations-file/require-change-set-ids-match-file-format`.
(s/def ::id string?)
(s/def ::author string?)
@@ -74,54 +67,6 @@
[]))) ; provide an empty list if dbms-val is nil
changes))))))
(def change-types-supporting-rollback
"This set was generated with a little grep and awk from the docs here:
https://docs.liquibase.com/workflows/liquibase-community/liquibase-auto-rollback.html
If a new change type is introduced that supports automatic rollback, it should be added
to this set."
#{:addCheckConstraint
:addColumn
:addDefaultValue
:addForeignKeyConstraint
:addLookupTable
:addNotNullConstraint
:addPrimaryKey
:addUniqueConstraint
:createIndex
:createSequence
:createSynonym
:createTable
:createView
:disableCheckConstraint
:disableTrigger
:dropNotNullConstraint
:enableCheckConstraint
:enableTrigger
:renameColumn
:renameSequence
:renameTable
:renameTrigger
:renameView
;; assumes all custom changes use the `def-migration` or `define-reversible-migration` in
;; [[metabase.app-db.custom-migrations]]
:customChange})
(defn- major-version
"Returns major version from id string, e.g. 44 from \"v44.00-034\""
[id-str]
(when (string? id-str)
(some-> (re-find #"\d+" id-str) Integer/parseInt)))
(defn- rollback-present-when-required?
"Ensures rollback key is present when change type doesn't support auto rollback"
[{:keys [id changes] :as change-set}]
(or
(int? id)
(< (major-version id) 45)
(some change-types-supporting-rollback (mapcat keys changes))
(contains? change-set :rollback)))
(defn- disallow-delete-cascade-with-add-column
"Returns false if addColumn changeSet uses deleteCascade. See Metabase issue #14321"
[{:keys [changes]}]
@@ -135,7 +80,6 @@
(s/def ::change-set
(s/and
rollback-present-when-required?
disallow-delete-cascade-with-add-column
(s/keys :req-un [::id ::author ::changes ::comment]
:opt-un [::preConditions])))

218
bin/lint-migrations-file/src/lint_migrations_file.cljc
View File

@@ -10,7 +10,8 @@
[clojure.spec.alpha :as s]
[clojure.string :as str])
(:import
[clojure.lang ExceptionInfo]))
(clojure.lang ExceptionInfo)
(java.lang Integer)))
#_:clj-kondo/ignore
(set! *warn-on-reflection* true)
@@ -52,26 +53,58 @@
(when (seq duplicates)
(throw (validation-error "Change set IDs are not distinct." {:duplicates duplicates})))))
(defn- require-change-set-ids-in-order [change-log]
(let [ids (change-set-ids change-log)
out-of-order-ids (->> ids
(partition 2 1)
(filter (fn [[id1 id2]]
(pos? (compare id1 id2)))))]
(defn- directory-based-migration-file?
"Returns true if the file is a directory-based migration file (e.g., `060/20260905_mq_indexes.yaml`)."
[file]
(boolean (re-matches #".*\d{3}/\d{8}_[a-z0-9_]+\.yaml$" (str file))))
(when (seq out-of-order-ids)
(throw (validation-error "Change set IDs are not in order"
{:out-of-order-ids out-of-order-ids})))))
(defn- file-version
"Extracts the migration version number from a file.
For per-release files like `059_update_migrations.yaml`, parses from the filename.
For directory-based files like `060/20260106_125531.yaml`, parses from the parent directory name."
[^java.io.File file]
(if (directory-based-migration-file? file)
(parse-long (re-find #"\d+" (.getName (.getParentFile file))))
(parse-long (re-find #"\d+" (.getName file)))))
(defn- changeset-version+id
"Returns [version local-id] for a changeset.
Parses from the ID itself, which should be of the form 'v{version}.{local-id}'."
[_file changeset-id]
(let [id-str (str changeset-id)]
(when-let [[_ version local-id] (re-matches #"^v(\d+)\.(.+)$" id-str)]
[(parse-long version) local-id])))
(defn- changeset-at-or-after?
"Returns true if the changeset (identified by file context and changeset ID) is at or after
the given [threshold-version threshold-id]."
[file changeset-id threshold-version threshold-id]
(when-let [[v local-id] (changeset-version+id file changeset-id)]
(or (> v threshold-version)
(and (= v threshold-version)
(not (neg? (compare local-id threshold-id)))))))
(defn- require-change-set-ids-in-order [change-log file]
(when-not (directory-based-migration-file? file)
(let [ids (change-set-ids change-log)
out-of-order-ids (->> ids
(partition 2 1)
(filter (fn [[id1 id2]]
(pos? (compare id1 id2)))))]
(when (seq out-of-order-ids)
(throw (validation-error "Change set IDs are not in order"
{:out-of-order-ids out-of-order-ids}))))))
(defn- require-change-set-ids-in-correct-file [change-log file]
(let [file-version (parse-long (re-find #"\d+" (.getName file)))
(let [fv (file-version file)
ids (change-set-ids change-log)
wrong-file-ids (->> ids
(filter (fn [id]
(let [id-version (parse-long (re-find #"\d+" id))]
(if (= file-version 1)
(> id-version 55)
(not= file-version id-version))))))]
wrong-file-ids
(->> ids
(filter (fn [id]
(let [id-version (parse-long (re-find #"\d+" id))]
(if (= fv 1)
(> id-version 55)
(not= fv id-version))))))]
(when (seq wrong-file-ids)
(throw (validation-error "Change set IDs are in the wrong file"
{:wrong-file-ids wrong-file-ids})))))
@@ -101,18 +134,19 @@
(some #(check-change-use-types? target-types %) (get-in change-set [:changeSet :changes])))
(defn- require-no-types-in-change-log!
"Returns true if none of the changes in the change-log contain usage of any type specified in `target-types`.
`id-filter-fn` is a function that takes an ID and return true if the changeset should be checked."
"Throws if any changeset at or after [version id] uses a type in target-types.
When version and id are not provided, checks all changesets."
([target-types change-log]
(require-no-types-in-change-log! target-types change-log (constantly true)))
([target-types change-log id-filter-fn]
(require-no-types-in-change-log! target-types change-log nil nil nil))
([target-types change-log file version id]
{:pre [(set? target-types)]}
(when-let [using-types? (->> change-log
(filter (fn [change-set]
(let [id (get-in change-set [:changeSet :id])]
(and (string? id)
(id-filter-fn id)))))
(let [cs-id (get-in change-set [:changeSet :id])]
(and cs-id
(if version
(changeset-at-or-after? file cs-id version id)
true)))))
(filter #(check-change-set-use-types? target-types %))
(map #(get-in % [:changeSet :id]))
seq)]
@@ -132,17 +166,46 @@
(defn require-no-bare-boolean-types
"Ensures that no \"boolean\" type columns are added in changesets with id later than v49.00-032. From that point on,
\"${boolean.type}\" should be used instead, so that we can consistently use `BIT(1)` for Boolean columns on MySQL."
[change-log]
(require-no-types-in-change-log! #{"boolean"} change-log #(pos? (compare % "v49.00-032"))))
[change-log file]
(require-no-types-in-change-log! #{"boolean"} change-log file 49 "00-032"))
(defn require-no-datetime-type
"Ensures that no \"datetime\" or \"timestamp without time zone\".
From that point on, \"${timestamp_type}\" should be used instead, so that all of our time related columns are tz-aware."
[change-log]
[change-log file]
(require-no-types-in-change-log!
#{"datetime" "timestamp" "timestamp without time zone"}
change-log
#(pos? (compare % "v49.00-000"))))
change-log file 49 "00-000"))
(def ^:private directory-based-id-format-re
"For directory-based migration files (v60+), IDs must be prefixed with the correct version.
E.g., `v60.my-migration-name` or `v60.1`"
#"^v\d{2,}\..+$")
(defn- require-change-set-ids-match-file-format
"Enforces that changeset IDs use the correct format for the file type:
- Directory-based files (v60+): IDs must be prefixed with v{version}. (e.g., v60.anything)
- 001_update_migrations.yaml: any string ID is allowed (legacy file)
- Other per-release files: IDs must match the timestamp format"
[change-log file]
(cond
(directory-based-migration-file? file)
(let [ids (change-set-ids change-log)
bad-ids (remove #(re-matches directory-based-id-format-re %) ids)]
(when (seq bad-ids)
(throw (validation-error
(format "Directory-based migration file contains IDs without v{version}. prefix: %s"
(str/join ", " bad-ids))
{:invalid-ids (vec bad-ids)}))))
(not= (file-version file) 1)
(let [ids (change-set-ids change-log)
bad-ids (remove #(re-matches change-set.strict/id-timestamp-format-re %) ids)]
(when (seq bad-ids)
(throw (validation-error
(format "Per-release migration file contains non-timestamp ID formats: %s"
(str/join ", " bad-ids))
{:invalid-ids (vec bad-ids)}))))))
(defn require-primary-key-exists-has-table-name
"Ensures that all primaryKeyExists preconditions specify a tableName."
@@ -165,25 +228,82 @@
(throw (validation-error "Expected exactly one key." {:keys keys})))
(first keys)))
(defn- major-version
"Returns major version from id string, e.g. 44 from \"v44.00-034\".
For directory-based migrations (file path matches `NNN/`), extracts
version from the file path. Otherwise parses from the id string."
[id-str file]
(if-let [[_ file-version] (when file (re-find #"(\d{3})[/\\]" (str file)))]
(Integer/parseInt file-version)
(when (string? id-str)
(some-> (re-find #"\d+" id-str) Integer/parseInt))))
(def change-types-supporting-rollback
"This set was generated with a little grep and awk from the docs here:
https://docs.liquibase.com/workflows/liquibase-community/liquibase-auto-rollback.html
If a new change type is introduced that supports automatic rollback, it should be added
to this set."
#{:addCheckConstraint
:addColumn
:addDefaultValue
:addForeignKeyConstraint
:addLookupTable
:addNotNullConstraint
:addPrimaryKey
:addUniqueConstraint
:createIndex
:createSequence
:createSynonym
:createTable
:createView
:disableCheckConstraint
:disableTrigger
:dropNotNullConstraint
:enableCheckConstraint
:enableTrigger
:renameColumn
:renameSequence
:renameTable
:renameTrigger
:renameView
;; assumes all custom changes use the `def-migration` or `define-reversible-migration` in
;; [[metabase.app-db.custom-migrations]]
:customChange})
(defn- rollback-present-when-required?
"Ensures rollback key is present when change type doesn't support auto rollback.
`file` is used to extract version for directory-based migrations."
[{:keys [id changes] :as change-set} file]
(or
(let [v (major-version (str id) file)]
(and v (< v 45)))
(some change-types-supporting-rollback (mapcat keys changes))
(contains? change-set :rollback)))
(defn- validate-database-change-log [change-log file]
(when (string? change-log)
(throw (validation-error "Expected `:databaseChangeLog` to be a map, not a string.")))
(require-distinct-change-set-ids change-log)
(require-change-set-ids-in-correct-file change-log file)
(require-change-set-ids-in-order change-log)
(require-change-set-ids-in-order change-log file)
(require-change-set-ids-match-file-format change-log file)
(require-no-bare-blob-or-text-types change-log)
(require-no-bare-boolean-types change-log)
(require-no-datetime-type change-log)
(require-no-bare-boolean-types change-log file)
(require-no-datetime-type change-log file)
(require-primary-key-exists-has-table-name change-log)
(let [{:keys [changeSet]
:as all} (group-by only-key change-log)]
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet})
(throw (validation-error "Expected exactly one of :property, :objectQuotingStrategy, or :changeSet."
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet :logicalFilePath})
(throw (validation-error "Expected exactly one of :property, :objectQuotingStrategy, :logicalFilePath, or :changeSet."
{:keys (keys all)})))
(doseq [{:keys [changeSet]} changeSet
:when (not (s/valid? ::changeSet changeSet))]
(throw (validation-error "Invalid change set." (s/explain-data ::changeSet changeSet))))))
(doseq [{:keys [changeSet]} changeSet]
(when-not (s/valid? ::changeSet changeSet)
(throw (validation-error "Invalid change set." (s/explain-data ::changeSet changeSet))))
(when-not (rollback-present-when-required? changeSet file)
(throw (validation-error "Rollback is required but not present."
{:id (:id changeSet)}))))))
(defn- validate-migrations [migrations file]
(require-database-change-log! migrations)
@@ -195,7 +315,14 @@
(let [dir-str #?(:bb "resources/migrations" :clj "../../resources/migrations")
dir (io/file dir-str)]
(->> (file-seq dir)
(filter #(and (.isFile %) (str/ends-with? (.getName %) "_update_migrations.yaml"))))))
(filter (fn [^java.io.File f]
(and (.isFile f)
(or
;; Per-release files: 059_update_migrations.yaml
(str/ends-with? (.getName f) "_update_migrations.yaml")
;; Directory-based files: 060/20260106_125531.yaml
(directory-based-migration-file? f)))))
(sort-by str))))
(defn- migrations [file]
(assert (.exists file) (format "%s does not exist" file))
@@ -207,13 +334,22 @@
(fix-vals (yaml/parse-string
#_:clj-kondo/ignore (slurp file)))))
(defn- display-name
"Returns a human-readable name for a migration file.
For per-release files, returns just the filename.
For directory-based files, returns `parent/filename`."
[^java.io.File file]
(if (directory-based-migration-file? file)
(str (.getName (.getParentFile file)) "/" (.getName file))
(.getName file)))
(defn- validate-all []
(doseq [file (migration-files)]
(println "Validating" (.getName file) "...")
(println "Validating" (display-name file) "...")
(try
(validate-migrations (migrations file) file)
(catch ExceptionInfo e
(throw (ex-info (.getMessage e) (assoc (ex-data e) :file (.getName file)) e))))))
(throw (ex-info (.getMessage e) (assoc (ex-data e) :file (display-name file)) e))))))
(defn -main
"Entry point for Clojure CLI task `lint-migrations-file`. Run it with

26
bin/lint-migrations-file/test/lint_migrations_file_test.clj
View File

@@ -200,17 +200,19 @@
(validate-id "v49.2024-01-01T10:30:00"))))
(testing "invalid date components should throw an error"
(are [msg id]
(thrown-with-msg?
clojure.lang.ExceptionInfo
#"Invalid change set\."
(validate-id "v49.2024-30-01T10:30:00")
msg)
"invalid month" "v49.2024-13-01T10:30:00"
"invalid day" "v49.2024-01-32T10:30:00"
"invalid hour" "v49.2024-01-01T25:30:00"
"invalid minute" "v49.2024-01-01T10:60:00"
"invalid second" "v49.2024-01-01T10:30:60"))))
(let [validate-id-strict (fn [id]
(validate-file (io/file "049_update_migrations.yaml")
(mock-change-set :id id)))]
(are [msg id]
(thrown-with-msg?
clojure.lang.ExceptionInfo
#"non-timestamp ID formats"
(validate-id-strict id))
"invalid month" "v49.2024-13-01T10:30:00"
"invalid day" "v49.2024-01-32T10:30:00"
"invalid hour" "v49.2024-01-01T25:30:00"
"invalid minute" "v49.2024-01-01T10:60:00"
"invalid second" "v49.2024-01-01T10:30:60")))))
(deftest ^:parallel validate-id-in-file-test
(letfn [(validate-id [id file]
@@ -285,7 +287,7 @@
(testing "missing rollback key fails"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Invalid change set\."
#"Rollback is required but not present\."
(validate (update (mock-change-set :id "v49.2024-01-01T10:30:00" :changes [{:sql {:sql "select 1"}}])
:changeSet dissoc :rollback)))))
(testing "nil rollback is allowed"

17
mage/src/mage/merge_yaml_migrations.clj
View File

@@ -17,11 +17,11 @@
(println (.getMessage e)))
(u/exit 2))))
(def ^:private FOOTER-PREFIX
(def ^:private footer-prefix
"# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<")
(defn- extract-footer [content]
(let [idx (some-> content (str/index-of FOOTER-PREFIX))]
(let [idx (some-> content (str/index-of footer-prefix))]
(if (and idx (pos? idx))
(subs content idx)
"")))
@@ -182,6 +182,10 @@
:conflicts (vec (keep #(when (= (:source %) :conflict) (:id %)) sorted-merged))
:cnt (count sorted-merged)}))
;;
;; Custom merge driver for per-release *_update_migrations.yaml files.
;; Directory-based migrations (e.g. 060/*.yaml) have one file per migration
;; and should use git's default merge strategy.
;;
;; Usage (called by git):
;; merge-yaml-migrations %O %A %B %L %P
@@ -196,7 +200,9 @@
;; 1 = conflicts detected (conflict markers added to file)
;; >1 = merge failed
(defn -main [{:keys [arguments]}]
(defn -main
"Main function"
[{:keys [arguments]}]
(when (< (count arguments) 3)
(binding [*out* *err*]
(println "Usage: merge-yaml-migrations <base> <ours> <theirs> [conflict-marker-size] [file-path]")
@@ -209,6 +215,11 @@
p
(str u/project-root-directory "/" p)))
[base ours theirs marker-size filepath] arguments
_ (when (and filepath (not (str/ends-with? filepath "_update_migrations.yaml")))
(binding [*out* *err*]
(println "merge-yaml-migrations is only for per-release *_update_migrations.yaml files, not:" filepath)
(println "Falling back to git's default merge."))
(u/exit 1))
base (resolve-path base)
ours (resolve-path ours)
theirs (resolve-path theirs)

75
mage/src/mage/new_migration.clj Normal file
View File

@@ -0,0 +1,75 @@
(ns mage.new-migration
"Mage task for creating new directory-based migration files (v60+).
Each migration file is a YAML file inside a version directory,
e.g., `resources/migrations/060/20260905_mq_indexes.yaml`."
(:require
[clojure.java.io :as io]
[mage.util :as u])
(:import
(java.security SecureRandom)
(java.time ZoneOffset ZonedDateTime)
(java.time.format DateTimeFormatter)))
(set! *warn-on-reflection* true)
(def ^:private valid-description-re
"Descriptions must be lowercase alphanumeric or underscores."
#"[a-z0-9][a-z0-9_]*[a-z0-9]|[a-z0-9]")
(defn- git-user-name []
(try
(u/sh "git" "config" "user.name")
(catch Exception _
"author")))
(defn- utc-date []
(.format (DateTimeFormatter/ofPattern "yyyyMMdd")
(ZonedDateTime/now ZoneOffset/UTC)))
(def ^:private alphanumeric "abcdefghijklmnopqrstuvwxyz0123456789")
(defn- random-id
"Generates a random 6-character lowercase alphanumeric string."
[]
(let [rng (SecureRandom.)]
(apply str (repeatedly 6 #(nth alphanumeric (.nextInt rng (count alphanumeric)))))))
(defn- migration-template [version author]
(let [id (random-id)]
(str "## TODO: quick description of what feature/bug this migration is for\n"
"\n"
"databaseChangeLog:\n"
" - changeSet:\n"
" id: v" version "." id "\n"
" author: " author "\n"
" comment: TODO\n"
" changes:\n"
" - TODO\n")))
(defn new-migration
"Creates a new directory-based migration file for the given version number.
Usage: `./bin/mage new-migration 60 mq_indexes`"
[{:keys [arguments]}]
(let [version (first arguments)
description (second arguments)
_ (when-not (and version description)
(binding [*out* *err*]
(println "Usage: mage new-migration <version> <description>")
(println " e.g., mage new-migration 60 mq_indexes"))
(u/exit 1))
_ (when-not (re-matches valid-description-re description)
(binding [*out* *err*]
(println (str "Error: description must be lowercase alphanumeric or underscores, got: \"" description "\""))
(println " e.g., mq_indexes, add_user_table, fix_fk"))
(u/exit 1))
dir-name (format "%03d" version)
dir (io/file u/project-root-directory "resources" "migrations" dir-name)
date (utc-date)
filename (str date "_" description ".yaml")
file (io/file dir filename)
author (git-user-name)]
(.mkdirs dir)
(spit file (migration-template version author))
(println "Created" (str "resources/migrations/" dir-name "/" filename))))

4
resources/liquibase.yaml
View File

@@ -47,5 +47,5 @@ databaseChangeLog:
- includeAll:
path: migrations
endsWithFilter: "_update_migrations.yaml"
maxDepth: 1
endsWithFilter: ".yaml"
maxDepth: 2

2
resources/liquibase_legacy.yaml
View File

@@ -1,5 +1,5 @@
databaseChangeLog:
- include:
file: "migrations/000_legacy_migrations.yaml"
file: "liquibase_legacy_migrations.yaml"
- include:
file: "liquibase.yaml"

1
resources/migrations/000_legacy_migrations.yaml → resources/liquibase_legacy_migrations.yaml
View File

@@ -5,6 +5,7 @@
# In order to keep instances running metabase < 45 able to upgrade to the latest, we need to keep this file here.
databaseChangeLog:
- logicalFilePath: migrations/000_legacy_migrations.yaml
- property:
name: timestamp_type
value: timestamp with time zone

79
resources/migrations/060/20260213_dimension_columns.yaml Normal file
View File

@@ -0,0 +1,79 @@
## TODO: quick description of what feature/bug this migration is for
databaseChangeLog:
- logicalFilePath: migrations/060_update_migrations.yaml
- changeSet:
id: v60.2026-02-13T21:43:00
author: epaget
comment: Add dimensions column to measure table
changes:
- addColumn:
tableName: measure
columns:
- column:
name: dimensions
type: ${text.type}
remarks: JSON array of persisted dimension definitions (id, display-name, types)
constraints:
nullable: true
rollback:
- dropColumn:
tableName: measure
columnName: dimensions
- changeSet:
id: v60.2026-02-13T21:43:01
author: epaget
comment: Add dimension_mappings column to measure table
changes:
- addColumn:
tableName: measure
columns:
- column:
name: dimension_mappings
type: ${text.type}
remarks: JSON array of dimension-to-field mappings (dimension-id, target, table-id)
constraints:
nullable: true
rollback:
- dropColumn:
tableName: measure
columnName: dimension_mappings
- changeSet:
id: v60.2026-02-13T21:43:02
author: epaget
comment: Add dimensions column to report_card table for metrics
changes:
- addColumn:
tableName: report_card
columns:
- column:
name: dimensions
type: ${text.type}
remarks: JSON array of persisted dimension definitions (id, display-name, types)
constraints:
nullable: true
rollback:
- dropColumn:
tableName: report_card
columnName: dimensions
- changeSet:
id: v60.2026-02-13T21:43:03
author: epaget
comment: Add dimension_mappings column to report_card table for metrics
changes:
- addColumn:
tableName: report_card
columns:
- column:
name: dimension_mappings
type: ${text.type}
remarks: JSON array of dimension-to-field mappings (dimension-id, target, table-id)
constraints:
nullable: true
rollback:
- dropColumn:
tableName: report_card
columnName: dimension_mappings

122
resources/migrations/060/20260310_source_replacement_run.yaml Normal file
View File

@@ -0,0 +1,122 @@
## TODO: quick description of what feature/bug this migration is for
databaseChangeLog:
- logicalFilePath: migrations/060_update_migrations.yaml
- changeSet:
id: v60.2026-03-06T12:00:00
author: bcm
comment: Create source_replacement_run table to track async source replacement executions
changes:
- createTable:
tableName: source_replacement_run
remarks: Table to track source replacement executions and their status
columns:
- column:
name: id
type: int
remarks: Unique ID
autoIncrement: true
constraints:
primaryKey: true
nullable: false
- column:
name: source_entity_type
type: varchar(50)
remarks: "Type of the source entity (card or table)"
constraints:
nullable: false
- column:
name: source_entity_id
type: int
remarks: "ID of the source entity"
constraints:
nullable: false
- column:
name: target_entity_type
type: varchar(50)
remarks: "Type of the target entity (card or table)"
constraints:
nullable: false
- column:
name: target_entity_id
type: int
remarks: "ID of the target entity"
constraints:
nullable: false
- column:
name: status
type: varchar(50)
remarks: "Current status (started, succeeded, failed, canceled, timeout)"
constraints:
nullable: false
- column:
name: is_active
type: ${boolean.type}
remarks: "True only for currently running replacements, null for completed"
constraints:
nullable: true
- column:
name: progress
type: float
remarks: "Progress from 0.0 to 1.0"
constraints:
nullable: true
- column:
name: message
type: ${text.type}
remarks: "Human-readable message; may contain error info"
constraints:
nullable: true
- column:
name: user_id
type: int
remarks: "User who initiated the replacement"
constraints:
nullable: true
- column:
name: start_time
type: ${timestamp_type}
remarks: "When the replacement started"
defaultValueComputed: current_timestamp
constraints:
nullable: false
- column:
name: end_time
type: ${timestamp_type}
remarks: "When the replacement completed (null if still running)"
constraints:
nullable: true
- changeSet:
id: v60.2026-03-06T12:00:01
author: bcm
comment: Enforce at most one active source replacement run cluster-wide
changes:
- addUniqueConstraint:
tableName: source_replacement_run
constraintName: idx_unique_active_source_replacement_run
columnNames: is_active
- changeSet:
id: v60.2026-03-06T12:00:02
author: bcm
comment: Add index on source_replacement_run.user_id
changes:
- createIndex:
tableName: source_replacement_run
indexName: idx_source_replacement_run_user_id
columns:
- column:
name: user_id
- changeSet:
id: v60.2026-03-06T12:00:03
author: bcm
comment: Add foreign key on source_replacement_run(user_id)
changes:
- addForeignKeyConstraint:
baseTableName: source_replacement_run
baseColumnNames: user_id
referencedTableName: core_user
referencedColumnNames: id
constraintName: fk_source_replacement_run_user_id

247
resources/migrations/060_update_migrations.yaml
View File

@@ -1,247 +0,0 @@
databaseChangeLog:
- objectQuotingStrategy: QUOTE_ALL_OBJECTS
- changeSet:
id: v60.2026-02-13T21:43:00
author: epaget
comment: Add dimensions column to measure table
changes:
- addColumn:
tableName: measure
columns:
- column:
name: dimensions
type: ${text.type}
remarks: JSON array of persisted dimension definitions (id, display-name, types)
constraints:
nullable: true
rollback:
- dropColumn:
tableName: measure
columnName: dimensions
- changeSet:
id: v60.2026-02-13T21:43:01
author: epaget
comment: Add dimension_mappings column to measure table
changes:
- addColumn:
tableName: measure
columns:
- column:
name: dimension_mappings
type: ${text.type}
remarks: JSON array of dimension-to-field mappings (dimension-id, target, table-id)
constraints:
nullable: true
rollback:
- dropColumn:
tableName: measure
columnName: dimension_mappings
- changeSet:
id: v60.2026-02-13T21:43:02
author: epaget
comment: Add dimensions column to report_card table for metrics
changes:
- addColumn:
tableName: report_card
columns:
- column:
name: dimensions
type: ${text.type}
remarks: JSON array of persisted dimension definitions (id, display-name, types)
constraints:
nullable: true
rollback:
- dropColumn:
tableName: report_card
columnName: dimensions
- changeSet:
id: v60.2026-02-13T21:43:03
author: epaget
comment: Add dimension_mappings column to report_card table for metrics
changes:
- addColumn:
tableName: report_card
columns:
- column:
name: dimension_mappings
type: ${text.type}
remarks: JSON array of dimension-to-field mappings (dimension-id, target, table-id)
constraints:
nullable: true
rollback:
- dropColumn:
tableName: report_card
columnName: dimension_mappings
- changeSet:
id: v60.2026-03-06T12:00:00
author: bcm
comment: Create source_replacement_run table to track async source replacement executions
changes:
- createTable:
tableName: source_replacement_run
remarks: Table to track source replacement executions and their status
columns:
- column:
name: id
type: int
remarks: Unique ID
autoIncrement: true
constraints:
primaryKey: true
nullable: false
- column:
name: source_entity_type
type: varchar(50)
remarks: "Type of the source entity (card or table)"
constraints:
nullable: false
- column:
name: source_entity_id
type: int
remarks: "ID of the source entity"
constraints:
nullable: false
- column:
name: target_entity_type
type: varchar(50)
remarks: "Type of the target entity (card or table)"
constraints:
nullable: false
- column:
name: target_entity_id
type: int
remarks: "ID of the target entity"
constraints:
nullable: false
- column:
name: status
type: varchar(50)
remarks: "Current status (started, succeeded, failed, canceled, timeout)"
constraints:
nullable: false
- column:
name: is_active
type: ${boolean.type}
remarks: "True only for currently running replacements, null for completed"
constraints:
nullable: true
- column:
name: progress
type: float
remarks: "Progress from 0.0 to 1.0"
constraints:
nullable: true
- column:
name: message
type: ${text.type}
remarks: "Human-readable message; may contain error info"
constraints:
nullable: true
- column:
name: user_id
type: int
remarks: "User who initiated the replacement"
constraints:
nullable: true
- column:
name: start_time
type: ${timestamp_type}
remarks: "When the replacement started"
defaultValueComputed: current_timestamp
constraints:
nullable: false
- column:
name: end_time
type: ${timestamp_type}
remarks: "When the replacement completed (null if still running)"
constraints:
nullable: true
- changeSet:
id: v60.2026-03-06T12:00:01
author: bcm
comment: Enforce at most one active source replacement run cluster-wide
changes:
- addUniqueConstraint:
tableName: source_replacement_run
constraintName: idx_unique_active_source_replacement_run
columnNames: is_active
- changeSet:
id: v60.2026-03-06T12:00:02
author: bcm
comment: Add index on source_replacement_run.user_id
changes:
- createIndex:
tableName: source_replacement_run
indexName: idx_source_replacement_run_user_id
columns:
- column:
name: user_id
- changeSet:
id: v60.2026-03-06T12:00:03
author: bcm
comment: Add foreign key on source_replacement_run(user_id)
changes:
- addForeignKeyConstraint:
baseTableName: source_replacement_run
baseColumnNames: user_id
referencedTableName: core_user
referencedColumnNames: id
constraintName: fk_source_replacement_run_user_id
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
#
# ADVICE:
#
# 1) Think through some of these points when writing a migration, learn from our past mistakes:
# - Do you really need a migration? Could the feature work without it? Prefer not to if possible.
# Data in the wild can be vastly different from what you expect, and it's hard to get right.
# - Make sure your migration is backward compatible: it might not be possible to add a constraint back
# if you drop it in a migration.
# - Postgres, MySQL and H2 have their differences. Make sure your migration works for all.
# - Database encryption is a major issue:
# - Fields like `metabase_database.details` or `setting.value` can be encrypted, so you need to decrypt them in
# your migration. See #42617, #44048.
# - Database could be partially encrypted, see https://www.notion.so/72575933ef2a446bafd16909e05a7387
# - Custom migrations:
# - Prefer SQL migrations when possible.
# - Never use application code: it can change and *will* break your migration.
# - Do not use Toucan models - refer table names directly.
# - If it's a big change, consider using Quartz, see #42279
# - More in `metabase.app_db.custom_migrations` namespace doc.
# - Never delete a migration: users won't be able to downgrade. If you really need to, see #44908
# - Migration id (`vXX.<date>`) must match its earliest released version:
# - Do not backport `v51....` to version 50, Metabase will try to downgrade it.
# - Instead, write a migration with an oldest target you plan to backport to in mind.
#
# 2) Migrations should go in the ###_update_migrations.yaml file for the target version.
#
# 3) Run mage lint-migrations to run core.spec checks against any changes you make here. Liquibase is pretty
# forgiving and won't complain if you accidentally mix up things like deleteCascade and onDelete: CASCADE. CI runs
# this check but it's nicer to know now instead of waiting for CI.
#
# 3) Migration IDs should follow the format
#
# vMM.TIMESTAMP
#
# Where
#
# M = major version
# TIMESTAMP = the current timestamp with format `yyyy-MM-dd'T'HH:mm:ss`
# To get this timestamp, evaluate this in your REPL: `(dev/migration-timestamp)`
#
# E.g: You're adding a new migration for version 49, And it's 10:30:00AM on April 1, 2023 (UTC),
# your migration id should be: `v49.2023-04-01T10:30:00`.
#
# PLEASE KEEP THIS MESSAGE AT THE BOTTOM OF THIS FILE!!!!! Add new migrations above the message.
#
########################################################################################################################

58
resources/migrations/999_update_migrations.yaml.template
View File

@@ -1,58 +0,0 @@
--------
THIS IS THE TEMPLATE MIGRATION FILE FOR NEW MAJOR VERSION OF METABASE
On each new major version, copy this file to `resources/migrations/###_update_migrations.yaml` where `###` is the major metabase version.
The file can be empty until migrations need to be added.
--------
databaseChangeLog:
- objectQuotingStrategy: QUOTE_ALL_OBJECTS
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
#
# ADVICE:
#
# 1) Think through some of these points when writing a migration, learn from our past mistakes:
# - Do you really need a migration? Could the feature work without it? Prefer not to if possible.
# Data in the wild can be vastly different from what you expect, and it's hard to get right.
# - Make sure your migration is backward compatible: it might not be possible to add a constraint back
# if you drop it in a migration.
# - Postgres, MySQL and H2 have their differences. Make sure your migration works for all.
# - Database encryption is a major issue:
# - Fields like `metabase_database.details` or `setting.value` can be encrypted, so you need to decrypt them in
# your migration. See #42617, #44048.
# - Database could be partially encrypted, see https://www.notion.so/72575933ef2a446bafd16909e05a7387
# - Custom migrations:
# - Prefer SQL migrations when possible.
# - Never use application code: it can change and *will* break your migration.
# - Do not use Toucan models - refer table names directly.
# - If it's a big change, consider using Quartz, see #42279
# - More in `metabase.app_db.custom_migrations` namespace doc.
# - Never delete a migration: users won't be able to downgrade. If you really need to, see #44908
# - Migration id (`vXX.<date>`) must match its earliest released version:
# - Do not backport `v51....` to version 50, Metabase will try to downgrade it.
# - Instead, write a migration with an oldest target you plan to backport to in mind.
#
# 2) Migrations should go in the ###_update_migrations.yaml file for the target version.
#
# 3) Run mage lint-migrations-file to run core.spec checks against any changes you make here. Liquibase is pretty
# forgiving and won't complain if you accidentally mix up things like deleteCascade and onDelete: CASCADE. CI runs
# this check but it's nicer to know now instead of waiting for CI.
#
# 3) Migration IDs should follow the format
#
# vMM.TIMESTAMP
#
# Where
#
# M = major version
# TIMESTAMP = the current timestamp with format `yyyy-MM-dd'T'HH:mm:ss`
# To get this timestamp, evaluate this in your REPL: `(dev/migration-timestamp)`
#
# E.g: You're adding a new migration for version 49, And it's 10:30:00AM on April 1, 2023 (UTC),
# your migration id should be: `v49.2023-04-01T10:30:00`.
#
# PLEASE KEEP THIS MESSAGE AT THE BOTTOM OF THIS FILE!!!!! Add new migrations above the message.
#
########################################################################################################################

15
src/metabase/app_db/liquibase.clj
View File

@@ -29,7 +29,7 @@
(liquibase.changelog.filter AlreadyRanChangeSetFilter ChangeSetFilter ChangeSetFilterResult DbmsChangeSetFilter IgnoreChangeSetFilter)
(liquibase.changelog.visitor AbstractChangeExecListener ChangeExecListener UpdateVisitor)
(liquibase.command.core AbstractRollbackCommandStep)
(liquibase.database Database DatabaseFactory)
(liquibase.database Database DatabaseFactory ObjectQuotingStrategy)
(liquibase.database.jvm JdbcConnection)
(liquibase.exception LockException)
(liquibase.lockservice LockService LockServiceFactory)
@@ -122,10 +122,11 @@
(.findCorrectDatabaseImplementation (DatabaseFactory/getInstance) liquibase-conn)))
(defn- liquibase ^Liquibase [^Connection conn ^Database database]
(Liquibase.
^String (decide-liquibase-file conn database)
(ClassLoaderResourceAccessor. (classloader/the-classloader))
database))
(u/prog1 (Liquibase.
^String (decide-liquibase-file conn database)
(ClassLoaderResourceAccessor. (classloader/the-classloader))
database)
(.setObjectQuotingStrategy (.getDatabaseChangeLog <>) ObjectQuotingStrategy/QUOTE_ALL_OBJECTS)))
(mu/defn do-with-liquibase
"Impl for [[with-liquibase-macro]]."
@@ -456,6 +457,10 @@
liquibase was checking the jar signature for each file. This function is required to correct the liquibase tables to
reflect that these migrations were grouped into 2 files.
NOTE: we are going back to more granular changelog files in v60, but should not as many as before.
If we do end up seeing a performance issue similar to what caused a need for this file, we can do a similar
(but different) consolidation function for the v60+ changelogs.
See https://github.com/metabase/metabase/issues/3715
Also see https://github.com/metabase/metabase/pull/34400"
[conn :- (ms/InstanceOfClass Connection)

5
test/metabase/app_db/liquibase_test.clj
View File

@@ -63,7 +63,7 @@
(liquibase/consolidate-liquibase-changesets! conn liquibase)
(testing "makes sure the change log filename are correctly set"
(is (= (set (mdb.test-util/liquibase-file->included-ids "migrations/000_legacy_migrations.yaml" driver/*driver* conn))
(is (= (set (mdb.test-util/liquibase-file->included-ids "liquibase_legacy_migrations.yaml" driver/*driver* conn))
(t2/select-fn-set :id table-name :filename "migrations/000_legacy_migrations.yaml")))
(is (= (set (mdb.test-util/liquibase-file->included-ids "migrations/001_update_migrations.yaml" driver/*driver* conn))
@@ -156,7 +156,8 @@
(deftest extract-numbers-special-case-test
(testing "when specific migration verison is passed reports different major version"
(is (= 55 (first (#'liquibase/extract-numbers "v56.2025-06-05T16:48:48"))))
(is (= 55 (first (#'liquibase/extract-numbers "v56.2025-05-19T16:48:48"))))))
(is (= 55 (first (#'liquibase/extract-numbers "v56.2025-05-19T16:48:48"))))
(is (= 60 (first (#'liquibase/extract-numbers "v60.ghdf99efd"))))))
(deftest rollback-major-version
(mt/test-drivers #{:h2 :mysql :rollback}

4
test/metabase/app_db/setup_test.clj
View File

@@ -176,14 +176,14 @@
;; the latest changeSet in `000_legacy_migrations.yaml` is `v44.00-044`. We can simulate a downgrade to that
;; version by telling Liquibase that's the migrations file.
(with-redefs [liquibase/decide-liquibase-file (fn [& _args] "migrations/000_legacy_migrations.yaml")]
(with-redefs [liquibase/decide-liquibase-file (fn [& _args] "liquibase_legacy_migrations.yaml")]
(is (thrown-with-msg?
Exception #"You must run `java --add-opens java.base/java.nio=ALL-UNNAMED -jar metabase.jar migrate down` from version 45."
(#'mdb.setup/error-if-downgrade-required! (mdb.connection/data-source)))))
;; check that the error correctly reports the version to run `downgrade` from
(update-to-changelog-id "v46.00-001" conn)
(with-redefs [liquibase/decide-liquibase-file (fn [& _args] "migrations/000_legacy_migrations.yaml")]
(with-redefs [liquibase/decide-liquibase-file (fn [& _args] "liquibase_legacy_migrations.yaml")]
(is (thrown-with-msg?
Exception #"You must run `java --add-opens java.base/java.nio=ALL-UNNAMED -jar metabase.jar migrate down` from version 46."
(#'mdb.setup/error-if-downgrade-required! (mdb.connection/data-source))))))))

19
test/metabase/app_db/test_util.clj
View File

@@ -92,10 +92,23 @@
"Returns a list of existing migration files."
[include-legacy?]
(into (if include-legacy?
["migrations/000_legacy_migrations.yaml" "migrations/001_update_migrations.yaml"]
["liquibase_legacy_migrations.yaml" "migrations/001_update_migrations.yaml"]
["migrations/001_update_migrations.yaml"])
(filter io/resource (for [n (range 56 100)]
(format "migrations/%03d_update_migrations.yaml" n)))))
(concat
;; Per-release migration files (v56-v59 pattern)
(filter io/resource (for [n (range 56 100)]
(format "migrations/%03d_update_migrations.yaml" n)))
;; Directory-based migration files (v60+ pattern)
(let [migrations-dir (io/resource "migrations")]
(when migrations-dir
(->> (io/file migrations-dir)
file-seq
(filter (fn [^java.io.File f]
(and (.isFile f)
(re-matches #".*\d{3}/\d{8}_.+\.yaml$" (str f)))))
sort
(map (fn [^java.io.File f]
(str "migrations/" (.getName (.getParentFile f)) "/" (.getName f))))))))))
(defn all-liquibase-ids
"Returns a set of all changeset IDs from all migration files."