Per-PR liquibase changelogs (#70263)
This commit is contained in:
8
.gitattributes
vendored
8
.gitattributes
vendored
@@ -1,7 +1,9 @@
|
|||||||
# Custom merge driver for YAML migration files
|
# Custom merge driver for per-release YAML migration files.
|
||||||
# Treats each changeset as an atomic unit to avoid conflicts within changesets
|
# Treats each changeset as an atomic unit to avoid conflicts within changesets.
|
||||||
|
# Only applies to per-release *_update_migrations.yaml files — directory-based
|
||||||
|
# migrations (e.g. 060/*.yaml) have one file per migration and don't need this.
|
||||||
# Execute this in your shell to enable custom merge driver:
|
# Execute this in your shell to enable custom merge driver:
|
||||||
# echo '\n[include]\npath=../.gitconfig' >> .git/config
|
# echo '\n[include]\npath=../.gitconfig' >> .git/config
|
||||||
#
|
#
|
||||||
# TODO (Bryan 12/12/25) -- reenable this when we can completely verify correctness
|
# TODO (Bryan 12/12/25) -- reenable this when we can completely verify correctness
|
||||||
# resources/migrations/*.yaml merge=yaml-migrations
|
# resources/migrations/*_update_migrations.yaml merge=yaml-migrations
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# execute this in your shell to enable custom merge driver:
|
# execute this in your shell to enable custom merge driver:
|
||||||
# echo '\n[include]\npath=../.gitconfig' >> .git/config
|
# echo '\n[include]\npath=../.gitconfig' >> .git/config
|
||||||
[merge "yaml-migrations"]
|
[merge "yaml-migrations"]
|
||||||
name = YAML migration changeset-aware merge
|
name = Per-release YAML migration changeset-aware merge
|
||||||
driver = bin/mage -merge-yaml-migrations %O %A %B %L %P
|
driver = bin/mage -merge-yaml-migrations %O %A %B %L %P
|
||||||
|
|||||||
9
bb.edn
9
bb.edn
@@ -162,6 +162,15 @@
|
|||||||
(p/shell "bin/pre-lint-migrations-file.sh")
|
(p/shell "bin/pre-lint-migrations-file.sh")
|
||||||
(lint-migrations-file/-main))}
|
(lint-migrations-file/-main))}
|
||||||
|
|
||||||
|
new-migration
|
||||||
|
{:doc "Create a new directory-based migration file (v60+)"
|
||||||
|
:requires [[mage.new-migration :as new-migration]]
|
||||||
|
:examples [["./bin/mage new-migration 60 mq_indexes" "Create resources/migrations/060/20260905_mq_indexes.yaml"]]
|
||||||
|
:arg-schema [:tuple
|
||||||
|
[:int {:name "version" :description "Major version number (e.g., 60)"}]
|
||||||
|
[:string {:name "description" :description "Lowercase alphanumeric/underscore name (e.g., mq_indexes)"}]]
|
||||||
|
:task (task! (new-migration/new-migration parsed))}
|
||||||
|
|
||||||
jar-download
|
jar-download
|
||||||
{:doc "Download (and optionally run) jar for a metabase version or branch"
|
{:doc "Download (and optionally run) jar for a metabase version or branch"
|
||||||
:examples [["./bin/mage jar-download 50" "Download the latest enterprise version of release 50 to ~/path/to/my/metabase/jars"]
|
:examples [["./bin/mage jar-download 50" "Download the latest enterprise version of release 50 to ~/path/to/my/metabase/jars"]
|
||||||
|
|||||||
@@ -18,16 +18,9 @@
|
|||||||
This format should no longer be used for new migrations, use `id-timestamp-format-re` instead."
|
This format should no longer be used for new migrations, use `id-timestamp-format-re` instead."
|
||||||
#"^v\d{2,}\.\d{2}-\d{3}$")
|
#"^v\d{2,}\.\d{2}-\d{3}$")
|
||||||
|
|
||||||
(s/def ::id (s/and string?
|
;; Basic type check only — format enforcement is file-aware and happens in
|
||||||
(s/or
|
;; `lint-migrations-file/require-change-set-ids-match-file-format`.
|
||||||
;; new ids should conform this format
|
(s/def ::id string?)
|
||||||
:id-with-timestamp ;; e.g: v49.2023-12-14T08:54:54
|
|
||||||
(s/and #(re-matches id-timestamp-format-re %))
|
|
||||||
:id-with-old-version-format ;; e.g: v49.00-008
|
|
||||||
(s/and #(re-matches id-number-format-re %)
|
|
||||||
;; the cut off id for this old format is v49.00-060
|
|
||||||
;; see #36787 for context
|
|
||||||
#(neg? (compare % "v49.00-061"))))))
|
|
||||||
|
|
||||||
(s/def ::author string?)
|
(s/def ::author string?)
|
||||||
|
|
||||||
@@ -74,54 +67,6 @@
|
|||||||
[]))) ; provide an empty list if dbms-val is nil
|
[]))) ; provide an empty list if dbms-val is nil
|
||||||
changes))))))
|
changes))))))
|
||||||
|
|
||||||
(def change-types-supporting-rollback
|
|
||||||
"This set was generated with a little grep and awk from the docs here:
|
|
||||||
https://docs.liquibase.com/workflows/liquibase-community/liquibase-auto-rollback.html
|
|
||||||
|
|
||||||
If a new change type is introduced that supports automatic rollback, it should be added
|
|
||||||
to this set."
|
|
||||||
#{:addCheckConstraint
|
|
||||||
:addColumn
|
|
||||||
:addDefaultValue
|
|
||||||
:addForeignKeyConstraint
|
|
||||||
:addLookupTable
|
|
||||||
:addNotNullConstraint
|
|
||||||
:addPrimaryKey
|
|
||||||
:addUniqueConstraint
|
|
||||||
:createIndex
|
|
||||||
:createSequence
|
|
||||||
:createSynonym
|
|
||||||
:createTable
|
|
||||||
:createView
|
|
||||||
:disableCheckConstraint
|
|
||||||
:disableTrigger
|
|
||||||
:dropNotNullConstraint
|
|
||||||
:enableCheckConstraint
|
|
||||||
:enableTrigger
|
|
||||||
:renameColumn
|
|
||||||
:renameSequence
|
|
||||||
:renameTable
|
|
||||||
:renameTrigger
|
|
||||||
:renameView
|
|
||||||
;; assumes all custom changes use the `def-migration` or `define-reversible-migration` in
|
|
||||||
;; [[metabase.app-db.custom-migrations]]
|
|
||||||
:customChange})
|
|
||||||
|
|
||||||
(defn- major-version
|
|
||||||
"Returns major version from id string, e.g. 44 from \"v44.00-034\""
|
|
||||||
[id-str]
|
|
||||||
(when (string? id-str)
|
|
||||||
(some-> (re-find #"\d+" id-str) Integer/parseInt)))
|
|
||||||
|
|
||||||
(defn- rollback-present-when-required?
|
|
||||||
"Ensures rollback key is present when change type doesn't support auto rollback"
|
|
||||||
[{:keys [id changes] :as change-set}]
|
|
||||||
(or
|
|
||||||
(int? id)
|
|
||||||
(< (major-version id) 45)
|
|
||||||
(some change-types-supporting-rollback (mapcat keys changes))
|
|
||||||
(contains? change-set :rollback)))
|
|
||||||
|
|
||||||
(defn- disallow-delete-cascade-with-add-column
|
(defn- disallow-delete-cascade-with-add-column
|
||||||
"Returns false if addColumn changeSet uses deleteCascade. See Metabase issue #14321"
|
"Returns false if addColumn changeSet uses deleteCascade. See Metabase issue #14321"
|
||||||
[{:keys [changes]}]
|
[{:keys [changes]}]
|
||||||
@@ -135,7 +80,6 @@
|
|||||||
|
|
||||||
(s/def ::change-set
|
(s/def ::change-set
|
||||||
(s/and
|
(s/and
|
||||||
rollback-present-when-required?
|
|
||||||
disallow-delete-cascade-with-add-column
|
disallow-delete-cascade-with-add-column
|
||||||
(s/keys :req-un [::id ::author ::changes ::comment]
|
(s/keys :req-un [::id ::author ::changes ::comment]
|
||||||
:opt-un [::preConditions])))
|
:opt-un [::preConditions])))
|
||||||
|
|||||||
@@ -10,7 +10,8 @@
|
|||||||
[clojure.spec.alpha :as s]
|
[clojure.spec.alpha :as s]
|
||||||
[clojure.string :as str])
|
[clojure.string :as str])
|
||||||
(:import
|
(:import
|
||||||
[clojure.lang ExceptionInfo]))
|
(clojure.lang ExceptionInfo)
|
||||||
|
(java.lang Integer)))
|
||||||
|
|
||||||
#_:clj-kondo/ignore
|
#_:clj-kondo/ignore
|
||||||
(set! *warn-on-reflection* true)
|
(set! *warn-on-reflection* true)
|
||||||
@@ -52,26 +53,58 @@
|
|||||||
(when (seq duplicates)
|
(when (seq duplicates)
|
||||||
(throw (validation-error "Change set IDs are not distinct." {:duplicates duplicates})))))
|
(throw (validation-error "Change set IDs are not distinct." {:duplicates duplicates})))))
|
||||||
|
|
||||||
(defn- require-change-set-ids-in-order [change-log]
|
(defn- directory-based-migration-file?
|
||||||
(let [ids (change-set-ids change-log)
|
"Returns true if the file is a directory-based migration file (e.g., `060/20260905_mq_indexes.yaml`)."
|
||||||
out-of-order-ids (->> ids
|
[file]
|
||||||
(partition 2 1)
|
(boolean (re-matches #".*\d{3}/\d{8}_[a-z0-9_]+\.yaml$" (str file))))
|
||||||
(filter (fn [[id1 id2]]
|
|
||||||
(pos? (compare id1 id2)))))]
|
|
||||||
|
|
||||||
(when (seq out-of-order-ids)
|
(defn- file-version
|
||||||
(throw (validation-error "Change set IDs are not in order"
|
"Extracts the migration version number from a file.
|
||||||
{:out-of-order-ids out-of-order-ids})))))
|
For per-release files like `059_update_migrations.yaml`, parses from the filename.
|
||||||
|
For directory-based files like `060/20260106_125531.yaml`, parses from the parent directory name."
|
||||||
|
[^java.io.File file]
|
||||||
|
(if (directory-based-migration-file? file)
|
||||||
|
(parse-long (re-find #"\d+" (.getName (.getParentFile file))))
|
||||||
|
(parse-long (re-find #"\d+" (.getName file)))))
|
||||||
|
|
||||||
|
(defn- changeset-version+id
|
||||||
|
"Returns [version local-id] for a changeset.
|
||||||
|
Parses from the ID itself, which should be of the form 'v{version}.{local-id}'."
|
||||||
|
[_file changeset-id]
|
||||||
|
(let [id-str (str changeset-id)]
|
||||||
|
(when-let [[_ version local-id] (re-matches #"^v(\d+)\.(.+)$" id-str)]
|
||||||
|
[(parse-long version) local-id])))
|
||||||
|
|
||||||
|
(defn- changeset-at-or-after?
|
||||||
|
"Returns true if the changeset (identified by file context and changeset ID) is at or after
|
||||||
|
the given [threshold-version threshold-id]."
|
||||||
|
[file changeset-id threshold-version threshold-id]
|
||||||
|
(when-let [[v local-id] (changeset-version+id file changeset-id)]
|
||||||
|
(or (> v threshold-version)
|
||||||
|
(and (= v threshold-version)
|
||||||
|
(not (neg? (compare local-id threshold-id)))))))
|
||||||
|
|
||||||
|
(defn- require-change-set-ids-in-order [change-log file]
|
||||||
|
(when-not (directory-based-migration-file? file)
|
||||||
|
(let [ids (change-set-ids change-log)
|
||||||
|
out-of-order-ids (->> ids
|
||||||
|
(partition 2 1)
|
||||||
|
(filter (fn [[id1 id2]]
|
||||||
|
(pos? (compare id1 id2)))))]
|
||||||
|
(when (seq out-of-order-ids)
|
||||||
|
(throw (validation-error "Change set IDs are not in order"
|
||||||
|
{:out-of-order-ids out-of-order-ids}))))))
|
||||||
|
|
||||||
(defn- require-change-set-ids-in-correct-file [change-log file]
|
(defn- require-change-set-ids-in-correct-file [change-log file]
|
||||||
(let [file-version (parse-long (re-find #"\d+" (.getName file)))
|
(let [fv (file-version file)
|
||||||
ids (change-set-ids change-log)
|
ids (change-set-ids change-log)
|
||||||
wrong-file-ids (->> ids
|
wrong-file-ids
|
||||||
(filter (fn [id]
|
(->> ids
|
||||||
(let [id-version (parse-long (re-find #"\d+" id))]
|
(filter (fn [id]
|
||||||
(if (= file-version 1)
|
(let [id-version (parse-long (re-find #"\d+" id))]
|
||||||
(> id-version 55)
|
(if (= fv 1)
|
||||||
(not= file-version id-version))))))]
|
(> id-version 55)
|
||||||
|
(not= fv id-version))))))]
|
||||||
(when (seq wrong-file-ids)
|
(when (seq wrong-file-ids)
|
||||||
(throw (validation-error "Change set IDs are in the wrong file"
|
(throw (validation-error "Change set IDs are in the wrong file"
|
||||||
{:wrong-file-ids wrong-file-ids})))))
|
{:wrong-file-ids wrong-file-ids})))))
|
||||||
@@ -101,18 +134,19 @@
|
|||||||
(some #(check-change-use-types? target-types %) (get-in change-set [:changeSet :changes])))
|
(some #(check-change-use-types? target-types %) (get-in change-set [:changeSet :changes])))
|
||||||
|
|
||||||
(defn- require-no-types-in-change-log!
|
(defn- require-no-types-in-change-log!
|
||||||
"Returns true if none of the changes in the change-log contain usage of any type specified in `target-types`.
|
"Throws if any changeset at or after [version id] uses a type in target-types.
|
||||||
|
When version and id are not provided, checks all changesets."
|
||||||
`id-filter-fn` is a function that takes an ID and return true if the changeset should be checked."
|
|
||||||
([target-types change-log]
|
([target-types change-log]
|
||||||
(require-no-types-in-change-log! target-types change-log (constantly true)))
|
(require-no-types-in-change-log! target-types change-log nil nil nil))
|
||||||
([target-types change-log id-filter-fn]
|
([target-types change-log file version id]
|
||||||
{:pre [(set? target-types)]}
|
{:pre [(set? target-types)]}
|
||||||
(when-let [using-types? (->> change-log
|
(when-let [using-types? (->> change-log
|
||||||
(filter (fn [change-set]
|
(filter (fn [change-set]
|
||||||
(let [id (get-in change-set [:changeSet :id])]
|
(let [cs-id (get-in change-set [:changeSet :id])]
|
||||||
(and (string? id)
|
(and cs-id
|
||||||
(id-filter-fn id)))))
|
(if version
|
||||||
|
(changeset-at-or-after? file cs-id version id)
|
||||||
|
true)))))
|
||||||
(filter #(check-change-set-use-types? target-types %))
|
(filter #(check-change-set-use-types? target-types %))
|
||||||
(map #(get-in % [:changeSet :id]))
|
(map #(get-in % [:changeSet :id]))
|
||||||
seq)]
|
seq)]
|
||||||
@@ -132,17 +166,46 @@
|
|||||||
(defn require-no-bare-boolean-types
|
(defn require-no-bare-boolean-types
|
||||||
"Ensures that no \"boolean\" type columns are added in changesets with id later than v49.00-032. From that point on,
|
"Ensures that no \"boolean\" type columns are added in changesets with id later than v49.00-032. From that point on,
|
||||||
\"${boolean.type}\" should be used instead, so that we can consistently use `BIT(1)` for Boolean columns on MySQL."
|
\"${boolean.type}\" should be used instead, so that we can consistently use `BIT(1)` for Boolean columns on MySQL."
|
||||||
[change-log]
|
[change-log file]
|
||||||
(require-no-types-in-change-log! #{"boolean"} change-log #(pos? (compare % "v49.00-032"))))
|
(require-no-types-in-change-log! #{"boolean"} change-log file 49 "00-032"))
|
||||||
|
|
||||||
(defn require-no-datetime-type
|
(defn require-no-datetime-type
|
||||||
"Ensures that no \"datetime\" or \"timestamp without time zone\".
|
"Ensures that no \"datetime\" or \"timestamp without time zone\".
|
||||||
From that point on, \"${timestamp_type}\" should be used instead, so that all of our time related columns are tz-aware."
|
From that point on, \"${timestamp_type}\" should be used instead, so that all of our time related columns are tz-aware."
|
||||||
[change-log]
|
[change-log file]
|
||||||
(require-no-types-in-change-log!
|
(require-no-types-in-change-log!
|
||||||
#{"datetime" "timestamp" "timestamp without time zone"}
|
#{"datetime" "timestamp" "timestamp without time zone"}
|
||||||
change-log
|
change-log file 49 "00-000"))
|
||||||
#(pos? (compare % "v49.00-000"))))
|
|
||||||
|
(def ^:private directory-based-id-format-re
|
||||||
|
"For directory-based migration files (v60+), IDs must be prefixed with the correct version.
|
||||||
|
E.g., `v60.my-migration-name` or `v60.1`"
|
||||||
|
#"^v\d{2,}\..+$")
|
||||||
|
|
||||||
|
(defn- require-change-set-ids-match-file-format
|
||||||
|
"Enforces that changeset IDs use the correct format for the file type:
|
||||||
|
- Directory-based files (v60+): IDs must be prefixed with v{version}. (e.g., v60.anything)
|
||||||
|
- 001_update_migrations.yaml: any string ID is allowed (legacy file)
|
||||||
|
- Other per-release files: IDs must match the timestamp format"
|
||||||
|
[change-log file]
|
||||||
|
(cond
|
||||||
|
(directory-based-migration-file? file)
|
||||||
|
(let [ids (change-set-ids change-log)
|
||||||
|
bad-ids (remove #(re-matches directory-based-id-format-re %) ids)]
|
||||||
|
(when (seq bad-ids)
|
||||||
|
(throw (validation-error
|
||||||
|
(format "Directory-based migration file contains IDs without v{version}. prefix: %s"
|
||||||
|
(str/join ", " bad-ids))
|
||||||
|
{:invalid-ids (vec bad-ids)}))))
|
||||||
|
|
||||||
|
(not= (file-version file) 1)
|
||||||
|
(let [ids (change-set-ids change-log)
|
||||||
|
bad-ids (remove #(re-matches change-set.strict/id-timestamp-format-re %) ids)]
|
||||||
|
(when (seq bad-ids)
|
||||||
|
(throw (validation-error
|
||||||
|
(format "Per-release migration file contains non-timestamp ID formats: %s"
|
||||||
|
(str/join ", " bad-ids))
|
||||||
|
{:invalid-ids (vec bad-ids)}))))))
|
||||||
|
|
||||||
(defn require-primary-key-exists-has-table-name
|
(defn require-primary-key-exists-has-table-name
|
||||||
"Ensures that all primaryKeyExists preconditions specify a tableName."
|
"Ensures that all primaryKeyExists preconditions specify a tableName."
|
||||||
@@ -165,25 +228,82 @@
|
|||||||
(throw (validation-error "Expected exactly one key." {:keys keys})))
|
(throw (validation-error "Expected exactly one key." {:keys keys})))
|
||||||
(first keys)))
|
(first keys)))
|
||||||
|
|
||||||
|
(defn- major-version
|
||||||
|
"Returns major version from id string, e.g. 44 from \"v44.00-034\".
|
||||||
|
For directory-based migrations (file path matches `NNN/`), extracts
|
||||||
|
version from the file path. Otherwise parses from the id string."
|
||||||
|
[id-str file]
|
||||||
|
(if-let [[_ file-version] (when file (re-find #"(\d{3})[/\\]" (str file)))]
|
||||||
|
(Integer/parseInt file-version)
|
||||||
|
(when (string? id-str)
|
||||||
|
(some-> (re-find #"\d+" id-str) Integer/parseInt))))
|
||||||
|
|
||||||
|
(def change-types-supporting-rollback
|
||||||
|
"This set was generated with a little grep and awk from the docs here:
|
||||||
|
https://docs.liquibase.com/workflows/liquibase-community/liquibase-auto-rollback.html
|
||||||
|
|
||||||
|
If a new change type is introduced that supports automatic rollback, it should be added
|
||||||
|
to this set."
|
||||||
|
#{:addCheckConstraint
|
||||||
|
:addColumn
|
||||||
|
:addDefaultValue
|
||||||
|
:addForeignKeyConstraint
|
||||||
|
:addLookupTable
|
||||||
|
:addNotNullConstraint
|
||||||
|
:addPrimaryKey
|
||||||
|
:addUniqueConstraint
|
||||||
|
:createIndex
|
||||||
|
:createSequence
|
||||||
|
:createSynonym
|
||||||
|
:createTable
|
||||||
|
:createView
|
||||||
|
:disableCheckConstraint
|
||||||
|
:disableTrigger
|
||||||
|
:dropNotNullConstraint
|
||||||
|
:enableCheckConstraint
|
||||||
|
:enableTrigger
|
||||||
|
:renameColumn
|
||||||
|
:renameSequence
|
||||||
|
:renameTable
|
||||||
|
:renameTrigger
|
||||||
|
:renameView
|
||||||
|
;; assumes all custom changes use the `def-migration` or `define-reversible-migration` in
|
||||||
|
;; [[metabase.app-db.custom-migrations]]
|
||||||
|
:customChange})
|
||||||
|
|
||||||
|
(defn- rollback-present-when-required?
|
||||||
|
"Ensures rollback key is present when change type doesn't support auto rollback.
|
||||||
|
`file` is used to extract version for directory-based migrations."
|
||||||
|
[{:keys [id changes] :as change-set} file]
|
||||||
|
(or
|
||||||
|
(let [v (major-version (str id) file)]
|
||||||
|
(and v (< v 45)))
|
||||||
|
(some change-types-supporting-rollback (mapcat keys changes))
|
||||||
|
(contains? change-set :rollback)))
|
||||||
|
|
||||||
(defn- validate-database-change-log [change-log file]
|
(defn- validate-database-change-log [change-log file]
|
||||||
(when (string? change-log)
|
(when (string? change-log)
|
||||||
(throw (validation-error "Expected `:databaseChangeLog` to be a map, not a string.")))
|
(throw (validation-error "Expected `:databaseChangeLog` to be a map, not a string.")))
|
||||||
|
|
||||||
(require-distinct-change-set-ids change-log)
|
(require-distinct-change-set-ids change-log)
|
||||||
(require-change-set-ids-in-correct-file change-log file)
|
(require-change-set-ids-in-correct-file change-log file)
|
||||||
(require-change-set-ids-in-order change-log)
|
(require-change-set-ids-in-order change-log file)
|
||||||
|
(require-change-set-ids-match-file-format change-log file)
|
||||||
(require-no-bare-blob-or-text-types change-log)
|
(require-no-bare-blob-or-text-types change-log)
|
||||||
(require-no-bare-boolean-types change-log)
|
(require-no-bare-boolean-types change-log file)
|
||||||
(require-no-datetime-type change-log)
|
(require-no-datetime-type change-log file)
|
||||||
(require-primary-key-exists-has-table-name change-log)
|
(require-primary-key-exists-has-table-name change-log)
|
||||||
(let [{:keys [changeSet]
|
(let [{:keys [changeSet]
|
||||||
:as all} (group-by only-key change-log)]
|
:as all} (group-by only-key change-log)]
|
||||||
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet})
|
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet :logicalFilePath})
|
||||||
(throw (validation-error "Expected exactly one of :property, :objectQuotingStrategy, or :changeSet."
|
(throw (validation-error "Expected exactly one of :property, :objectQuotingStrategy, :logicalFilePath, or :changeSet."
|
||||||
{:keys (keys all)})))
|
{:keys (keys all)})))
|
||||||
(doseq [{:keys [changeSet]} changeSet
|
(doseq [{:keys [changeSet]} changeSet]
|
||||||
:when (not (s/valid? ::changeSet changeSet))]
|
(when-not (s/valid? ::changeSet changeSet)
|
||||||
(throw (validation-error "Invalid change set." (s/explain-data ::changeSet changeSet))))))
|
(throw (validation-error "Invalid change set." (s/explain-data ::changeSet changeSet))))
|
||||||
|
(when-not (rollback-present-when-required? changeSet file)
|
||||||
|
(throw (validation-error "Rollback is required but not present."
|
||||||
|
{:id (:id changeSet)}))))))
|
||||||
|
|
||||||
(defn- validate-migrations [migrations file]
|
(defn- validate-migrations [migrations file]
|
||||||
(require-database-change-log! migrations)
|
(require-database-change-log! migrations)
|
||||||
@@ -195,7 +315,14 @@
|
|||||||
(let [dir-str #?(:bb "resources/migrations" :clj "../../resources/migrations")
|
(let [dir-str #?(:bb "resources/migrations" :clj "../../resources/migrations")
|
||||||
dir (io/file dir-str)]
|
dir (io/file dir-str)]
|
||||||
(->> (file-seq dir)
|
(->> (file-seq dir)
|
||||||
(filter #(and (.isFile %) (str/ends-with? (.getName %) "_update_migrations.yaml"))))))
|
(filter (fn [^java.io.File f]
|
||||||
|
(and (.isFile f)
|
||||||
|
(or
|
||||||
|
;; Per-release files: 059_update_migrations.yaml
|
||||||
|
(str/ends-with? (.getName f) "_update_migrations.yaml")
|
||||||
|
;; Directory-based files: 060/20260106_125531.yaml
|
||||||
|
(directory-based-migration-file? f)))))
|
||||||
|
(sort-by str))))
|
||||||
|
|
||||||
(defn- migrations [file]
|
(defn- migrations [file]
|
||||||
(assert (.exists file) (format "%s does not exist" file))
|
(assert (.exists file) (format "%s does not exist" file))
|
||||||
@@ -207,13 +334,22 @@
|
|||||||
(fix-vals (yaml/parse-string
|
(fix-vals (yaml/parse-string
|
||||||
#_:clj-kondo/ignore (slurp file)))))
|
#_:clj-kondo/ignore (slurp file)))))
|
||||||
|
|
||||||
|
(defn- display-name
|
||||||
|
"Returns a human-readable name for a migration file.
|
||||||
|
For per-release files, returns just the filename.
|
||||||
|
For directory-based files, returns `parent/filename`."
|
||||||
|
[^java.io.File file]
|
||||||
|
(if (directory-based-migration-file? file)
|
||||||
|
(str (.getName (.getParentFile file)) "/" (.getName file))
|
||||||
|
(.getName file)))
|
||||||
|
|
||||||
(defn- validate-all []
|
(defn- validate-all []
|
||||||
(doseq [file (migration-files)]
|
(doseq [file (migration-files)]
|
||||||
(println "Validating" (.getName file) "...")
|
(println "Validating" (display-name file) "...")
|
||||||
(try
|
(try
|
||||||
(validate-migrations (migrations file) file)
|
(validate-migrations (migrations file) file)
|
||||||
(catch ExceptionInfo e
|
(catch ExceptionInfo e
|
||||||
(throw (ex-info (.getMessage e) (assoc (ex-data e) :file (.getName file)) e))))))
|
(throw (ex-info (.getMessage e) (assoc (ex-data e) :file (display-name file)) e))))))
|
||||||
|
|
||||||
(defn -main
|
(defn -main
|
||||||
"Entry point for Clojure CLI task `lint-migrations-file`. Run it with
|
"Entry point for Clojure CLI task `lint-migrations-file`. Run it with
|
||||||
|
|||||||
@@ -200,17 +200,19 @@
|
|||||||
(validate-id "v49.2024-01-01T10:30:00"))))
|
(validate-id "v49.2024-01-01T10:30:00"))))
|
||||||
|
|
||||||
(testing "invalid date components should throw an error"
|
(testing "invalid date components should throw an error"
|
||||||
(are [msg id]
|
(let [validate-id-strict (fn [id]
|
||||||
(thrown-with-msg?
|
(validate-file (io/file "049_update_migrations.yaml")
|
||||||
clojure.lang.ExceptionInfo
|
(mock-change-set :id id)))]
|
||||||
#"Invalid change set\."
|
(are [msg id]
|
||||||
(validate-id "v49.2024-30-01T10:30:00")
|
(thrown-with-msg?
|
||||||
msg)
|
clojure.lang.ExceptionInfo
|
||||||
"invalid month" "v49.2024-13-01T10:30:00"
|
#"non-timestamp ID formats"
|
||||||
"invalid day" "v49.2024-01-32T10:30:00"
|
(validate-id-strict id))
|
||||||
"invalid hour" "v49.2024-01-01T25:30:00"
|
"invalid month" "v49.2024-13-01T10:30:00"
|
||||||
"invalid minute" "v49.2024-01-01T10:60:00"
|
"invalid day" "v49.2024-01-32T10:30:00"
|
||||||
"invalid second" "v49.2024-01-01T10:30:60"))))
|
"invalid hour" "v49.2024-01-01T25:30:00"
|
||||||
|
"invalid minute" "v49.2024-01-01T10:60:00"
|
||||||
|
"invalid second" "v49.2024-01-01T10:30:60")))))
|
||||||
|
|
||||||
(deftest ^:parallel validate-id-in-file-test
|
(deftest ^:parallel validate-id-in-file-test
|
||||||
(letfn [(validate-id [id file]
|
(letfn [(validate-id [id file]
|
||||||
@@ -285,7 +287,7 @@
|
|||||||
(testing "missing rollback key fails"
|
(testing "missing rollback key fails"
|
||||||
(is (thrown-with-msg?
|
(is (thrown-with-msg?
|
||||||
clojure.lang.ExceptionInfo
|
clojure.lang.ExceptionInfo
|
||||||
#"Invalid change set\."
|
#"Rollback is required but not present\."
|
||||||
(validate (update (mock-change-set :id "v49.2024-01-01T10:30:00" :changes [{:sql {:sql "select 1"}}])
|
(validate (update (mock-change-set :id "v49.2024-01-01T10:30:00" :changes [{:sql {:sql "select 1"}}])
|
||||||
:changeSet dissoc :rollback)))))
|
:changeSet dissoc :rollback)))))
|
||||||
(testing "nil rollback is allowed"
|
(testing "nil rollback is allowed"
|
||||||
|
|||||||
@@ -17,11 +17,11 @@
|
|||||||
(println (.getMessage e)))
|
(println (.getMessage e)))
|
||||||
(u/exit 2))))
|
(u/exit 2))))
|
||||||
|
|
||||||
(def ^:private FOOTER-PREFIX
|
(def ^:private footer-prefix
|
||||||
"# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<")
|
"# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<")
|
||||||
|
|
||||||
(defn- extract-footer [content]
|
(defn- extract-footer [content]
|
||||||
(let [idx (some-> content (str/index-of FOOTER-PREFIX))]
|
(let [idx (some-> content (str/index-of footer-prefix))]
|
||||||
(if (and idx (pos? idx))
|
(if (and idx (pos? idx))
|
||||||
(subs content idx)
|
(subs content idx)
|
||||||
"")))
|
"")))
|
||||||
@@ -182,6 +182,10 @@
|
|||||||
:conflicts (vec (keep #(when (= (:source %) :conflict) (:id %)) sorted-merged))
|
:conflicts (vec (keep #(when (= (:source %) :conflict) (:id %)) sorted-merged))
|
||||||
:cnt (count sorted-merged)}))
|
:cnt (count sorted-merged)}))
|
||||||
|
|
||||||
|
;;
|
||||||
|
;; Custom merge driver for per-release *_update_migrations.yaml files.
|
||||||
|
;; Directory-based migrations (e.g. 060/*.yaml) have one file per migration
|
||||||
|
;; and should use git's default merge strategy.
|
||||||
;;
|
;;
|
||||||
;; Usage (called by git):
|
;; Usage (called by git):
|
||||||
;; merge-yaml-migrations %O %A %B %L %P
|
;; merge-yaml-migrations %O %A %B %L %P
|
||||||
@@ -196,7 +200,9 @@
|
|||||||
;; 1 = conflicts detected (conflict markers added to file)
|
;; 1 = conflicts detected (conflict markers added to file)
|
||||||
;; >1 = merge failed
|
;; >1 = merge failed
|
||||||
|
|
||||||
(defn -main [{:keys [arguments]}]
|
(defn -main
|
||||||
|
"Main function"
|
||||||
|
[{:keys [arguments]}]
|
||||||
(when (< (count arguments) 3)
|
(when (< (count arguments) 3)
|
||||||
(binding [*out* *err*]
|
(binding [*out* *err*]
|
||||||
(println "Usage: merge-yaml-migrations <base> <ours> <theirs> [conflict-marker-size] [file-path]")
|
(println "Usage: merge-yaml-migrations <base> <ours> <theirs> [conflict-marker-size] [file-path]")
|
||||||
@@ -209,6 +215,11 @@
|
|||||||
p
|
p
|
||||||
(str u/project-root-directory "/" p)))
|
(str u/project-root-directory "/" p)))
|
||||||
[base ours theirs marker-size filepath] arguments
|
[base ours theirs marker-size filepath] arguments
|
||||||
|
_ (when (and filepath (not (str/ends-with? filepath "_update_migrations.yaml")))
|
||||||
|
(binding [*out* *err*]
|
||||||
|
(println "merge-yaml-migrations is only for per-release *_update_migrations.yaml files, not:" filepath)
|
||||||
|
(println "Falling back to git's default merge."))
|
||||||
|
(u/exit 1))
|
||||||
base (resolve-path base)
|
base (resolve-path base)
|
||||||
ours (resolve-path ours)
|
ours (resolve-path ours)
|
||||||
theirs (resolve-path theirs)
|
theirs (resolve-path theirs)
|
||||||
|
|||||||
75
mage/src/mage/new_migration.clj
Normal file
75
mage/src/mage/new_migration.clj
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
(ns mage.new-migration
|
||||||
|
"Mage task for creating new directory-based migration files (v60+).
|
||||||
|
|
||||||
|
Each migration file is a YAML file inside a version directory,
|
||||||
|
e.g., `resources/migrations/060/20260905_mq_indexes.yaml`."
|
||||||
|
(:require
|
||||||
|
[clojure.java.io :as io]
|
||||||
|
[mage.util :as u])
|
||||||
|
(:import
|
||||||
|
(java.security SecureRandom)
|
||||||
|
(java.time ZoneOffset ZonedDateTime)
|
||||||
|
(java.time.format DateTimeFormatter)))
|
||||||
|
|
||||||
|
(set! *warn-on-reflection* true)
|
||||||
|
|
||||||
|
(def ^:private valid-description-re
|
||||||
|
"Descriptions must be lowercase alphanumeric or underscores."
|
||||||
|
#"[a-z0-9][a-z0-9_]*[a-z0-9]|[a-z0-9]")
|
||||||
|
|
||||||
|
(defn- git-user-name []
|
||||||
|
(try
|
||||||
|
(u/sh "git" "config" "user.name")
|
||||||
|
(catch Exception _
|
||||||
|
"author")))
|
||||||
|
|
||||||
|
(defn- utc-date []
|
||||||
|
(.format (DateTimeFormatter/ofPattern "yyyyMMdd")
|
||||||
|
(ZonedDateTime/now ZoneOffset/UTC)))
|
||||||
|
|
||||||
|
(def ^:private alphanumeric "abcdefghijklmnopqrstuvwxyz0123456789")
|
||||||
|
|
||||||
|
(defn- random-id
|
||||||
|
"Generates a random 6-character lowercase alphanumeric string."
|
||||||
|
[]
|
||||||
|
(let [rng (SecureRandom.)]
|
||||||
|
(apply str (repeatedly 6 #(nth alphanumeric (.nextInt rng (count alphanumeric)))))))
|
||||||
|
|
||||||
|
(defn- migration-template [version author]
|
||||||
|
(let [id (random-id)]
|
||||||
|
(str "## TODO: quick description of what feature/bug this migration is for\n"
|
||||||
|
"\n"
|
||||||
|
"databaseChangeLog:\n"
|
||||||
|
" - changeSet:\n"
|
||||||
|
" id: v" version "." id "\n"
|
||||||
|
" author: " author "\n"
|
||||||
|
" comment: TODO\n"
|
||||||
|
" changes:\n"
|
||||||
|
" - TODO\n")))
|
||||||
|
|
||||||
|
(defn new-migration
|
||||||
|
"Creates a new directory-based migration file for the given version number.
|
||||||
|
|
||||||
|
Usage: `./bin/mage new-migration 60 mq_indexes`"
|
||||||
|
[{:keys [arguments]}]
|
||||||
|
(let [version (first arguments)
|
||||||
|
description (second arguments)
|
||||||
|
_ (when-not (and version description)
|
||||||
|
(binding [*out* *err*]
|
||||||
|
(println "Usage: mage new-migration <version> <description>")
|
||||||
|
(println " e.g., mage new-migration 60 mq_indexes"))
|
||||||
|
(u/exit 1))
|
||||||
|
_ (when-not (re-matches valid-description-re description)
|
||||||
|
(binding [*out* *err*]
|
||||||
|
(println (str "Error: description must be lowercase alphanumeric or underscores, got: \"" description "\""))
|
||||||
|
(println " e.g., mq_indexes, add_user_table, fix_fk"))
|
||||||
|
(u/exit 1))
|
||||||
|
dir-name (format "%03d" version)
|
||||||
|
dir (io/file u/project-root-directory "resources" "migrations" dir-name)
|
||||||
|
date (utc-date)
|
||||||
|
filename (str date "_" description ".yaml")
|
||||||
|
file (io/file dir filename)
|
||||||
|
author (git-user-name)]
|
||||||
|
(.mkdirs dir)
|
||||||
|
(spit file (migration-template version author))
|
||||||
|
(println "Created" (str "resources/migrations/" dir-name "/" filename))))
|
||||||
@@ -47,5 +47,5 @@ databaseChangeLog:
|
|||||||
|
|
||||||
- includeAll:
|
- includeAll:
|
||||||
path: migrations
|
path: migrations
|
||||||
endsWithFilter: "_update_migrations.yaml"
|
endsWithFilter: ".yaml"
|
||||||
maxDepth: 1
|
maxDepth: 2
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
databaseChangeLog:
|
databaseChangeLog:
|
||||||
- include:
|
- include:
|
||||||
file: "migrations/000_legacy_migrations.yaml"
|
file: "liquibase_legacy_migrations.yaml"
|
||||||
- include:
|
- include:
|
||||||
file: "liquibase.yaml"
|
file: "liquibase.yaml"
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
# In order to keep instances running metabase < 45 able to upgrade to the latest, we need to keep this file here.
|
# In order to keep instances running metabase < 45 able to upgrade to the latest, we need to keep this file here.
|
||||||
|
|
||||||
databaseChangeLog:
|
databaseChangeLog:
|
||||||
|
- logicalFilePath: migrations/000_legacy_migrations.yaml
|
||||||
- property:
|
- property:
|
||||||
name: timestamp_type
|
name: timestamp_type
|
||||||
value: timestamp with time zone
|
value: timestamp with time zone
|
||||||
79
resources/migrations/060/20260213_dimension_columns.yaml
Normal file
79
resources/migrations/060/20260213_dimension_columns.yaml
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
## TODO: quick description of what feature/bug this migration is for
|
||||||
|
|
||||||
|
databaseChangeLog:
|
||||||
|
- logicalFilePath: migrations/060_update_migrations.yaml
|
||||||
|
- changeSet:
|
||||||
|
id: v60.2026-02-13T21:43:00
|
||||||
|
author: epaget
|
||||||
|
comment: Add dimensions column to measure table
|
||||||
|
changes:
|
||||||
|
- addColumn:
|
||||||
|
tableName: measure
|
||||||
|
columns:
|
||||||
|
- column:
|
||||||
|
name: dimensions
|
||||||
|
type: ${text.type}
|
||||||
|
remarks: JSON array of persisted dimension definitions (id, display-name, types)
|
||||||
|
constraints:
|
||||||
|
nullable: true
|
||||||
|
rollback:
|
||||||
|
- dropColumn:
|
||||||
|
tableName: measure
|
||||||
|
columnName: dimensions
|
||||||
|
|
||||||
|
- changeSet:
|
||||||
|
id: v60.2026-02-13T21:43:01
|
||||||
|
author: epaget
|
||||||
|
comment: Add dimension_mappings column to measure table
|
||||||
|
changes:
|
||||||
|
- addColumn:
|
||||||
|
tableName: measure
|
||||||
|
columns:
|
||||||
|
- column:
|
||||||
|
name: dimension_mappings
|
||||||
|
type: ${text.type}
|
||||||
|
remarks: JSON array of dimension-to-field mappings (dimension-id, target, table-id)
|
||||||
|
constraints:
|
||||||
|
nullable: true
|
||||||
|
rollback:
|
||||||
|
- dropColumn:
|
||||||
|
tableName: measure
|
||||||
|
columnName: dimension_mappings
|
||||||
|
|
||||||
|
- changeSet:
|
||||||
|
id: v60.2026-02-13T21:43:02
|
||||||
|
author: epaget
|
||||||
|
comment: Add dimensions column to report_card table for metrics
|
||||||
|
changes:
|
||||||
|
- addColumn:
|
||||||
|
tableName: report_card
|
||||||
|
columns:
|
||||||
|
- column:
|
||||||
|
name: dimensions
|
||||||
|
type: ${text.type}
|
||||||
|
remarks: JSON array of persisted dimension definitions (id, display-name, types)
|
||||||
|
constraints:
|
||||||
|
nullable: true
|
||||||
|
rollback:
|
||||||
|
- dropColumn:
|
||||||
|
tableName: report_card
|
||||||
|
columnName: dimensions
|
||||||
|
|
||||||
|
- changeSet:
|
||||||
|
id: v60.2026-02-13T21:43:03
|
||||||
|
author: epaget
|
||||||
|
comment: Add dimension_mappings column to report_card table for metrics
|
||||||
|
changes:
|
||||||
|
- addColumn:
|
||||||
|
tableName: report_card
|
||||||
|
columns:
|
||||||
|
- column:
|
||||||
|
name: dimension_mappings
|
||||||
|
type: ${text.type}
|
||||||
|
remarks: JSON array of dimension-to-field mappings (dimension-id, target, table-id)
|
||||||
|
constraints:
|
||||||
|
nullable: true
|
||||||
|
rollback:
|
||||||
|
- dropColumn:
|
||||||
|
tableName: report_card
|
||||||
|
columnName: dimension_mappings
|
||||||
122
resources/migrations/060/20260310_source_replacement_run.yaml
Normal file
122
resources/migrations/060/20260310_source_replacement_run.yaml
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
## TODO: quick description of what feature/bug this migration is for
|
||||||
|
|
||||||
|
databaseChangeLog:
|
||||||
|
- logicalFilePath: migrations/060_update_migrations.yaml
|
||||||
|
- changeSet:
|
||||||
|
id: v60.2026-03-06T12:00:00
|
||||||
|
author: bcm
|
||||||
|
comment: Create source_replacement_run table to track async source replacement executions
|
||||||
|
changes:
|
||||||
|
- createTable:
|
||||||
|
tableName: source_replacement_run
|
||||||
|
remarks: Table to track source replacement executions and their status
|
||||||
|
columns:
|
||||||
|
- column:
|
||||||
|
name: id
|
||||||
|
type: int
|
||||||
|
remarks: Unique ID
|
||||||
|
autoIncrement: true
|
||||||
|
constraints:
|
||||||
|
primaryKey: true
|
||||||
|
nullable: false
|
||||||
|
- column:
|
||||||
|
name: source_entity_type
|
||||||
|
type: varchar(50)
|
||||||
|
remarks: "Type of the source entity (card or table)"
|
||||||
|
constraints:
|
||||||
|
nullable: false
|
||||||
|
- column:
|
||||||
|
name: source_entity_id
|
||||||
|
type: int
|
||||||
|
remarks: "ID of the source entity"
|
||||||
|
constraints:
|
||||||
|
nullable: false
|
||||||
|
- column:
|
||||||
|
name: target_entity_type
|
||||||
|
type: varchar(50)
|
||||||
|
remarks: "Type of the target entity (card or table)"
|
||||||
|
constraints:
|
||||||
|
nullable: false
|
||||||
|
- column:
|
||||||
|
name: target_entity_id
|
||||||
|
type: int
|
||||||
|
remarks: "ID of the target entity"
|
||||||
|
constraints:
|
||||||
|
nullable: false
|
||||||
|
- column:
|
||||||
|
name: status
|
||||||
|
type: varchar(50)
|
||||||
|
remarks: "Current status (started, succeeded, failed, canceled, timeout)"
|
||||||
|
constraints:
|
||||||
|
nullable: false
|
||||||
|
- column:
|
||||||
|
name: is_active
|
||||||
|
type: ${boolean.type}
|
||||||
|
remarks: "True only for currently running replacements, null for completed"
|
||||||
|
constraints:
|
||||||
|
nullable: true
|
||||||
|
- column:
|
||||||
|
name: progress
|
||||||
|
type: float
|
||||||
|
remarks: "Progress from 0.0 to 1.0"
|
||||||
|
constraints:
|
||||||
|
nullable: true
|
||||||
|
- column:
|
||||||
|
name: message
|
||||||
|
type: ${text.type}
|
||||||
|
remarks: "Human-readable message; may contain error info"
|
||||||
|
constraints:
|
||||||
|
nullable: true
|
||||||
|
- column:
|
||||||
|
name: user_id
|
||||||
|
type: int
|
||||||
|
remarks: "User who initiated the replacement"
|
||||||
|
constraints:
|
||||||
|
nullable: true
|
||||||
|
- column:
|
||||||
|
name: start_time
|
||||||
|
type: ${timestamp_type}
|
||||||
|
remarks: "When the replacement started"
|
||||||
|
defaultValueComputed: current_timestamp
|
||||||
|
constraints:
|
||||||
|
nullable: false
|
||||||
|
- column:
|
||||||
|
name: end_time
|
||||||
|
type: ${timestamp_type}
|
||||||
|
remarks: "When the replacement completed (null if still running)"
|
||||||
|
constraints:
|
||||||
|
nullable: true
|
||||||
|
|
||||||
|
- changeSet:
|
||||||
|
id: v60.2026-03-06T12:00:01
|
||||||
|
author: bcm
|
||||||
|
comment: Enforce at most one active source replacement run cluster-wide
|
||||||
|
changes:
|
||||||
|
- addUniqueConstraint:
|
||||||
|
tableName: source_replacement_run
|
||||||
|
constraintName: idx_unique_active_source_replacement_run
|
||||||
|
columnNames: is_active
|
||||||
|
|
||||||
|
- changeSet:
|
||||||
|
id: v60.2026-03-06T12:00:02
|
||||||
|
author: bcm
|
||||||
|
comment: Add index on source_replacement_run.user_id
|
||||||
|
changes:
|
||||||
|
- createIndex:
|
||||||
|
tableName: source_replacement_run
|
||||||
|
indexName: idx_source_replacement_run_user_id
|
||||||
|
columns:
|
||||||
|
- column:
|
||||||
|
name: user_id
|
||||||
|
|
||||||
|
- changeSet:
|
||||||
|
id: v60.2026-03-06T12:00:03
|
||||||
|
author: bcm
|
||||||
|
comment: Add foreign key on source_replacement_run(user_id)
|
||||||
|
changes:
|
||||||
|
- addForeignKeyConstraint:
|
||||||
|
baseTableName: source_replacement_run
|
||||||
|
baseColumnNames: user_id
|
||||||
|
referencedTableName: core_user
|
||||||
|
referencedColumnNames: id
|
||||||
|
constraintName: fk_source_replacement_run_user_id
|
||||||
@@ -1,247 +0,0 @@
|
|||||||
databaseChangeLog:
|
|
||||||
- objectQuotingStrategy: QUOTE_ALL_OBJECTS
|
|
||||||
|
|
||||||
- changeSet:
|
|
||||||
id: v60.2026-02-13T21:43:00
|
|
||||||
author: epaget
|
|
||||||
comment: Add dimensions column to measure table
|
|
||||||
changes:
|
|
||||||
- addColumn:
|
|
||||||
tableName: measure
|
|
||||||
columns:
|
|
||||||
- column:
|
|
||||||
name: dimensions
|
|
||||||
type: ${text.type}
|
|
||||||
remarks: JSON array of persisted dimension definitions (id, display-name, types)
|
|
||||||
constraints:
|
|
||||||
nullable: true
|
|
||||||
rollback:
|
|
||||||
- dropColumn:
|
|
||||||
tableName: measure
|
|
||||||
columnName: dimensions
|
|
||||||
|
|
||||||
- changeSet:
|
|
||||||
id: v60.2026-02-13T21:43:01
|
|
||||||
author: epaget
|
|
||||||
comment: Add dimension_mappings column to measure table
|
|
||||||
changes:
|
|
||||||
- addColumn:
|
|
||||||
tableName: measure
|
|
||||||
columns:
|
|
||||||
- column:
|
|
||||||
name: dimension_mappings
|
|
||||||
type: ${text.type}
|
|
||||||
remarks: JSON array of dimension-to-field mappings (dimension-id, target, table-id)
|
|
||||||
constraints:
|
|
||||||
nullable: true
|
|
||||||
rollback:
|
|
||||||
- dropColumn:
|
|
||||||
tableName: measure
|
|
||||||
columnName: dimension_mappings
|
|
||||||
|
|
||||||
- changeSet:
|
|
||||||
id: v60.2026-02-13T21:43:02
|
|
||||||
author: epaget
|
|
||||||
comment: Add dimensions column to report_card table for metrics
|
|
||||||
changes:
|
|
||||||
- addColumn:
|
|
||||||
tableName: report_card
|
|
||||||
columns:
|
|
||||||
- column:
|
|
||||||
name: dimensions
|
|
||||||
type: ${text.type}
|
|
||||||
remarks: JSON array of persisted dimension definitions (id, display-name, types)
|
|
||||||
constraints:
|
|
||||||
nullable: true
|
|
||||||
rollback:
|
|
||||||
- dropColumn:
|
|
||||||
tableName: report_card
|
|
||||||
columnName: dimensions
|
|
||||||
|
|
||||||
- changeSet:
|
|
||||||
id: v60.2026-02-13T21:43:03
|
|
||||||
author: epaget
|
|
||||||
comment: Add dimension_mappings column to report_card table for metrics
|
|
||||||
changes:
|
|
||||||
- addColumn:
|
|
||||||
tableName: report_card
|
|
||||||
columns:
|
|
||||||
- column:
|
|
||||||
name: dimension_mappings
|
|
||||||
type: ${text.type}
|
|
||||||
remarks: JSON array of dimension-to-field mappings (dimension-id, target, table-id)
|
|
||||||
constraints:
|
|
||||||
nullable: true
|
|
||||||
rollback:
|
|
||||||
- dropColumn:
|
|
||||||
tableName: report_card
|
|
||||||
columnName: dimension_mappings
|
|
||||||
|
|
||||||
- changeSet:
|
|
||||||
id: v60.2026-03-06T12:00:00
|
|
||||||
author: bcm
|
|
||||||
comment: Create source_replacement_run table to track async source replacement executions
|
|
||||||
changes:
|
|
||||||
- createTable:
|
|
||||||
tableName: source_replacement_run
|
|
||||||
remarks: Table to track source replacement executions and their status
|
|
||||||
columns:
|
|
||||||
- column:
|
|
||||||
name: id
|
|
||||||
type: int
|
|
||||||
remarks: Unique ID
|
|
||||||
autoIncrement: true
|
|
||||||
constraints:
|
|
||||||
primaryKey: true
|
|
||||||
nullable: false
|
|
||||||
- column:
|
|
||||||
name: source_entity_type
|
|
||||||
type: varchar(50)
|
|
||||||
remarks: "Type of the source entity (card or table)"
|
|
||||||
constraints:
|
|
||||||
nullable: false
|
|
||||||
- column:
|
|
||||||
name: source_entity_id
|
|
||||||
type: int
|
|
||||||
remarks: "ID of the source entity"
|
|
||||||
constraints:
|
|
||||||
nullable: false
|
|
||||||
- column:
|
|
||||||
name: target_entity_type
|
|
||||||
type: varchar(50)
|
|
||||||
remarks: "Type of the target entity (card or table)"
|
|
||||||
constraints:
|
|
||||||
nullable: false
|
|
||||||
- column:
|
|
||||||
name: target_entity_id
|
|
||||||
type: int
|
|
||||||
remarks: "ID of the target entity"
|
|
||||||
constraints:
|
|
||||||
nullable: false
|
|
||||||
- column:
|
|
||||||
name: status
|
|
||||||
type: varchar(50)
|
|
||||||
remarks: "Current status (started, succeeded, failed, canceled, timeout)"
|
|
||||||
constraints:
|
|
||||||
nullable: false
|
|
||||||
- column:
|
|
||||||
name: is_active
|
|
||||||
type: ${boolean.type}
|
|
||||||
remarks: "True only for currently running replacements, null for completed"
|
|
||||||
constraints:
|
|
||||||
nullable: true
|
|
||||||
- column:
|
|
||||||
name: progress
|
|
||||||
type: float
|
|
||||||
remarks: "Progress from 0.0 to 1.0"
|
|
||||||
constraints:
|
|
||||||
nullable: true
|
|
||||||
- column:
|
|
||||||
name: message
|
|
||||||
type: ${text.type}
|
|
||||||
remarks: "Human-readable message; may contain error info"
|
|
||||||
constraints:
|
|
||||||
nullable: true
|
|
||||||
- column:
|
|
||||||
name: user_id
|
|
||||||
type: int
|
|
||||||
remarks: "User who initiated the replacement"
|
|
||||||
constraints:
|
|
||||||
nullable: true
|
|
||||||
- column:
|
|
||||||
name: start_time
|
|
||||||
type: ${timestamp_type}
|
|
||||||
remarks: "When the replacement started"
|
|
||||||
defaultValueComputed: current_timestamp
|
|
||||||
constraints:
|
|
||||||
nullable: false
|
|
||||||
- column:
|
|
||||||
name: end_time
|
|
||||||
type: ${timestamp_type}
|
|
||||||
remarks: "When the replacement completed (null if still running)"
|
|
||||||
constraints:
|
|
||||||
nullable: true
|
|
||||||
|
|
||||||
- changeSet:
|
|
||||||
id: v60.2026-03-06T12:00:01
|
|
||||||
author: bcm
|
|
||||||
comment: Enforce at most one active source replacement run cluster-wide
|
|
||||||
changes:
|
|
||||||
- addUniqueConstraint:
|
|
||||||
tableName: source_replacement_run
|
|
||||||
constraintName: idx_unique_active_source_replacement_run
|
|
||||||
columnNames: is_active
|
|
||||||
|
|
||||||
- changeSet:
|
|
||||||
id: v60.2026-03-06T12:00:02
|
|
||||||
author: bcm
|
|
||||||
comment: Add index on source_replacement_run.user_id
|
|
||||||
changes:
|
|
||||||
- createIndex:
|
|
||||||
tableName: source_replacement_run
|
|
||||||
indexName: idx_source_replacement_run_user_id
|
|
||||||
columns:
|
|
||||||
- column:
|
|
||||||
name: user_id
|
|
||||||
|
|
||||||
- changeSet:
|
|
||||||
id: v60.2026-03-06T12:00:03
|
|
||||||
author: bcm
|
|
||||||
comment: Add foreign key on source_replacement_run(user_id)
|
|
||||||
changes:
|
|
||||||
- addForeignKeyConstraint:
|
|
||||||
baseTableName: source_replacement_run
|
|
||||||
baseColumnNames: user_id
|
|
||||||
referencedTableName: core_user
|
|
||||||
referencedColumnNames: id
|
|
||||||
constraintName: fk_source_replacement_run_user_id
|
|
||||||
|
|
||||||
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
|
|
||||||
|
|
||||||
########################################################################################################################
|
|
||||||
#
|
|
||||||
# ADVICE:
|
|
||||||
#
|
|
||||||
# 1) Think through some of these points when writing a migration, learn from our past mistakes:
|
|
||||||
# - Do you really need a migration? Could the feature work without it? Prefer not to if possible.
|
|
||||||
# Data in the wild can be vastly different from what you expect, and it's hard to get right.
|
|
||||||
# - Make sure your migration is backward compatible: it might not be possible to add a constraint back
|
|
||||||
# if you drop it in a migration.
|
|
||||||
# - Postgres, MySQL and H2 have their differences. Make sure your migration works for all.
|
|
||||||
# - Database encryption is a major issue:
|
|
||||||
# - Fields like `metabase_database.details` or `setting.value` can be encrypted, so you need to decrypt them in
|
|
||||||
# your migration. See #42617, #44048.
|
|
||||||
# - Database could be partially encrypted, see https://www.notion.so/72575933ef2a446bafd16909e05a7387
|
|
||||||
# - Custom migrations:
|
|
||||||
# - Prefer SQL migrations when possible.
|
|
||||||
# - Never use application code: it can change and *will* break your migration.
|
|
||||||
# - Do not use Toucan models - refer table names directly.
|
|
||||||
# - If it's a big change, consider using Quartz, see #42279
|
|
||||||
# - More in `metabase.app_db.custom_migrations` namespace doc.
|
|
||||||
# - Never delete a migration: users won't be able to downgrade. If you really need to, see #44908
|
|
||||||
# - Migration id (`vXX.<date>`) must match its earliest released version:
|
|
||||||
# - Do not backport `v51....` to version 50, Metabase will try to downgrade it.
|
|
||||||
# - Instead, write a migration with an oldest target you plan to backport to in mind.
|
|
||||||
#
|
|
||||||
# 2) Migrations should go in the ###_update_migrations.yaml file for the target version.
|
|
||||||
#
|
|
||||||
# 3) Run mage lint-migrations to run core.spec checks against any changes you make here. Liquibase is pretty
|
|
||||||
# forgiving and won't complain if you accidentally mix up things like deleteCascade and onDelete: CASCADE. CI runs
|
|
||||||
# this check but it's nicer to know now instead of waiting for CI.
|
|
||||||
#
|
|
||||||
# 3) Migration IDs should follow the format
|
|
||||||
#
|
|
||||||
# vMM.TIMESTAMP
|
|
||||||
#
|
|
||||||
# Where
|
|
||||||
#
|
|
||||||
# M = major version
|
|
||||||
# TIMESTAMP = the current timestamp with format `yyyy-MM-dd'T'HH:mm:ss`
|
|
||||||
# To get this timestamp, evaluate this in your REPL: `(dev/migration-timestamp)`
|
|
||||||
#
|
|
||||||
# E.g: You're adding a new migration for version 49, And it's 10:30:00AM on April 1, 2023 (UTC),
|
|
||||||
# your migration id should be: `v49.2023-04-01T10:30:00`.
|
|
||||||
#
|
|
||||||
# PLEASE KEEP THIS MESSAGE AT THE BOTTOM OF THIS FILE!!!!! Add new migrations above the message.
|
|
||||||
#
|
|
||||||
########################################################################################################################
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
--------
|
|
||||||
THIS IS THE TEMPLATE MIGRATION FILE FOR NEW MAJOR VERSION OF METABASE
|
|
||||||
On each new major version, copy this file to `resources/migrations/###_update_migrations.yaml` where `###` is the major metabase version.
|
|
||||||
The file can be empty until migrations need to be added.
|
|
||||||
--------
|
|
||||||
|
|
||||||
databaseChangeLog:
|
|
||||||
- objectQuotingStrategy: QUOTE_ALL_OBJECTS
|
|
||||||
|
|
||||||
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
|
|
||||||
|
|
||||||
########################################################################################################################
|
|
||||||
#
|
|
||||||
# ADVICE:
|
|
||||||
#
|
|
||||||
# 1) Think through some of these points when writing a migration, learn from our past mistakes:
|
|
||||||
# - Do you really need a migration? Could the feature work without it? Prefer not to if possible.
|
|
||||||
# Data in the wild can be vastly different from what you expect, and it's hard to get right.
|
|
||||||
# - Make sure your migration is backward compatible: it might not be possible to add a constraint back
|
|
||||||
# if you drop it in a migration.
|
|
||||||
# - Postgres, MySQL and H2 have their differences. Make sure your migration works for all.
|
|
||||||
# - Database encryption is a major issue:
|
|
||||||
# - Fields like `metabase_database.details` or `setting.value` can be encrypted, so you need to decrypt them in
|
|
||||||
# your migration. See #42617, #44048.
|
|
||||||
# - Database could be partially encrypted, see https://www.notion.so/72575933ef2a446bafd16909e05a7387
|
|
||||||
# - Custom migrations:
|
|
||||||
# - Prefer SQL migrations when possible.
|
|
||||||
# - Never use application code: it can change and *will* break your migration.
|
|
||||||
# - Do not use Toucan models - refer table names directly.
|
|
||||||
# - If it's a big change, consider using Quartz, see #42279
|
|
||||||
# - More in `metabase.app_db.custom_migrations` namespace doc.
|
|
||||||
# - Never delete a migration: users won't be able to downgrade. If you really need to, see #44908
|
|
||||||
# - Migration id (`vXX.<date>`) must match its earliest released version:
|
|
||||||
# - Do not backport `v51....` to version 50, Metabase will try to downgrade it.
|
|
||||||
# - Instead, write a migration with an oldest target you plan to backport to in mind.
|
|
||||||
#
|
|
||||||
# 2) Migrations should go in the ###_update_migrations.yaml file for the target version.
|
|
||||||
#
|
|
||||||
# 3) Run mage lint-migrations-file to run core.spec checks against any changes you make here. Liquibase is pretty
|
|
||||||
# forgiving and won't complain if you accidentally mix up things like deleteCascade and onDelete: CASCADE. CI runs
|
|
||||||
# this check but it's nicer to know now instead of waiting for CI.
|
|
||||||
#
|
|
||||||
# 3) Migration IDs should follow the format
|
|
||||||
#
|
|
||||||
# vMM.TIMESTAMP
|
|
||||||
#
|
|
||||||
# Where
|
|
||||||
#
|
|
||||||
# M = major version
|
|
||||||
# TIMESTAMP = the current timestamp with format `yyyy-MM-dd'T'HH:mm:ss`
|
|
||||||
# To get this timestamp, evaluate this in your REPL: `(dev/migration-timestamp)`
|
|
||||||
#
|
|
||||||
# E.g: You're adding a new migration for version 49, And it's 10:30:00AM on April 1, 2023 (UTC),
|
|
||||||
# your migration id should be: `v49.2023-04-01T10:30:00`.
|
|
||||||
#
|
|
||||||
# PLEASE KEEP THIS MESSAGE AT THE BOTTOM OF THIS FILE!!!!! Add new migrations above the message.
|
|
||||||
#
|
|
||||||
########################################################################################################################
|
|
||||||
@@ -29,7 +29,7 @@
|
|||||||
(liquibase.changelog.filter AlreadyRanChangeSetFilter ChangeSetFilter ChangeSetFilterResult DbmsChangeSetFilter IgnoreChangeSetFilter)
|
(liquibase.changelog.filter AlreadyRanChangeSetFilter ChangeSetFilter ChangeSetFilterResult DbmsChangeSetFilter IgnoreChangeSetFilter)
|
||||||
(liquibase.changelog.visitor AbstractChangeExecListener ChangeExecListener UpdateVisitor)
|
(liquibase.changelog.visitor AbstractChangeExecListener ChangeExecListener UpdateVisitor)
|
||||||
(liquibase.command.core AbstractRollbackCommandStep)
|
(liquibase.command.core AbstractRollbackCommandStep)
|
||||||
(liquibase.database Database DatabaseFactory)
|
(liquibase.database Database DatabaseFactory ObjectQuotingStrategy)
|
||||||
(liquibase.database.jvm JdbcConnection)
|
(liquibase.database.jvm JdbcConnection)
|
||||||
(liquibase.exception LockException)
|
(liquibase.exception LockException)
|
||||||
(liquibase.lockservice LockService LockServiceFactory)
|
(liquibase.lockservice LockService LockServiceFactory)
|
||||||
@@ -122,10 +122,11 @@
|
|||||||
(.findCorrectDatabaseImplementation (DatabaseFactory/getInstance) liquibase-conn)))
|
(.findCorrectDatabaseImplementation (DatabaseFactory/getInstance) liquibase-conn)))
|
||||||
|
|
||||||
(defn- liquibase ^Liquibase [^Connection conn ^Database database]
|
(defn- liquibase ^Liquibase [^Connection conn ^Database database]
|
||||||
(Liquibase.
|
(u/prog1 (Liquibase.
|
||||||
^String (decide-liquibase-file conn database)
|
^String (decide-liquibase-file conn database)
|
||||||
(ClassLoaderResourceAccessor. (classloader/the-classloader))
|
(ClassLoaderResourceAccessor. (classloader/the-classloader))
|
||||||
database))
|
database)
|
||||||
|
(.setObjectQuotingStrategy (.getDatabaseChangeLog <>) ObjectQuotingStrategy/QUOTE_ALL_OBJECTS)))
|
||||||
|
|
||||||
(mu/defn do-with-liquibase
|
(mu/defn do-with-liquibase
|
||||||
"Impl for [[with-liquibase-macro]]."
|
"Impl for [[with-liquibase-macro]]."
|
||||||
@@ -456,6 +457,10 @@
|
|||||||
liquibase was checking the jar signature for each file. This function is required to correct the liquibase tables to
|
liquibase was checking the jar signature for each file. This function is required to correct the liquibase tables to
|
||||||
reflect that these migrations were grouped into 2 files.
|
reflect that these migrations were grouped into 2 files.
|
||||||
|
|
||||||
|
NOTE: we are going back to more granular changelog files in v60, but should not as many as before.
|
||||||
|
If we do end up seeing a performance issue similar to what caused a need for this file, we can do a similar
|
||||||
|
(but different) consolidation function for the v60+ changelogs.
|
||||||
|
|
||||||
See https://github.com/metabase/metabase/issues/3715
|
See https://github.com/metabase/metabase/issues/3715
|
||||||
Also see https://github.com/metabase/metabase/pull/34400"
|
Also see https://github.com/metabase/metabase/pull/34400"
|
||||||
[conn :- (ms/InstanceOfClass Connection)
|
[conn :- (ms/InstanceOfClass Connection)
|
||||||
|
|||||||
@@ -63,7 +63,7 @@
|
|||||||
(liquibase/consolidate-liquibase-changesets! conn liquibase)
|
(liquibase/consolidate-liquibase-changesets! conn liquibase)
|
||||||
|
|
||||||
(testing "makes sure the change log filename are correctly set"
|
(testing "makes sure the change log filename are correctly set"
|
||||||
(is (= (set (mdb.test-util/liquibase-file->included-ids "migrations/000_legacy_migrations.yaml" driver/*driver* conn))
|
(is (= (set (mdb.test-util/liquibase-file->included-ids "liquibase_legacy_migrations.yaml" driver/*driver* conn))
|
||||||
(t2/select-fn-set :id table-name :filename "migrations/000_legacy_migrations.yaml")))
|
(t2/select-fn-set :id table-name :filename "migrations/000_legacy_migrations.yaml")))
|
||||||
|
|
||||||
(is (= (set (mdb.test-util/liquibase-file->included-ids "migrations/001_update_migrations.yaml" driver/*driver* conn))
|
(is (= (set (mdb.test-util/liquibase-file->included-ids "migrations/001_update_migrations.yaml" driver/*driver* conn))
|
||||||
@@ -156,7 +156,8 @@
|
|||||||
(deftest extract-numbers-special-case-test
|
(deftest extract-numbers-special-case-test
|
||||||
(testing "when specific migration verison is passed reports different major version"
|
(testing "when specific migration verison is passed reports different major version"
|
||||||
(is (= 55 (first (#'liquibase/extract-numbers "v56.2025-06-05T16:48:48"))))
|
(is (= 55 (first (#'liquibase/extract-numbers "v56.2025-06-05T16:48:48"))))
|
||||||
(is (= 55 (first (#'liquibase/extract-numbers "v56.2025-05-19T16:48:48"))))))
|
(is (= 55 (first (#'liquibase/extract-numbers "v56.2025-05-19T16:48:48"))))
|
||||||
|
(is (= 60 (first (#'liquibase/extract-numbers "v60.ghdf99efd"))))))
|
||||||
|
|
||||||
(deftest rollback-major-version
|
(deftest rollback-major-version
|
||||||
(mt/test-drivers #{:h2 :mysql :rollback}
|
(mt/test-drivers #{:h2 :mysql :rollback}
|
||||||
|
|||||||
@@ -176,14 +176,14 @@
|
|||||||
|
|
||||||
;; the latest changeSet in `000_legacy_migrations.yaml` is `v44.00-044`. We can simulate a downgrade to that
|
;; the latest changeSet in `000_legacy_migrations.yaml` is `v44.00-044`. We can simulate a downgrade to that
|
||||||
;; version by telling Liquibase that's the migrations file.
|
;; version by telling Liquibase that's the migrations file.
|
||||||
(with-redefs [liquibase/decide-liquibase-file (fn [& _args] "migrations/000_legacy_migrations.yaml")]
|
(with-redefs [liquibase/decide-liquibase-file (fn [& _args] "liquibase_legacy_migrations.yaml")]
|
||||||
(is (thrown-with-msg?
|
(is (thrown-with-msg?
|
||||||
Exception #"You must run `java --add-opens java.base/java.nio=ALL-UNNAMED -jar metabase.jar migrate down` from version 45."
|
Exception #"You must run `java --add-opens java.base/java.nio=ALL-UNNAMED -jar metabase.jar migrate down` from version 45."
|
||||||
(#'mdb.setup/error-if-downgrade-required! (mdb.connection/data-source)))))
|
(#'mdb.setup/error-if-downgrade-required! (mdb.connection/data-source)))))
|
||||||
|
|
||||||
;; check that the error correctly reports the version to run `downgrade` from
|
;; check that the error correctly reports the version to run `downgrade` from
|
||||||
(update-to-changelog-id "v46.00-001" conn)
|
(update-to-changelog-id "v46.00-001" conn)
|
||||||
(with-redefs [liquibase/decide-liquibase-file (fn [& _args] "migrations/000_legacy_migrations.yaml")]
|
(with-redefs [liquibase/decide-liquibase-file (fn [& _args] "liquibase_legacy_migrations.yaml")]
|
||||||
(is (thrown-with-msg?
|
(is (thrown-with-msg?
|
||||||
Exception #"You must run `java --add-opens java.base/java.nio=ALL-UNNAMED -jar metabase.jar migrate down` from version 46."
|
Exception #"You must run `java --add-opens java.base/java.nio=ALL-UNNAMED -jar metabase.jar migrate down` from version 46."
|
||||||
(#'mdb.setup/error-if-downgrade-required! (mdb.connection/data-source))))))))
|
(#'mdb.setup/error-if-downgrade-required! (mdb.connection/data-source))))))))
|
||||||
|
|||||||
@@ -92,10 +92,23 @@
|
|||||||
"Returns a list of existing migration files."
|
"Returns a list of existing migration files."
|
||||||
[include-legacy?]
|
[include-legacy?]
|
||||||
(into (if include-legacy?
|
(into (if include-legacy?
|
||||||
["migrations/000_legacy_migrations.yaml" "migrations/001_update_migrations.yaml"]
|
["liquibase_legacy_migrations.yaml" "migrations/001_update_migrations.yaml"]
|
||||||
["migrations/001_update_migrations.yaml"])
|
["migrations/001_update_migrations.yaml"])
|
||||||
(filter io/resource (for [n (range 56 100)]
|
(concat
|
||||||
(format "migrations/%03d_update_migrations.yaml" n)))))
|
;; Per-release migration files (v56-v59 pattern)
|
||||||
|
(filter io/resource (for [n (range 56 100)]
|
||||||
|
(format "migrations/%03d_update_migrations.yaml" n)))
|
||||||
|
;; Directory-based migration files (v60+ pattern)
|
||||||
|
(let [migrations-dir (io/resource "migrations")]
|
||||||
|
(when migrations-dir
|
||||||
|
(->> (io/file migrations-dir)
|
||||||
|
file-seq
|
||||||
|
(filter (fn [^java.io.File f]
|
||||||
|
(and (.isFile f)
|
||||||
|
(re-matches #".*\d{3}/\d{8}_.+\.yaml$" (str f)))))
|
||||||
|
sort
|
||||||
|
(map (fn [^java.io.File f]
|
||||||
|
(str "migrations/" (.getName (.getParentFile f)) "/" (.getName f))))))))))
|
||||||
|
|
||||||
(defn all-liquibase-ids
|
(defn all-liquibase-ids
|
||||||
"Returns a set of all changeset IDs from all migration files."
|
"Returns a set of all changeset IDs from all migration files."
|
||||||
|
|||||||
Reference in New Issue
Block a user