Compare commits

...

1 Commits

Author SHA1 Message Date
Cal Herries
246afa6c64 WIP 2024-07-31 23:55:50 +02:00
10 changed files with 143 additions and 98 deletions

View File

@@ -16,12 +16,12 @@
(log/warnf "Database not found. Metabase will create a new database at %s and proceeed encrypting." "2")
(mdb/setup-db! :create-sample-content? true))
(log/infof "Connected to: %s | %s" (mdb/db-type) (mdb/db-file))
(let [make-encrypt-fn (fn [maybe-encrypt-fn]
(let [make-encrypt-fn (fn [encrypt-fn]
(if to-key
(partial maybe-encrypt-fn (encryption/validate-and-hash-secret-key to-key))
(partial encrypt-fn (encryption/validate-and-hash-secret-key to-key))
identity))
encrypt-str-fn (make-encrypt-fn encryption/maybe-encrypt)
encrypt-bytes-fn (make-encrypt-fn encryption/maybe-encrypt-bytes)]
encrypt-str-fn (make-encrypt-fn encryption/encrypt)
encrypt-bytes-fn (make-encrypt-fn encryption/encrypt-bytes)]
(t2/with-transaction [t-conn {:datasource (mdb/data-source)}]
(doseq [[id details] (t2/select-pk->fn :details Database)]
(when (encryption/possibly-encrypted-string? details)

View File

@@ -125,12 +125,12 @@
(def ^:private encrypted-json-in
"Should mirror [[metabase.models.interface/encrypted-json-in]]"
(comp encryption/maybe-encrypt json-in))
(comp encryption/encrypt json-in))
(defn- encrypted-json-out
"Should mirror [[metabase.models.interface/encrypted-json-out]]"
[v]
(let [decrypted (encryption/maybe-decrypt v)]
(let [decrypted (encryption/decrypt v)]
(try
(json/parse-string decrypted true)
(catch Throwable e
@@ -1433,7 +1433,7 @@
(defn- raw-setting-value [key]
(some-> (t2/query-one {:select [:value], :from :setting, :where [:= :key key]})
:value
encryption/maybe-decrypt))
encryption/decrypt))
(define-reversible-migration MigrateUploadsSettings
(do (when (some-> (raw-setting-value "uploads-enabled") parse-boolean)
@@ -1451,10 +1451,10 @@
"uploads-schema-name"
"uploads-table-prefix"]]}))
(when-let [db (t2/query-one {:select [:*], :from :metabase_database, :where :uploads_enabled})]
(let [settings [{:key "uploads-database-id", :value (encryption/maybe-encrypt (str (:id db)))}
{:key "uploads-enabled", :value (encryption/maybe-encrypt "true")}
{:key "uploads-table-prefix", :value (encryption/maybe-encrypt (:uploads_table_prefix db))}
{:key "uploads-schema-name", :value (encryption/maybe-encrypt (:uploads_schema_name db))}]]
(let [settings [{:key "uploads-database-id", :value (encryption/encrypt (str (:id db)))}
{:key "uploads-enabled", :value (encryption/encrypt "true")}
{:key "uploads-table-prefix", :value (encryption/encrypt (:uploads_table_prefix db))}
{:key "uploads-schema-name", :value (encryption/encrypt (:uploads_schema_name db))}]]
(->> settings
(filter :value)
(t2/insert! :setting)))))
@@ -1465,3 +1465,55 @@
(run! decrypt! ["query-caching-ttl-ratio"
"query-caching-min-ttl"
"enable-query-caching"])))
(define-reversible-migration EncryptAllDocuments
(let [encrypt-fn (fn [value]
(if (encryption/possibly-encrypted-string? value)
value
(encryption/encrypt value)))
encrypt-bytes-fn (fn [value]
(if (encryption/possibly-encrypted-bytes? value)
value
(encryption/encrypt-bytes value)))]
;; Forward migration
(t2/with-transaction [t-conn {:datasource (mdb/data-source)}]
;; Encrypt Database details
(doseq [[id details] (t2/select-pk->fn :details Database)]
(t2/update! :conn t-conn :metabase_database
{:id id}
{:details (encrypt-fn (json/encode details))}))
;; Encrypt Setting values
(doseq [[key value] (t2/select-fn->fn :key :value Setting)]
(t2/update! :conn t-conn :setting
{:key key}
{:value (encrypt-fn value)}))
;; Encrypt Secret values
(doseq [[id value] (t2/select-pk->fn :value Secret)]
(t2/update! :conn t-conn :secret
{:id id}
{:value (encrypt-bytes-fn value)}))))
;; Reverse migration
(let [decrypt-fn (fn [value]
(if (encryption/possibly-encrypted-string? value)
(encryption/decrypt value)
value))
decrypt-bytes-fn (fn [value]
(if (encryption/possibly-encrypted-bytes? value)
(encryption/decrypt-bytes value)
value))]
(t2/with-transaction [t-conn {:datasource (mdb/data-source)}]
;; Decrypt Database details
(doseq [[id details] (t2/select-pk->fn :details Database)]
(t2/update! :conn t-conn :metabase_database
{:id id}
{:details (decrypt-fn details)}))
;; Decrypt Setting values
(doseq [[key value] (t2/select-fn->fn :key :value Setting)]
(t2/update! :conn t-conn :setting
{:key key}
{:value (decrypt-fn value)}))
;; Decrypt Secret values
(doseq [[id value] (t2/select-pk->fn :value Secret)]
(t2/update! :conn t-conn :secret
{:id id}
{:value (decrypt-bytes-fn value)})))))

View File

@@ -253,12 +253,12 @@
(def encrypted-json-in
"Serialize encrypted json."
(comp encryption/maybe-encrypt json-in))
(comp encryption/encrypt json-in))
(defn encrypted-json-out
"Deserialize encrypted json."
[v]
(let [decrypted (encryption/maybe-decrypt v)]
(let [decrypted (encryption/decrypt v)]
(try
(json/parse-string decrypted true)
(catch Throwable e
@@ -387,8 +387,8 @@
(def transform-secret-value
"Transform for secret value."
{:in (comp encryption/maybe-encrypt-bytes codecs/to-bytes)
:out (comp encryption/maybe-decrypt maybe-blob->bytes)})
{:in (comp encryption/encrypt-bytes codecs/to-bytes)
:out (comp encryption/decrypt maybe-blob->bytes)})
(defn decompress
"Decompress `compressed-bytes`."

View File

@@ -1539,14 +1539,14 @@
;; use a raw query to use `:for :update`
(t2/with-transaction [_conn]
(when-let [v (t2/select-one-fn :value :setting :key (setting-name setting) {:for :update})]
(when (not= (encryption/maybe-decrypt v) v)
(when (not= (encryption/decrypt v) v)
;; similarly, use `:setting` vs `:model/Setting` here to ensure the update is actually run even though Toucan
;; thinks nothing has changed
(t2/update! :setting :key (setting-name setting) {:value (encryption/maybe-decrypt v)}))))))
(t2/update! :setting :key (setting-name setting) {:value (encryption/decrypt v)}))))))
(defn- maybe-encrypt [setting-model]
(defn- encrypt [setting-model]
;; In tests, sometimes we need to insert/update settings that don't have definitions in the code and therefore can't
;; be resolved. Fall back to maybe-encrypting these.
;; be resolved. Fall back to encrypting these.
(let [resolved (try (resolve-setting (:key setting-model))
(catch clojure.lang.ExceptionInfo e
(when (not (::unknown-setting-error (ex-data e)))
@@ -1554,16 +1554,16 @@
(cond-> setting-model
(or (nil? resolved)
(not (prohibits-encryption? resolved)))
(update :value encryption/maybe-encrypt))))
(update :value encryption/encrypt))))
(t2/define-before-update :model/Setting
[setting]
(maybe-encrypt setting))
(encrypt setting))
(t2/define-before-insert :model/Setting
[setting]
(maybe-encrypt setting))
(encrypt setting))
(t2/define-after-select :model/Setting
[setting]
(update setting :value encryption/maybe-decrypt))
(update setting :value encryption/decrypt))

View File

@@ -26,24 +26,22 @@
(defn validate-and-hash-secret-key
"Check the minimum length of the key and hash it for internal usage."
[^String secret-key]
(when-let [secret-key secret-key]
(when (seq secret-key)
(assert (>= (count secret-key) 16)
(str (trs "MB_ENCRYPTION_SECRET_KEY must be at least 16 characters.")))
(secret-key->hash secret-key))))
(assert (>= (count secret-key) 16)
(str (trs "MB_ENCRYPTION_SECRET_KEY must be at least 16 characters.")))
(secret-key->hash secret-key))
;; apperently if you're not tagging in an arglist, `^bytes` will set the `:tag` metadata to `clojure.core/bytes` (ick)
;; so you have to do `^{:tag 'bytes}` instead
(defonce ^:private ^{:tag 'bytes} default-secret-key
(validate-and-hash-secret-key (env/env :mb-encryption-secret-key)))
(validate-and-hash-secret-key (or (env/env :mb-encryption-secret-key) "DEFAULT_SECRET_KEY")))
;; log a nice message letting people know whether DB details encryption is enabled
(when-not *compile-files*
(log/info
(if default-secret-key
(if (env/env :mb-encryption-secret-key)
"Saved credentials encryption is ENABLED for this Metabase instance."
"Saved credentials encryption is DISABLED for this Metabase instance.")
(u/emoji (if default-secret-key "🔐" "🔓"))
(u/emoji (if (env/env :mb-encryption-secret-key) "🔐" "🔓"))
"\n"
"For more information, see https://metabase.com/docs/latest/operations-guide/encrypting-database-details-at-rest.html"))
@@ -72,12 +70,19 @@
(encrypt-bytes secret-key)
codec/base64-encode)))
(defn default-encrypted-bytes?
"Returns true if bytes have been encrypted with a default secret key (not ), otherwise false."
[^bytes b]
:nyi)
(defn decrypt-bytes
"Decrypt bytes `b` using a `secret-key` (a 64-byte byte array), which by default is the hashed value of
`MB_ENCRYPTION_SECRET_KEY`."
{:added "0.41.0"}
(^String [^bytes b]
(decrypt-bytes default-secret-key b))
(if (default-encrypted-bytes? b)
:nyi
(decrypt-bytes default-secret-key b)))
(^String [secret-key, ^bytes b]
(let [[initialization-vector message] (split-at 16 b)]
(crypto/decrypt (byte-array message)
@@ -93,28 +98,6 @@
(^String [secret-key, ^String s]
(codecs/bytes->str (decrypt-bytes secret-key (codec/base64-decode s)))))
(defn maybe-encrypt
"If `MB_ENCRYPTION_SECRET_KEY` is set, return an encrypted version of `s`; otherwise return `s` as-is."
(^String [^String s]
(maybe-encrypt default-secret-key s))
(^String [secret-key, ^String s]
(if secret-key
(when (seq s)
(encrypt secret-key s))
s)))
(defn maybe-encrypt-bytes
"If `MB_ENCRYPTION_SECRET_KEY` is set, return an encrypted version of the given bytes `b`; otherwise return `b`
as-is."
{:added "0.41.0"}
(^bytes [^bytes b]
(maybe-encrypt-bytes default-secret-key b))
(^bytes [secret-key, ^bytes b]
(if secret-key
(when (seq b)
(encrypt-bytes secret-key b))
b)))
(def ^:private ^:const aes256-tag-length 32)
(def ^:private ^:const aes256-block-size 16)
@@ -142,7 +125,19 @@
(codec/base64-decode s))]
(possibly-encrypted-bytes? b))))
(defn maybe-decrypt
(defn default-encrypted-string?
"Returns true if it's likely that `s` is an encrypted string. Specifically we need `s` to be a non-blank, base64
encoded string of the correct length. See docstring for `possibly-encrypted-bytes?` for an explanation of correct
length."
[^String s]
(u/ignore-exceptions
(when-let [b (and (not (str/blank? s))
(u/base64-string? s)
(codec/base64-decode s))]
(possibly-encrypted-bytes? b))))
(defn decrypt
"If `MB_ENCRYPTION_SECRET_KEY` is set and `v` is encrypted, decrypt `v`; otherwise return `s` as-is. Attempts to check
whether `v` is an encrypted String, in which case the decrypted String is returned, or whether `v` is encrypted bytes,
in which case the decrypted bytes are returned."

View File

@@ -2146,15 +2146,13 @@
(testing "database_enabled_actions should hydrate according to database-enable-actions setting"
(mt/test-drivers (mt/normal-drivers-with-feature :actions)
(mt/with-actions-test-data
(doseq [enable-actions? [true false]
encrypt-db? [true false]]
(mt/with-temp-env-var-value! [mb-encryption-secret-key encrypt-db?]
(mt/with-temp-vals-in-db Database (mt/id) {:settings {:database-enable-actions enable-actions?}}
(mt/with-actions [{:keys [action-id]} {:type :query :visualization_settings {:hello true}}]
(mt/with-temp [Dashboard {dashboard-id :id} {}
DashboardCard _ {:action_id action-id, :dashboard_id dashboard-id}]
(is (partial= {:dashcards [{:action {:database_enabled_actions enable-actions?}}]}
(mt/user-http-request :crowberto :get 200 (format "dashboard/%s" dashboard-id)))))))))))))
(doseq [enable-actions? [true false]]
(mt/with-temp-vals-in-db Database (mt/id) {:settings {:database-enable-actions enable-actions?}}
(mt/with-actions [{:keys [action-id]} {:type :query :visualization_settings {:hello true}}]
(mt/with-temp [Dashboard {dashboard-id :id} {}
DashboardCard _ {:action_id action-id, :dashboard_id dashboard-id}]
(is (partial= {:dashcards [{:action {:database_enabled_actions enable-actions?}}]}
(mt/user-http-request :crowberto :get 200 (format "dashboard/%s" dashboard-id))))))))))))
(deftest add-card-parameter-mapping-permissions-test
(testing "PUT /api/dashboard/:id"

View File

@@ -1719,7 +1719,7 @@
;; run migration to the latest migration
(impl/test-migrations ["v49.2024-04-09T10:00:03"] [migrate!]
;; create a db because db.details should be encrypted
(let [db-id (:id (new-instance-with-default :metabase_database {:details (encryption/maybe-encrypt "{}")}))
(let [db-id (:id (new-instance-with-default :metabase_database {:details (encryption/encrypt "{}")}))
db-detail (fn []
(:details (t2/query-one {:select [:details]
:from [:metabase_database]
@@ -1899,10 +1899,10 @@
(impl/test-migrations ["v50.2024-05-17T19:54:26"] [migrate!]
(let [uploads-db-id (t2/insert-returning-pk! :metabase_database (assoc migrate-uploads-default-db :name "DB 1"))
not-uploads-db-id (t2/insert-returning-pk! :metabase_database (assoc migrate-uploads-default-db :name "DB 2"))]
(let [settings [{:key "uploads-database-id", :value (encryption/maybe-encrypt (str uploads-db-id))}
{:key "uploads-enabled", :value (encryption/maybe-encrypt "true")}
{:key "uploads-table-prefix", :value (encryption/maybe-encrypt "uploads_")}
{:key "uploads-schema-name", :value (encryption/maybe-encrypt "uploads")}]
(let [settings [{:key "uploads-database-id", :value (encryption/encrypt (str uploads-db-id))}
{:key "uploads-enabled", :value (encryption/encrypt "true")}
{:key "uploads-table-prefix", :value (encryption/encrypt "uploads_")}
{:key "uploads-schema-name", :value (encryption/encrypt "uploads")}]
_ (t2/insert! :setting settings)
get-settings #(t2/query {:select [:key :value], :from :setting, :where [:in :key (map :key settings)]})
settings-before (get-settings)]
@@ -1923,8 +1923,8 @@
(is (not-empty settings-after))
(is (every? encryption/possibly-encrypted-string?
(map :value settings-after)))
(is (= (set (map #(update % :value encryption/maybe-decrypt) settings-before))
(set (map #(update % :value encryption/maybe-decrypt) settings-after)))))))))))))
(is (= (set (map #(update % :value encryption/decrypt) settings-before))
(set (map #(update % :value encryption/decrypt) settings-after)))))))))))))
(deftest migrate-uploads-settings-test-2
(testing "MigrateUploadsSettings with invalid settings state (missing uploads-database-id) doesn't fail."
@@ -1932,8 +1932,8 @@
(impl/test-migrations ["v50.2024-05-17T19:54:26"] [migrate!]
(let [uploads-db-id (t2/insert-returning-pk! :metabase_database migrate-uploads-default-db)
settings [;; no uploads-database-id and uploads-schema-name
{:key "uploads-enabled", :value (encryption/maybe-encrypt "true")}
{:key "uploads-table-prefix", :value (encryption/maybe-encrypt "uploads_")}]
{:key "uploads-enabled", :value (encryption/encrypt "true")}
{:key "uploads-table-prefix", :value (encryption/encrypt "uploads_")}]
_ (t2/insert! :setting settings)
get-settings #(t2/query {:select [:key :value], :from :setting, :where [:in :key (map :key settings)]})]
(migrate!)
@@ -1950,7 +1950,7 @@
(impl/test-migrations ["v50.2024-05-17T19:54:26"] [migrate!]
(let [uploads-db-id (t2/insert-returning-pk! :metabase_database migrate-uploads-default-db)
settings [;; no uploads-enabled
{:key "uploads-database-id", :value (encryption/maybe-encrypt "uploads_")}]
{:key "uploads-database-id", :value (encryption/encrypt "uploads_")}]
_ (t2/insert! :setting settings)
get-settings #(t2/query {:select [:key :value], :from :setting, :where [:in :key (map :key settings)]})]
(migrate!)
@@ -2003,9 +2003,9 @@
(deftest decrypt-cache-settings-test
(impl/test-migrations "v50.2024-06-12T12:33:07" [migrate!]
(encryption-test/with-secret-key "whateverwhatever"
(t2/insert! :setting [{:key "enable-query-caching", :value (encryption/maybe-encrypt "true")}
{:key "query-caching-ttl-ratio", :value (encryption/maybe-encrypt "100")}
{:key "query-caching-min-ttl", :value (encryption/maybe-encrypt "123")}]))
(t2/insert! :setting [{:key "enable-query-caching", :value (encryption/encrypt "true")}
{:key "query-caching-ttl-ratio", :value (encryption/encrypt "100")}
{:key "query-caching-min-ttl", :value (encryption/encrypt "123")}]))
(testing "Values were indeed encrypted"
(is (not= "true" (t2/select-one-fn :value :setting :key "enable-query-caching"))))

View File

@@ -1507,10 +1507,10 @@
(impl/test-migrations ["v50.2024-06-12T12:33:07"] [migrate!]
;; this peculiar setup is to reproduce #44012, `enable-query-caching` should be unencrypted for the condition
;; to hit it
(t2/insert! :setting [{:key "enable-query-caching", :value (encryption/maybe-encrypt "true")}])
(t2/insert! :setting [{:key "enable-query-caching", :value (encryption/encrypt "true")}])
(encryption-test/with-secret-key "whateverwhatever"
(t2/insert! :setting [{:key "query-caching-ttl-ratio", :value (encryption/maybe-encrypt "100.4")}
{:key "query-caching-min-ttl", :value (encryption/maybe-encrypt "123.4")}]))
(t2/insert! :setting [{:key "query-caching-ttl-ratio", :value (encryption/encrypt "100.4")}
{:key "query-caching-min-ttl", :value (encryption/encrypt "123.4")}]))
(let [user (create-raw-user! (mt/random-email))
db (t2/insert-returning-pk! :metabase_database (-> (mt/with-temp-defaults Database)
(update :details json/generate-string)
@@ -1559,9 +1559,9 @@
(deftest cache-config-migration-test-2
(testing "And not copied if caching is disabled"
(impl/test-migrations ["v50.2024-04-12T12:33:07"] [migrate!]
(t2/insert! :setting [{:key "enable-query-caching", :value (encryption/maybe-encrypt "false")}
{:key "query-caching-ttl-ratio", :value (encryption/maybe-encrypt "100")}
{:key "query-caching-min-ttl", :value (encryption/maybe-encrypt "123")}])
(t2/insert! :setting [{:key "enable-query-caching", :value (encryption/encrypt "false")}
{:key "query-caching-ttl-ratio", :value (encryption/encrypt "100")}
{:key "query-caching-min-ttl", :value (encryption/encrypt "123")}])
;; this one to have custom configuration to check they are not copied over
(t2/insert-returning-pk! :metabase_database (-> (mt/with-temp-defaults Database)
(update :details json/generate-string)
@@ -1577,9 +1577,9 @@
(testing "Root cache config for mysql is updated with correct values"
(encryption-test/with-secret-key "whateverwhatever"
(impl/test-migrations ["v50.2024-06-12T12:33:07"] [migrate!]
(t2/insert! :setting [{:key "enable-query-caching", :value (encryption/maybe-encrypt "true")}
{:key "query-caching-ttl-ratio", :value (encryption/maybe-encrypt "100.4")}
{:key "query-caching-min-ttl", :value (encryption/maybe-encrypt "123.4")}])
(t2/insert! :setting [{:key "enable-query-caching", :value (encryption/encrypt "true")}
{:key "query-caching-ttl-ratio", :value (encryption/encrypt "100.4")}
{:key "query-caching-min-ttl", :value (encryption/encrypt "123.4")}])
;; the idea here is that `v50.2024-04-12T12:33:09` during execution with partially encrypted data (see
;; `cache-config-migration-test`) instead of throwing an error just silently put zeros in config. If config

View File

@@ -70,19 +70,19 @@
#"Message seems corrupt or manipulated"
(encryption/decrypt secret-2 (encryption/encrypt secret "WOW"))))))
(deftest ^:parallel maybe-decrypt-not-encrypted-test
(testing "trying to `maybe-decrypt` something that's not encrypted should return it as-is"
(deftest ^:parallel decrypt-not-encrypted-test
(testing "trying to `decrypt` something that's not encrypted should return it as-is"
(is (= "{\"a\":100}"
(encryption/maybe-decrypt secret "{\"a\":100}")))
(encryption/decrypt secret "{\"a\":100}")))
(is (= "abc"
(encryption/maybe-decrypt secret "abc")))))
(encryption/decrypt secret "abc")))))
(deftest ^:parallel maybe-decrypt-with-wrong-key-test
(testing (str "trying to decrypt something that is encrypted with the wrong key with `maybe-decrypt` should return "
(deftest ^:parallel decrypt-with-wrong-key-test
(testing (str "trying to decrypt something that is encrypted with the wrong key with `decrypt` should return "
"the ciphertext...")
(let [original-ciphertext (encryption/encrypt secret "WOW")]
(is (= original-ciphertext
(encryption/maybe-decrypt secret-2 original-ciphertext))))))
(encryption/decrypt secret-2 original-ciphertext))))))
(defn- includes-encryption-warning? [log-messages]
(some (fn [[level _ message]]
@@ -94,7 +94,7 @@
(deftest no-errors-for-unencrypted-test
(testing "Something obviously not encrypted should avoiding trying to decrypt it (and thus not log an error)"
(is (empty? (mt/with-log-messages-for-level :warn
(encryption/maybe-decrypt secret "abc"))))))
(encryption/decrypt secret "abc"))))))
(def ^:private fake-ciphertext
"AES+CBC's block size is 16 bytes and the tag length is 32 bytes. This is a string of characters that is the same
@@ -107,12 +107,12 @@
"decrypted. If unable to decrypt it, log a warning.")
(is (includes-encryption-warning?
(mt/with-log-messages-for-level :warn
(encryption/maybe-decrypt secret fake-ciphertext))))
(encryption/decrypt secret fake-ciphertext))))
(is (includes-encryption-warning?
(mt/with-log-messages-for-level :warn
(encryption/maybe-decrypt secret-2 (encryption/encrypt secret "WOW")))))))
(encryption/decrypt secret-2 (encryption/encrypt secret "WOW")))))))
(deftest ^:parallel possibly-encrypted-test
(testing "Something that is not encrypted, but might be should return the original text"
(is (= fake-ciphertext
(encryption/maybe-decrypt secret fake-ciphertext)))))
(encryption/decrypt secret fake-ciphertext)))))

View File

@@ -134,7 +134,7 @@
(is (= "en"
(i18n.impl/site-locale-from-setting)))
;; rotate the encryption key, which will trigger an error being logged
;; in [[metabase.util.encryption/maybe-decrypt]]... this will cause a Stack Overflow if `log/error` tries to
;; in [[metabase.util.encryption/decrypt]]... this will cause a Stack Overflow if `log/error` tries to
;; access `:site-locale` recursively to log the message.
(encryption-test/with-secret-key "secret_key__2"
(testing "low-level functions should return the encrypted String since we can't successfully decrypt it"