Secrets 🔐 PR 7 - Enable Postgres driver to use secrets for configuring SSL parameters (#18968)

* Enable Postgres driver to use secrets for configuring SSL parameters

Adding secret related properties for SSL options to Postgres driver

Move `conn-props->secret-props-by-name` to secret.clj since it's needed directly there, too

Add `us-east-2-bundle.pem` to `test-resources/certificates` for testing Postgres with SSL connectivity to our RDS instance (and a README.md explaining how it differs from the existing `ssl` directory)

Updating `value->file!` to have better logic for building the error message when the existing file is not found (for better UX from the Database admin page)

Updating frontend to support the `visible-if` value being an array, in which case any value may match the current details value, in order for that field to be visible

Adding secret related properties for SSL options to Postgres driver

Updating CircleCI config.yml to refer to the RDS instance when running Postgres SSL test

Implement server side expansion of transitive visible-if dependency chain

Update shouldShowEngineProvidedField on client to consider multiple key/value pairs in the visible-if map, and only show the field if ALL are true

Adding new test to confirm transitive visible-if resolution, and cycle detection

Add Cypress test for SSL field visibility
This commit is contained in:
Jeff Evans
2021-12-03 16:38:55 -06:00
committed by GitHub
parent 5460e4b983
commit f8d712d6e1
16 changed files with 539 additions and 154 deletions

View File

@@ -1223,6 +1223,10 @@ workflows:
- be-tests-ee
e: postgres-latest
driver: postgres
extra-env: >-
MB_POSTGRES_SSL_TEST_SSL=true
MB_POSTGRES_SSL_TEST_SSL_MODE=verify-full
MB_POSTGRES_SSL_TEST_SSL_ROOT_CERT_PATH=/home/circleci/metabase/metabase/test-resources/certificates/us-east-2-bundle.pem
- test-driver:
name: be-tests-presto-ee

View File

@@ -213,12 +213,22 @@ function shouldShowEngineProvidedField(field, details) {
const detailAndValueRequiredToShowField = field["visible-if"];
if (detailAndValueRequiredToShowField) {
const [detail, expectedDetailValue] = Object.entries(
detailAndValueRequiredToShowField,
)[0];
const pred = currentValue => {
const [detail, expectedDetailValue] = currentValue;
if (Array.isArray(expectedDetailValue)) {
// if the expectedDetailValue is itself an array, then consider the condition satisfied if any of those values
// match the current detail value
return expectedDetailValue.includes(details[detail]);
} else {
return details[detail] === expectedDetailValue;
}
};
// check all entries in the visible-if map, and only show this field if all key/values are satisfied
// (i.e. boolean AND)
return Object.entries(detailAndValueRequiredToShowField).every(pred);
}
return true;
}

View File

@@ -19,6 +19,16 @@ function toggleFieldWithDisplayName(displayName) {
.click();
}
function selectFieldOption(fieldName, option) {
cy.contains(fieldName)
.parents(".Form-field")
.find(".AdminSelect")
.click();
popover()
.contains(option)
.click({ force: true });
}
describe("scenarios > admin > databases > add", () => {
beforeEach(() => {
restore();
@@ -364,16 +374,53 @@ describe("scenarios > admin > databases > add", () => {
});
});
});
it("should show the various Postgres SSL options correctly", () => {
const confirmSSLFields = (visible, hidden) => {
visible.forEach(field => cy.findByText(field));
hidden.forEach(field => cy.findByText(field).should("not.exist"));
};
const ssl = "Use a secure connection (SSL)?",
sslMode = "SSL Mode",
useClientCert = "Authenticate client certificate?",
clientPemCert = "SSL Client Certificate (PEM)",
clientPkcsCert = "SSL Client Key (PKCS-8/DER or PKCS-12)",
sslRootCert = "SSL Root Certificate (PEM)";
cy.visit("/admin/databases/create");
chooseDatabase("PostgreSQL");
// initially, all SSL sub-properties should be hidden
confirmSSLFields(
[ssl],
[sslMode, useClientCert, clientPemCert, clientPkcsCert, sslRootCert],
);
toggleFieldWithDisplayName(ssl);
// when ssl is enabled, the mode and "enable client cert" options should be shown
confirmSSLFields(
[ssl, sslMode, useClientCert],
[clientPemCert, clientPkcsCert, sslRootCert],
);
toggleFieldWithDisplayName(useClientCert);
// when the "enable client cert" option is enabled, its sub-properties should be shown
confirmSSLFields(
[ssl, sslMode, useClientCert, clientPemCert, clientPkcsCert],
[sslRootCert],
);
selectFieldOption(sslMode, "verify-ca");
// when the ssl mode is set to "verify-ca", then the root cert option should be shown
confirmSSLFields(
[ssl, sslMode, useClientCert, clientPemCert, clientPkcsCert, sslRootCert],
[],
);
});
});
function chooseDatabase(database) {
cy.contains("Database type")
.parents(".Form-field")
.find(".AdminSelect")
.click();
popover()
.contains(database)
.click({ force: true });
selectFieldOption("Database type", database);
}
function isSyncOptionSelected(option) {

View File

@@ -97,7 +97,7 @@
(-> ; from outer cond->
(assoc :javax.net.ssl.keyStoreType "JKS"
:javax.net.ssl.keyStore (-> (secret/db-details-prop->secret-map details "ssl-keystore")
secret/value->file!)
(secret/value->file! :oracle))
:javax.net.ssl.keyStorePassword (-> (secret/db-details-prop->secret-map details "ssl-keystore-password")
secret/value->string))
(dissoc :ssl-use-keystore :ssl-keystore-value :ssl-keystore-path :ssl-keystore-password-value))
@@ -106,7 +106,7 @@
(-> ; from outer cond->
(assoc :javax.net.ssl.trustStoreType "JKS"
:javax.net.ssl.trustStore (-> (secret/db-details-prop->secret-map details "ssl-truststore")
secret/value->file!)
(secret/value->file! :oracle))
:javax.net.ssl.trustStorePassword (-> (secret/db-details-prop->secret-map details
"ssl-truststore-password")
secret/value->string))

View File

@@ -135,8 +135,8 @@
{:name "tunnel-pass"}
{:name "tunnel-private-key"}
{:name "tunnel-private-key-passphrase"}]
actual (-> (driver/connection-properties :oracle)
driver.u/connection-props-server->client)]
actual (->> (driver/connection-properties :oracle)
(driver.u/connection-props-server->client :oracle))]
(is (= expected (mt/select-keys-sequentially expected actual))))))
(deftest test-ssh-connection

View File

@@ -18,9 +18,11 @@
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.util.unprepare :as unprepare]
[metabase.models :refer [Field]]
[metabase.models.secret :as secret]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.honeysql-extensions :as hx]
[metabase.util.i18n :refer [trs]]
[metabase.util.ssh :as ssh]
[pretty.core :refer [PrettyPrintable]])
(:import [java.sql ResultSet ResultSetMetaData Time Types]
@@ -87,6 +89,48 @@
driver.common/default-password-details
driver.common/cloud-ip-address-info
driver.common/default-ssl-details
{:name "ssl-mode"
:display-name (trs "SSL Mode")
:type :select
:options [{:name "allow"
:value "allow"}
{:name "prefer"
:value "prefer"}
{:name "require"
:value "require"}
{:name "verify-ca"
:value "verify-ca"}
{:name "verify-full"
:value "verify-full"}]
:default "require"
:visible-if {"ssl" true}}
{:name "ssl-root-cert"
:display-name (trs "SSL Root Certificate (PEM)")
:type :secret
:secret-kind :pem-cert
;; only need to specify the root CA if we are doing one of the verify modes
:visible-if {"ssl-mode" ["verify-ca" "verify-full"]}}
{:name "ssl-use-client-auth"
:display-name (trs "Authenticate client certificate?")
:type :boolean
;; TODO: does this somehow depend on any of the ssl-mode vals? it seems not (and is in fact orthogonal)
:visible-if {"ssl" true}}
{:name "ssl-client-cert"
:display-name (trs "SSL Client Certificate (PEM)")
:type :secret
:secret-kind :pem-cert
:visible-if {"ssl-use-client-auth" true}}
{:name "ssl-key"
:display-name (trs "SSL Client Key (PKCS-8/DER or PKCS-12)")
:type :secret
;; since this can be either PKCS-8 or PKCS-12, we can't model it as a :keystore
:secret-kind :binary-blob
:visible-if {"ssl-use-client-auth" true}}
{:name "ssl-key-password"
:display-name (trs "SSL Client Key Password")
:type :secret
:secret-kind :password
:visible-if {"ssl-use-client-auth" true}}
(assoc driver.common/default-additional-options-details
:placeholder "prepareThreshold=0")]))
@@ -298,10 +342,40 @@
"inet" :type/IPAddress
nil))
(def ^:private ssl-params
"Params to include in the JDBC connection spec for an SSL connection."
{:ssl true
:sslmode "require"})
(defn- ssl-params
"Builds the params to include in the JDBC connection spec for an SSL connection."
[db-details]
(let [ssl-root-cert (when (contains? #{"verify-ca" "verify-full"} (:ssl-mode db-details))
(secret/db-details-prop->secret-map db-details "ssl-root-cert"))
ssl-client-key (when (:ssl-use-client-auth db-details)
(secret/db-details-prop->secret-map db-details "ssl-client-key"))
ssl-client-cert (when (:ssl-use-client-auth db-details)
(secret/db-details-prop->secret-map db-details "ssl-client-cert"))
ssl-key-pw (when (:ssl-use-client-auth db-details)
(secret/db-details-prop->secret-map db-details "ssl-key-password"))
all-subprops (apply concat (map :subprops [ssl-root-cert ssl-client-key ssl-client-cert ssl-key-pw]))]
(cond-> (set/rename-keys db-details {:ssl-mode :sslmode})
;; if somehow there was no ssl-mode set, just make it required (preserves existing behavior)
(nil? (:ssl-mode db-details))
(assoc :sslmode "require")
ssl-root-cert
(assoc :sslrootcert (secret/value->file! ssl-root-cert :postgres))
ssl-client-key
(assoc :sslkey (secret/value->file! ssl-client-key :postgres))
ssl-client-cert
(assoc :sslcert (secret/value->file! ssl-client-cert :postgres))
ssl-key-pw
(assoc :sslpassword (secret/value->string ssl-key-pw))
true
(as-> params ;; from outer cond->
(dissoc params :ssl-root-cert :ssl-root-cert-options :ssl-client-key :ssl-client-cert :ssl-key-password
:ssl-use-client-auth)
(apply dissoc params all-subprops)))))
(def ^:private disable-ssl-params
"Params to include in the JDBC connection spec to disable SSL."
@@ -316,12 +390,15 @@
port)))
;; remove :ssl in case it's false; DB will still try (& fail) to connect if the key is there
(dissoc :ssl))
;; this happens via ->> so that the users props will override the ssl-params stuff.
;; if the user has specified a sslmode, it must always take precedence over our default.
props (->> props
(merge (if ssl?
ssl-params
disable-ssl-params)))
props (if ssl?
(let [ssl-prms (ssl-params details-map)]
;; if the user happened to specify any of the SSL options directly, allow those to take
;; precedence, but only if they match a key from our own
;; our `ssl-params` function is also removing various internal properties, ex: for secret resolution,
;; so we can't just merge the entire `props` map back in here because it will bring all those
;; internal property values back; only merge in the ones the driver might recognize
(merge ssl-prms (select-keys props (keys ssl-prms))))
(merge disable-ssl-params props))
props (-> props
(set/rename-keys {:dbname :db})
db.spec/postgres

View File

@@ -1,11 +1,13 @@
(ns metabase.driver.util
"Utility functions for common operations on drivers."
(:require [clojure.core.memoize :as memoize]
[clojure.set :as set]
[clojure.tools.logging :as log]
[metabase.config :as config]
[metabase.driver :as driver]
[metabase.models.setting :refer [defsetting]]
[metabase.public-settings.premium-features :as premium-features]
[metabase.query-processor.error-type :as error-type]
[metabase.util :as u]
[metabase.util.i18n :refer [trs]]
[toucan.db :as db])
@@ -99,19 +101,15 @@
:when (driver/available? driver)]
driver)))
(defn- expand-secret-conn-prop [{prop-name :name, visible-if :visible-if, :as conn-prop}]
(case (:secret-kind conn-prop)
"password" [(-> conn-prop
(assoc :type "password")
(assoc :name (str prop-name "-value"))
(dissoc :secret-kind))]
"keystore" (if (premium-features/is-hosted?)
(defn- file-upload-props [{prop-name :name, visible-if :visible-if, disp-nm :display-name, :as conn-prop}]
(if (premium-features/is-hosted?)
[(-> (assoc conn-prop
:name (str prop-name "-value")
:type "textFile"
:treat-before-posting "base64")
(dissoc :secret-kind))]
[(cond-> {:name (str prop-name "-options")
:display-name disp-nm
:type "select"
:options [{:name "Local file path"
:value "local"}
@@ -119,16 +117,36 @@
:value "uploaded"}]
:default "local"}
visible-if (assoc :visible-if visible-if))
(-> (assoc conn-prop
:name (str prop-name "-value")
(-> {:name (str prop-name "-value")
:type "textFile"
:treat-before-posting "base64"
:visible-if {(keyword (str prop-name "-options")) "uploaded"})
:visible-if {(keyword (str prop-name "-options")) "uploaded"}}
(dissoc :secret-kind))
{:name (str prop-name "-path")
:type "string"
:display-name "File path"
:placeholder (:placeholder conn-prop)
:visible-if {(keyword (str prop-name "-options")) "local"}}])
:visible-if {(keyword (str prop-name "-options")) "local"}}]))
(defn- ->str
"Turns `x` into a String. If `x` a keyword, then `name` is used. Otherwise, `str` is called on it."
[k]
(if (keyword? k)
(name k)
(str k)))
(defn- expand-secret-conn-prop [{prop-name :name, visible-if :visible-if, :as conn-prop}]
(case (->str (:secret-kind conn-prop))
"password" [(-> conn-prop
(assoc :type "password")
(assoc :name (str prop-name "-value"))
(dissoc :secret-kind))]
"keystore" (file-upload-props conn-prop)
;; this may not necessarily be a keystore (could be a standalone PKCS-8 or PKCS-12 file)
"binary-blob" (file-upload-props conn-prop)
;; PEM is a plaintext format
;; TODO: do we need to also allow a textarea type paste for this? would require another special case
"pem-cert" (file-upload-props conn-prop)
[conn-prop]))
(defn- resolve-info-conn-prop
@@ -147,7 +165,7 @@
(dissoc :getter)))))
(defn connection-props-server->client
"Transforms connection-properties from their server side definition into a client side definition.
"Transforms `conn-props` for the given `driver` from their server side definition into a client side definition.
This transforms :type :secret properties from the server side definition into other types for client
display/editing. For example, a :secret-kind :keystore turns into a bunch of different properties, to encapsulate
@@ -155,20 +173,51 @@
This also resolves the :getter function on :type :info properties, if one was provided."
{:added "0.42.0"}
[conn-props]
(reduce (fn [acc conn-prop]
(case (keyword (:type conn-prop))
[driver conn-props]
(let [res (reduce (fn [acc conn-prop]
;; TODO: change this to expanded- and use that as the basis for all calcs below (not conn-prop)
(let [expanded-props (case (keyword (:type conn-prop))
:secret
(concat acc (expand-secret-conn-prop conn-prop))
(expand-secret-conn-prop conn-prop)
:info
(if-let [conn-prop' (resolve-info-conn-prop conn-prop)]
(concat acc [conn-prop'])
acc)
[conn-prop']
[])
(concat acc [conn-prop])))
[]
conn-props))
[conn-prop])]
(-> (update acc ::final-props concat expanded-props)
(update ::props-by-name merge (into {} (map (fn [p]
[(:name p) p])) expanded-props)))))
{::final-props [] ::props-by-name {}}
conn-props)
{:keys [::final-props ::props-by-name]} res]
;; now, traverse the visible-if-edges and update all visible-if entries with their full set of "transitive"
;; dependencies (if property x depends on y having a value, but y itself depends on z having a value, then x
;; should be hidden if y is)
(mapv (fn [prop]
(let [v-ifs* (loop [props* [prop]
acc {}]
(if (seq props*)
(let [all-visible-ifs (apply merge (map :visible-if props*))
transitive-props (map (comp (partial get props-by-name) ->str)
(keys all-visible-ifs))
next-acc (merge all-visible-ifs acc)
cyclic-props (set/intersection (into #{} (keys all-visible-ifs))
(into #{} (keys acc)))]
(if (empty? cyclic-props)
(recur transitive-props next-acc)
(-> "Cycle detected resolving dependent visible-if properties for driver {0}: {1}"
(trs driver cyclic-props)
(ex-info {:type error-type/driver
:driver driver
:cyclic-visible-ifs cyclic-props})
throw)))
acc))]
(cond-> prop
(seq v-ifs*)
(assoc :visible-if v-ifs*))))
final-props)))
(defn db-details-client->server
"Currently, this transforms client side values for the various back into :type :secret for storage on the server.
@@ -209,8 +258,8 @@
[]
(into {} (for [driver (available-drivers)
:let [props (try
(-> (driver/connection-properties driver)
connection-props-server->client)
(->> (driver/connection-properties driver)
(connection-props-server->client driver))
(catch Throwable e
(log/error e (trs "Unable to determine connection properties for driver {0}" driver))))]
:when props]
@@ -272,6 +321,6 @@
[driver]
(if-some [conn-prop-fn (get-method driver/connection-properties driver)]
(let [all-fields (conn-prop-fn driver)
password-fields (filter #(= (get % :type) :password) all-fields)]
password-fields (filter #(contains? #{:password :secret} (get % :type)) all-fields)]
(into default-sensitive-fields (map (comp keyword :name) password-fields)))
default-sensitive-fields))

View File

@@ -60,13 +60,6 @@
(driver/normalize-db-details driver db*)
db*))))
(defn- conn-props->secret-props-by-name
"For the given `conn-props` (output of `driver/connection-properties`), return a map of all secret type properties,
keyed by property name."
[conn-props]
(->> (filter #(= "secret" (:type %)) conn-props)
(reduce (fn [acc prop] (assoc acc (:name prop) prop)) {})))
(defn- delete-orphaned-secrets!
"Delete Secret instances from the app DB, that will become orphaned when `database` is deleted. For now, this will
simply delete any Secret whose ID appears in the details blob, since every Secret instance that is currently created
@@ -77,7 +70,7 @@
[{:keys [id details] :as database}]
(when-let [conn-props-fn (get-method driver/connection-properties (driver.u/database->driver database))]
(let [conn-props (conn-props-fn (driver.u/database->driver database))
possible-secret-prop-names (keys (conn-props->secret-props-by-name conn-props))]
possible-secret-prop-names (keys (secret/conn-props->secret-props-by-name conn-props))]
(doseq [secret-id (reduce (fn [acc prop-name]
(if-let [secret-id (get details (keyword (str prop-name "-id")))]
(conj acc secret-id)
@@ -141,7 +134,7 @@
details ; we have details populated in this Database instance, so handle them
(let [conn-props (conn-props-fn (driver.u/database->driver database))
conn-secrets-by-name (conn-props->secret-props-by-name conn-props)
conn-secrets-by-name (secret/conn-props->secret-props-by-name conn-props)
updated-details (reduce-kv (partial handle-db-details-secret-prop! database)
details
conn-secrets-by-name)]

View File

@@ -3,6 +3,7 @@
[clojure.java.io :as io]
[clojure.tools.logging :as log]
[metabase.api.common :as api]
[metabase.driver :as driver]
[metabase.models.interface :as i]
[metabase.public-settings.premium-features :as premium-features]
[metabase.util :as u]
@@ -42,23 +43,44 @@
(bytes? value)
(String. ^bytes value StandardCharsets/UTF_8)))
(defn value->file!
"Returns the value of the given `secret` in the form of a file. `secret` can be a Secret model object, or a
secret-map (i.e. return value from `db-details-prop->secret-map`).
In any case, if the `:source` is `:file-path`, then the value will be returned unchanged (since it is ostensibly
already a file). Otherwise, a temp file will be created and the secret value will be written to it, and the path to
that temp file will be returned."
(defn conn-props->secret-props-by-name
"For the given `conn-props` (output of `driver/connection-properties`), return a map of all `:type` `:secret`
properties, keyed by property name."
{:added "0.42.0"}
[{:keys [id ^bytes value] :as secret}]
[conn-props]
(->> (filter #(= :secret (keyword (:type %))) conn-props)
(reduce (fn [acc prop] (assoc acc (:name prop) prop)) {})))
(defn value->file!
"Returns the value of the given `secret` instance in the form of a file. If the given instance has a `:file-path` as
its source, a `File` referring to that is returned. Otherwise, the `:value` is written to a temporary file, which is
then returned.
`driver?` is an optional argument that is only used if an ostensibly existing file value (i.e. `:file-path`) can't be
resolved, in order to render a more user-friendly error message (by looking up the display names of the connection
properties involved)."
{:added "0.42.0"}
[{:keys [connection-property-name id value] :as secret} driver?]
(if (= :file-path (:source secret))
(let [secret-val (value->string secret)
^File existing-file (File. secret-val)]
(if (.exists existing-file)
existing-file
(throw (ex-info (tru "Secret {0} points to non-existent file: {1}" (or id "") secret-val)
{:secret-id id
:file-path secret-val}))))
(let [error-source (cond
id
(tru "Secret ID {0}" id)
(and connection-property-name driver?)
(let [secret-props (-> (driver/connection-properties driver?)
conn-props->secret-props-by-name)]
(tru "File path for {0}" (-> (get secret-props connection-property-name)
:display-name)))
true
(tru "Path"))]
(throw (ex-info (tru "{0} points to non-existent file: {1}" error-source secret-val)
{:file-path secret-val
:secret secret})))))
(let [^File tmp-file (doto (File/createTempFile "metabase-secret_" nil)
;; make the file only readable by owner
(.setReadable false false)
@@ -66,9 +88,59 @@
(.deleteOnExit))]
(log/tracef "Creating temp file for secret %s value at %s" (or id "") (.getAbsolutePath tmp-file))
(with-open [out (io/output-stream tmp-file)]
(.write out value))
(let [^bytes v (cond
(string? value)
(.getBytes ^String value "UTF-8")
(bytes? value)
^bytes value)]
(.write out v)))
tmp-file)))
(defn db-details-prop->secret-map
"Returns a map containing `:value` and `:source` for the given `conn-prop-nm`. `conn-prop-nm` is expected to be the
name of a connection property having `:type` `:secret`, and the relevant sub-properties (ex: -value, -path, etc.) will
be resolved in order to calculate the returned map.
This returned map represents a partial Secret model instance (having some of the required properties set), but also
represents a discrete property that can be used in connection testing (even without the Secret needing to be
persisted). In addition to possibly having `:value` and `:source` populated (if the secret value can be resolved), its
keys will always include:
`:connection-property-name` - the `conn-prop-nm` that was initially passed in, for use later in error handling.
`:subprops` - a sequence of subproperties (keywords) that represent all secret related subproperties that might
exist and be manipulated by the secret handling code (which are used to ensure all these internal and
intermediate subproperties are removed from the connection-properties before building the JDBC spec)."
{:added "0.42.0"}
[details conn-prop-nm]
(let [sub-prop (fn [suffix]
(keyword (str conn-prop-nm suffix)))
path-kw (sub-prop "-path")
value-kw (sub-prop "-value")
options-kw (sub-prop "-options")
id-kw (sub-prop "-id")
value (if-let [v (value-kw details)] ; the -value suffix was specified; use that
v
(if-let [path (path-kw details)] ; the -path suffix was specified; this is actually a :file-path
(do
(when (premium-features/is-hosted?)
(throw (ex-info
(tru "{0} (a local file path) cannot be used in Metabase hosted environment" path-kw)
{:invalid-db-details-entry (select-keys details [path-kw])})))
path)
(when-let [id (id-kw details)]
(:value (Secret id)))))
source (cond
;; set the :source due to the -path suffix (see above))
(and (not= "uploaded" (options-kw details)) (path-kw details))
:file-path
(id-kw details)
(:source (Secret (id-kw details))))]
(cond-> {:connection-property-name conn-prop-nm, :subprops [path-kw value-kw id-kw]}
value
(assoc :value value, :source source))))
(def
^{:doc "The attributes of a secret which, if changed, will result in a version bump" :private true}
bump-version-keys
@@ -109,40 +181,6 @@
(insert-new (u/the-id latest-version) (inc (:version latest-version))))
(insert-new nil 1))))
(defn db-details-prop->secret-map
"Returns a map containing `:value` and `:source` for the given `conn-prop-nm`. `conn-prop-nm` is expected to be the
name of a connection property having `:type` `:secret`, and the relevant sub-properties (ex: -value, -path, etc.) will
be resolved in order to calculate the returned map.
This returned map represents a partial Secret model instance
(having some of the required properties set), but also represents a discrete property that can be used in connection
testing (even without the Secret needing to be persisted)."
{:added "0.42.0"}
[details conn-prop-nm]
(let [sub-prop (fn [suffix]
(keyword (str conn-prop-nm suffix)))
path-kw (sub-prop "-path")
value-kw (sub-prop "-value")
id-kw (sub-prop "-id")
value (if-let [v (value-kw details)] ; the -value suffix was specified; use that
v
(if-let [path (path-kw details)] ; the -path suffix was specified; this is actually a :file-path
(do
(when (premium-features/is-hosted?)
(throw (ex-info
(tru "{0} (a local file path) cannot be used in Metabase hosted environment" path-kw)
{:invalid-db-details-entry (select-keys details [path-kw])})))
path)
(when-let [id (id-kw details)]
(:value (Secret id)))))
source (cond
(path-kw details) ; set the :source due to the -path suffix (see above))
:file-path
(id-kw details)
(:source (Secret (id-kw details))))]
{:value value :source source}))
;;; -------------------------------------------------- JSON Encoder --------------------------------------------------
(add-encoder SecretInstance (fn [secret json-generator]

View File

@@ -662,3 +662,14 @@
"GROUP BY attempts.date "
"ORDER BY attempts.date ASC")
(some-> (qp/query->native query) :query pretty-sql))))))))
(deftest postgres-ssl-connectivity-test
(mt/test-driver :postgres
(if (System/getenv "MB_POSTGRES_SSL_TEST_SSL")
(testing "We should be able to connect to a Postgres instance, providing our own root CA via a secret property"
(mt/with-env-keys-renamed-by #(str/replace-first % "mb-postgres-ssl-test" "mb-postgres-test")
(id-field-parameter-test)))
(println (u/format-color 'yellow
"Skipping %s because %s env var is not set"
"postgres-ssl-connectivity-test"
"MB_POSTGRES_SSL_TEST_SSL")))))

View File

@@ -51,49 +51,55 @@
:type "password"
:placeholder "foo"
:required false}
{:name "use-keystore"}
{:name "ssl"}
{:name "use-keystore"
:visible-if {:ssl true}}
{:name "keystore-password-value"
:display-name "Keystore Password",
:type "password",
:required false,
:visible-if {:use-keystore true}}
:visible-if {:use-keystore true
;; this should have been filled in as a transitive dependency
:ssl true}}
{:name "keystore-options"
:display-name "Keystore"
:options [{:name "Local file path"
:value "local"}
{:name "Uploaded file path"
:value "uploaded"}]
:type "select"
:default "local"
:visible-if {:use-keystore true}}
:visible-if {:use-keystore true
:ssl true}}
{:name "keystore-value"
:display-name "Keystore"
:type "textFile"
:required false
:treat-before-posting "base64"
:visible-if {:keystore-options "uploaded"}}
{:name "keystore-path"
:type "string"
:visible-if {:keystore-options "local"}}]
:visible-if {:keystore-options "local"
:use-keystore true
:ssl true}}]
false]
[[{:name "host"}
{:name "password-value"
:type "password"
:placeholder "foo"
:required false}
{:name "use-keystore"}
{:name "ssl"}
{:name "use-keystore"
:visible-if {:ssl true}}
{:name "keystore-password-value"
:display-name "Keystore Password"
:type "password"
:required false
:visible-if {:use-keystore true}}
{:name "keystore-value"
:display-name "Keystore"
:type "textFile"
:required false
:treat-before-posting "base64"
:visible-if {:use-keystore true}}]
true]]]
(testing (str "with is-hosted? " is-hosted?)
(testing (str " with is-hosted? " is-hosted?)
;; TODO: create capability to temporarily override token-features for testing
(with-redefs [premium-features/is-hosted? (constantly is-hosted?)]
(let [client-conn-props (-> (driver.u/available-drivers-info) ; this calls connection-props-server->client
@@ -105,26 +111,40 @@
(testing "info fields with placeholder defined are unmodified"
(is (= [{:name "test", :type :info, :placeholder "placeholder"}]
(driver.u/connection-props-server->client
nil
[{:name "test", :type :info, :placeholder "placeholder"}]))))
(testing "info fields with getter defined invoke the getter to generate the placeholder"
(is (= [{:name "test", :type :info, :placeholder "placeholder"}]
(driver.u/connection-props-server->client
nil
[{:name "test", :type :info, :getter (constantly "placeholder")}]))))
(testing "info fields are omitted if getter returns nil, a non-string value, or throws an exception"
(is (= []
(driver.u/connection-props-server->client
nil
[{:name "test", :type :info, :getter (constantly nil)}])))
(is (= []
(driver.u/connection-props-server->client
nil
[{:name "test", :type :info, :getter (constantly 0)}])))
(is (= []
(driver.u/connection-props-server->client
nil
[{:name "test", :type :info, :getter #(throw (Exception. "test error"))}])))
(is (= []
(driver.u/connection-props-server->client
[{:name "test", :type :info}])))))))
nil
[{:name "test", :type :info}]))))))
(testing "connection-props-server->client detects cycles in visible-if dependencies"
(let [fake-props [{:name "prop-a", :visible-if {:prop-c "something"}}
{:name "prop-b", :visible-if {:prop-a "something else"}}
{:name "prop-c", :visible-if {:prop-b "something else entirely"}}]]
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Cycle detected"
(driver.u/connection-props-server->client :fake-cyclic-driver fake-props))))))
(deftest connection-details-client->server-test
(testing "db-details-client->server works as expected"

View File

@@ -67,7 +67,7 @@
:value (.getAbsolutePath tmp-file)}]]]
(testing (format " with a %s value" value-kind)
(mt/with-temp Secret [{:keys [id value] :as secret} (assoc secret-map :creator_id (mt/user->id :crowberto))]
(let [val-file (secret/value->file! secret)]
(let [val-file (secret/value->file! secret nil)]
(is (value-matches? (or exp-val value)
(let [result (byte-array (.length val-file))]
(with-open [in (DataInputStream. (io/input-stream val-file))]

View File

@@ -4,7 +4,7 @@
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]))
(use-fixtures :once (fixtures/initialize :plugins :test-drivers))
(use-fixtures :once (fixtures/initialize :db :plugins :test-drivers))
(deftest driver-deprecation-test
(mt/with-driver :driver-deprecation-test-legacy

View File

@@ -17,9 +17,14 @@ driver:
secret-kind: password
placeholder: foo
required: false
- name: ssl
display-name: SSL enabled?
type: boolean
- name: use-keystore
type: boolean
display-name: Use Keystore?
visible-if:
ssl: true
- name: keystore-password
display-name: Keystore Password
type: secret

View File

@@ -0,0 +1,7 @@
# Driver Testing Certificates
These are certificate files used for driver testing. They are kept separate from the `ssl` directory, which contains
various certificates related to hosting the Metabase app itself using self-signed SSL certs.
`us-east-2-bundle.pem` is the PEM certificate bundle for AWS `us-east-2`, fetched from
[here](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html).

View File

@@ -0,0 +1,124 @@
-----BEGIN CERTIFICATE-----
MIIEBjCCAu6gAwIBAgIJAMc0ZzaSUK51MA0GCSqGSIb3DQEBCwUAMIGPMQswCQYD
VQQGEwJVUzEQMA4GA1UEBwwHU2VhdHRsZTETMBEGA1UECAwKV2FzaGluZ3RvbjEi
MCAGA1UECgwZQW1hem9uIFdlYiBTZXJ2aWNlcywgSW5jLjETMBEGA1UECwwKQW1h
em9uIFJEUzEgMB4GA1UEAwwXQW1hem9uIFJEUyBSb290IDIwMTkgQ0EwHhcNMTkw
ODIyMTcwODUwWhcNMjQwODIyMTcwODUwWjCBjzELMAkGA1UEBhMCVVMxEDAOBgNV
BAcMB1NlYXR0bGUxEzARBgNVBAgMCldhc2hpbmd0b24xIjAgBgNVBAoMGUFtYXpv
biBXZWIgU2VydmljZXMsIEluYy4xEzARBgNVBAsMCkFtYXpvbiBSRFMxIDAeBgNV
BAMMF0FtYXpvbiBSRFMgUm9vdCAyMDE5IENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
AQ8AMIIBCgKCAQEArXnF/E6/Qh+ku3hQTSKPMhQQlCpoWvnIthzX6MK3p5a0eXKZ
oWIjYcNNG6UwJjp4fUXl6glp53Jobn+tWNX88dNH2n8DVbppSwScVE2LpuL+94vY
0EYE/XxN7svKea8YvlrqkUBKyxLxTjh+U/KrGOaHxz9v0l6ZNlDbuaZw3qIWdD/I
6aNbGeRUVtpM6P+bWIoxVl/caQylQS6CEYUk+CpVyJSkopwJlzXT07tMoDL5WgX9
O08KVgDNz9qP/IGtAcRduRcNioH3E9v981QO1zt/Gpb2f8NqAjUUCUZzOnij6mx9
McZ+9cWX88CRzR0vQODWuZscgI08NvM69Fn2SQIDAQABo2MwYTAOBgNVHQ8BAf8E
BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUc19g2LzLA5j0Kxc0LjZa
pmD/vB8wHwYDVR0jBBgwFoAUc19g2LzLA5j0Kxc0LjZapmD/vB8wDQYJKoZIhvcN
AQELBQADggEBAHAG7WTmyjzPRIM85rVj+fWHsLIvqpw6DObIjMWokpliCeMINZFV
ynfgBKsf1ExwbvJNzYFXW6dihnguDG9VMPpi2up/ctQTN8tm9nDKOy08uNZoofMc
NUZxKCEkVKZv+IL4oHoeayt8egtv3ujJM6V14AstMQ6SwvwvA93EP/Ug2e4WAXHu
cbI1NAbUgVDqp+DRdfvZkgYKryjTWd/0+1fS8X1bBZVWzl7eirNVnHbSH2ZDpNuY
0SBd8dj5F6ld3t58ydZbrTHze7JJOd8ijySAp4/kiu9UfZWuTPABzDa/DSdz9Dk/
zPW4CXXvhLmE02TA9/HeCw3KEHIwicNuEfw=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIECDCCAvCgAwIBAgIDAIVCMA0GCSqGSIb3DQEBCwUAMIGPMQswCQYDVQQGEwJV
UzEQMA4GA1UEBwwHU2VhdHRsZTETMBEGA1UECAwKV2FzaGluZ3RvbjEiMCAGA1UE
CgwZQW1hem9uIFdlYiBTZXJ2aWNlcywgSW5jLjETMBEGA1UECwwKQW1hem9uIFJE
UzEgMB4GA1UEAwwXQW1hem9uIFJEUyBSb290IDIwMTkgQ0EwHhcNMTkwOTEzMTcw
NjQxWhcNMjQwODIyMTcwODUwWjCBlDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldh
c2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxIjAgBgNVBAoMGUFtYXpvbiBXZWIg
U2VydmljZXMsIEluYy4xEzARBgNVBAsMCkFtYXpvbiBSRFMxJTAjBgNVBAMMHEFt
YXpvbiBSRFMgdXMtZWFzdC0yIDIwMTkgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB
DwAwggEKAoIBAQDE+T2xYjUbxOp+pv+gRA3FO24+1zCWgXTDF1DHrh1lsPg5k7ht
2KPYzNc+Vg4E+jgPiW0BQnA6jStX5EqVh8BU60zELlxMNvpg4KumniMCZ3krtMUC
au1NF9rM7HBh+O+DYMBLK5eSIVt6lZosOb7bCi3V6wMLA8YqWSWqabkxwN4w0vXI
8lu5uXXFRemHnlNf+yA/4YtN4uaAyd0ami9+klwdkZfkrDOaiy59haOeBGL8EB/c
dbJJlguHH5CpCscs3RKtOOjEonXnKXldxarFdkMzi+aIIjQ8GyUOSAXHtQHb3gZ4
nS6Ey0CMlwkB8vUObZU9fnjKJcL5QCQqOfwvAgMBAAGjZjBkMA4GA1UdDwEB/wQE
AwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBQUPuRHohPxx4VjykmH
6usGrLL1ETAfBgNVHSMEGDAWgBRzX2DYvMsDmPQrFzQuNlqmYP+8HzANBgkqhkiG
9w0BAQsFAAOCAQEAUdR9Vb3y33Yj6X6KGtuthZ08SwjImVQPtknzpajNE5jOJAh8
quvQnU9nlnMO85fVDU1Dz3lLHGJ/YG1pt1Cqq2QQ200JcWCvBRgdvH6MjHoDQpqZ
HvQ3vLgOGqCLNQKFuet9BdpsHzsctKvCVaeBqbGpeCtt3Hh/26tgx0rorPLw90A2
V8QSkZJjlcKkLa58N5CMM8Xz8KLWg3MZeT4DmlUXVCukqK2RGuP2L+aME8dOxqNv
OnOz1zrL5mR2iJoDpk8+VE/eBDmJX40IJk6jBjWoxAO/RXq+vBozuF5YHN1ujE92
tO8HItgTp37XT8bJBAiAnt5mxw+NLSqtxk2QdQ==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIID/zCCAuegAwIBAgIRAJYlnmkGRj4ju/2jBQsnXJYwDQYJKoZIhvcNAQELBQAw
gZcxCzAJBgNVBAYTAlVTMSIwIAYDVQQKDBlBbWF6b24gV2ViIFNlcnZpY2VzLCBJ
bmMuMRMwEQYDVQQLDApBbWF6b24gUkRTMQswCQYDVQQIDAJXQTEwMC4GA1UEAwwn
QW1hem9uIFJEUyB1cy1lYXN0LTIgUm9vdCBDQSBSU0EyMDQ4IEcxMRAwDgYDVQQH
DAdTZWF0dGxlMCAXDTIxMDUyMTIzMDQ0NFoYDzIwNjEwNTIyMDAwNDQ0WjCBlzEL
MAkGA1UEBhMCVVMxIjAgBgNVBAoMGUFtYXpvbiBXZWIgU2VydmljZXMsIEluYy4x
EzARBgNVBAsMCkFtYXpvbiBSRFMxCzAJBgNVBAgMAldBMTAwLgYDVQQDDCdBbWF6
b24gUkRTIHVzLWVhc3QtMiBSb290IENBIFJTQTIwNDggRzExEDAOBgNVBAcMB1Nl
YXR0bGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC74V3eigv+pCj5
nqDBqplY0Jp16pTeNB06IKbzb4MOTvNde6QjsZxrE1xUmprT8LxQqN9tI3aDYEYk
b9v4F99WtQVgCv3Y34tYKX9NwWQgwS1vQwnIR8zOFBYqsAsHEkeJuSqAB12AYUSd
Zv2RVFjiFmYJho2X30IrSLQfS/IE3KV7fCyMMm154+/K1Z2IJlcissydEAwgsUHw
edrE6CxJVkkJ3EvIgG4ugK/suxd8eEMztaQYJwSdN8TdfT59LFuSPl7zmF3fIBdJ
//WexcQmGabaJ7Xnx+6o2HTfkP8Zzzzaq8fvjAcvA7gyFH5EP26G2ZqMG+0y4pTx
SPVTrQEXAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIWWuNEF
sUMOC82XlfJeqazzrkPDMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOC
AQEAgClmxcJaQTGpEZmjElL8G2Zc8lGc+ylGjiNlSIw8X25/bcLRptbDA90nuP+q
zXAMhEf0ccbdpwxG/P5a8JipmHgqQLHfpkvaXx+0CuP++3k+chAJ3Gk5XtY587jX
+MJfrPgjFt7vmMaKmynndf+NaIJAYczjhJj6xjPWmGrjM3MlTa9XesmelMwP3jep
bApIWAvCYVjGndbK9byyMq1nyj0TUzB8oJZQooaR3MMjHTmADuVBylWzkRMxbKPl
4Nlsk4Ef1JvIWBCzsMt+X17nuKfEatRfp3c9tbpGlAE/DSP0W2/Lnayxr4RpE9ds
ICF35uSis/7ZlsftODUe8wtpkQ==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIF/zCCA+egAwIBAgIRAPvvd+MCcp8E36lHziv0xhMwDQYJKoZIhvcNAQEMBQAw
gZcxCzAJBgNVBAYTAlVTMSIwIAYDVQQKDBlBbWF6b24gV2ViIFNlcnZpY2VzLCBJ
bmMuMRMwEQYDVQQLDApBbWF6b24gUkRTMQswCQYDVQQIDAJXQTEwMC4GA1UEAwwn
QW1hem9uIFJEUyB1cy1lYXN0LTIgUm9vdCBDQSBSU0E0MDk2IEcxMRAwDgYDVQQH
DAdTZWF0dGxlMCAXDTIxMDUyMTIzMTEwNloYDzIxMjEwNTIyMDAxMTA2WjCBlzEL
MAkGA1UEBhMCVVMxIjAgBgNVBAoMGUFtYXpvbiBXZWIgU2VydmljZXMsIEluYy4x
EzARBgNVBAsMCkFtYXpvbiBSRFMxCzAJBgNVBAgMAldBMTAwLgYDVQQDDCdBbWF6
b24gUkRTIHVzLWVhc3QtMiBSb290IENBIFJTQTQwOTYgRzExEDAOBgNVBAcMB1Nl
YXR0bGUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDbvwekKIKGcV/s
lDU96a71ZdN2pTYkev1X2e2/ICb765fw/i1jP9MwCzs8/xHBEQBJSxdfO4hPeNx3
ENi0zbM+TrMKliS1kFVe1trTTEaHYjF8BMK9yTY0VgSpWiGxGwg4tshezIA5lpu8
sF6XMRxosCEVCxD/44CFqGZTzZaREIvvFPDTXKJ6yOYnuEkhH3OcoOajHN2GEMMQ
ShuyRFDQvYkqOC/Q5icqFbKg7eGwfl4PmimdV7gOVsxSlw2s/0EeeIILXtHx22z3
8QBhX25Lrq2rMuaGcD3IOMBeBo2d//YuEtd9J+LGXL9AeOXHAwpvInywJKAtXTMq
Wsy3LjhuANFrzMlzjR2YdjkGVzeQVx3dKUzJ2//Qf7IXPSPaEGmcgbxuatxjnvfT
H85oeKr3udKnXm0Kh7CLXeqJB5ITsvxI+Qq2iXtYCc+goHNR01QJwtGDSzuIMj3K
f+YMrqBXZgYBwU2J/kCNTH31nfw96WTbOfNGwLwmVRDgguzFa+QzmQsJW4FTDMwc
7cIjwdElQQVA+Gqa67uWmyDKAnoTkudmgAP+OTBkhnmc6NJuZDcy6f/iWUdl0X0u
/tsfgXXR6ZovnHonM13ANiN7VmEVqFlEMa0VVmc09m+2FYjjlk8F9sC7Rc4wt214
7u5YvCiCsFZwx44baP5viyRZgkJVpQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/
MB0GA1UdDgQWBBQgCZCsc34nVTRbWsniXBPjnUTQ2DAOBgNVHQ8BAf8EBAMCAYYw
DQYJKoZIhvcNAQEMBQADggIBAAQas3x1G6OpsIvQeMS9BbiHG3+kU9P/ba6Rrg+E
lUz8TmL04Bcd+I+R0IyMBww4NznT+K60cFdk+1iSmT8Q55bpqRekyhcdWda1Qu0r
JiTi7zz+3w2v66akofOnGevDpo/ilXGvCUJiLOBnHIF0izUqzvfczaMZGJT6xzKq
PcEVRyAN1IHHf5KnGzUlVFv9SGy47xJ9I1vTk24JU0LWkSLzMMoxiUudVmHSqJtN
u0h+n/x3Q6XguZi1/C1KOntH56ewRh8n5AF7c+9LJJSRM9wunb0Dzl7BEy21Xe9q
03xRYjf5wn8eDELB8FZPa1PrNKXIOLYM9egdctbKEcpSsse060+tkyBrl507+SJT
04lvJ4tcKjZFqxn+bUkDQvXYj0D3WK+iJ7a8kZJPRvz8BDHfIqancY8Tgw+69SUn
WqIb+HNZqFuRs16WFSzlMksqzXv6wcDSyI7aZOmCGGEcYW9NHk8EuOnOQ+1UMT9C
Qb1GJcipjRzry3M4KN/t5vN3hIetB+/PhmgTO4gKhBETTEyPC3HC1QbdVfRndB6e
U/NF2U/t8U2GvD26TTFLK4pScW7gyw4FQyXWs8g8FS8f+R2yWajhtS9++VDJQKom
fAUISoCH+PlPRJpu/nHd1Zrddeiiis53rBaLbXu2J1Q3VqjWOmtj0HjxJJxWnYmz
Pqj2
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIICrDCCAjOgAwIBAgIQGcztRyV40pyMKbNeSN+vXTAKBggqhkjOPQQDAzCBljEL
MAkGA1UEBhMCVVMxIjAgBgNVBAoMGUFtYXpvbiBXZWIgU2VydmljZXMsIEluYy4x
EzARBgNVBAsMCkFtYXpvbiBSRFMxCzAJBgNVBAgMAldBMS8wLQYDVQQDDCZBbWF6
b24gUkRTIHVzLWVhc3QtMiBSb290IENBIEVDQzM4NCBHMTEQMA4GA1UEBwwHU2Vh
dHRsZTAgFw0yMTA1MjEyMzE1NTZaGA8yMTIxMDUyMjAwMTU1NlowgZYxCzAJBgNV
BAYTAlVTMSIwIAYDVQQKDBlBbWF6b24gV2ViIFNlcnZpY2VzLCBJbmMuMRMwEQYD
VQQLDApBbWF6b24gUkRTMQswCQYDVQQIDAJXQTEvMC0GA1UEAwwmQW1hem9uIFJE
UyB1cy1lYXN0LTIgUm9vdCBDQSBFQ0MzODQgRzExEDAOBgNVBAcMB1NlYXR0bGUw
djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQfDcv+GGRESD9wT+I5YIPRsD3L+/jsiIis
Tr7t9RSbFl+gYpO7ZbDXvNbV5UGOC5lMJo/SnqFRTC6vL06NF7qOHfig3XO8QnQz
6T5uhhrhnX2RSY3/10d2kTyHq3ZZg3+jQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYD
VR0OBBYEFLDyD3PRyNXpvKHPYYxjHXWOgfPnMA4GA1UdDwEB/wQEAwIBhjAKBggq
hkjOPQQDAwNnADBkAjB20HQp6YL7CqYD82KaLGzgw305aUKw2aMrdkBR29J183jY
6Ocj9+Wcif9xnRMS+7oCMAvrt03rbh4SU9BohpRUcQ2Pjkh7RoY0jDR4Xq4qzjNr
5UFr3BXpFvACxXF51BksGQ==
-----END CERTIFICATE-----