rebase and tidy up

globs over, tries to prune trees finding the correct files

bin/build/src/build.clj

@@ -89,7 +89,7 @@
                    (build-frontend! edition))
    :licenses     (fn [{:keys [edition]}]
                    (build-licenses! edition))
-   :drivers      (fn [{:keys [edition]}]
+   #_ #_ :drivers      (fn [{:keys [edition]}]
                    (build-drivers/build-drivers! edition))
    :python       (fn [{:keys [edition]}]
                    (build.python/build-python-deps! edition))

bin/build/src/build/uberjar.clj

@@ -9,7 +9,7 @@
    [metabuild-common.core :as u]
    [org.corfield.log4j2-conflict-handler :refer [log4j2-conflict-handler]])
   (:import
-   (java.io OutputStream)
+   (java.io File OutputStream)
    (java.nio.file Files OpenOption StandardOpenOption)
    (java.util.jar Manifest)))
 
@@ -47,7 +47,8 @@
 
 (defn- create-basis [edition]
   {:pre [(#{:ee :oss} edition)]}
-  (b/create-basis {:project "deps.edn", :aliases #{edition}}))
+  (b/create-basis {:project "deps.edn",
+                   :aliases #{edition :drivers}}))
 
 (defn- all-paths [basis]
   (concat (:paths basis)
@@ -78,6 +79,18 @@
    (ns.deps/graph)
    ns-decls))
 
+(defn- all-drivers []
+  (->> (.listFiles (io/file (u/filename u/project-root-directory "modules" "drivers")))
+       (filter (fn [^File d]                                        ;
+                 (and
+                  ;; watch for errant DS_Store files on os_x
+                  (.isDirectory d)
+                  ;; ignore stuff like .cpcache
+                  (not (.isHidden d))
+                  ;; only consider a directory to be a driver if it contains a lein or deps build file
+                  (.exists (io/file d "deps.edn")))))
+       (map (comp symbol #(str "metabase.driver." %) #(.getName ^File %)))))
+
 (defn- metabase-namespaces-in-topo-order [basis]
   (let [ns-decls   (into []
                          (comp (map io/file)
@@ -88,7 +101,7 @@
                         ns.deps/topo-sort
                         (filter ns-symbols))
         orphans    (remove (set sorted) ns-symbols)
-        all        (concat orphans sorted)]
+        all        (concat orphans sorted (all-drivers))]
     (assert (contains? (set all) 'metabase.core.bootstrap))
     (when (contains? ns-symbols 'metabase-enterprise.core.dummy-namespace)
       (assert (contains? (set all) 'metabase-enterprise.core.dummy-namespace)))
@@ -121,7 +134,10 @@
    ;; clean Docker env, so it's more of a nice to have to keep the clutter in our JARs down when building locally.
    #"\~$"
    #"^\.?#"
-   #"\.rej$"])
+   #"\.rej$"
+   ;; Driver classes are now flattened directly into the uberjar via the :drivers alias — the old nested
+   ;; driver JARs in resources/modules/ must not be included or we'd ship everything twice.
+   #"^modules/"])
 
 (defn- copy-resources! [basis]
   (u/step "Copy resources"
@@ -141,7 +157,11 @@
    ;; tf` -- see Slacc thread
    ;; https://metaboat.slack.com/archives/C5XHN8GLW/p1731633690703149?thread_ts=1731504670.951389&cid=C5XHN8GLW
    #".*module-info\.class"
-   #"^LICENSE$"])
+   #"^LICENSE$"
+   #".*LICENSE.jol.txt"
+   #"META-INF/license/LICENSE.jol.txt"
+   #"META-INF/license.*"
+   #"META-INF/LICENSE.*"])
 
 (defn- create-uberjar! [basis]
   (u/step "Create uberjar"

enterprise/backend/src/metabase_enterprise/audit_app/audit.clj

@@ -23,19 +23,6 @@
 
 (set! *warn-on-reflection* true)
 
-(defn- get-jar-path
-  "Returns the path to the currently running jar file.
-
-  More info: https://stackoverflow.com/questions/320542/how-to-get-the-path-of-a-running-jar-file"
-  []
-  (assert (config/jar?) "Can only get-jar-path when running from a jar.")
-  (-> (class {})
-      (.getProtectionDomain)
-      (.getCodeSource)
-      (.getLocation)
-      (.toURI) ;; avoid problems with special characters in path.
-      (.getPath)))
-
 (defn copy-from-jar!
   "Recursively copies a subdirectory (at resource-path) from the jar at jar-path into out-dir.
 
@@ -199,8 +186,8 @@
   (let [ia-dir (instance-analytics-plugin-dir plugins-dir)]
     (when (fs/exists? (u.files/relative-path ia-dir))
       (fs/delete-tree (u.files/relative-path ia-dir)))
-    (if (config/jar?)
-      (let [path-to-jar (get-jar-path)]
+    (if (u.files/running-from-jar?)
+      (let [path-to-jar (u.files/get-jar-path)]
         (log/info "The app is running from a jar, starting copy...")
         (log/info (str "Copying " path-to-jar "::" jar-resource-path " -> " plugins-dir))
         (copy-from-jar! path-to-jar jar-resource-path plugins-dir)

modules/drivers/athena/resources/metabase/athena/metabase-plugin.yaml

@@ -0,0 +1,64 @@
+# Complete list of options here: https://github.com/metabase/metabase/wiki/Metabase-Plugin-Manifest-Reference
+info:
+  name: Metabase Athena Driver
+  version: 1.0.0-athena-jdbc-2.0.13
+  description: Allows Metabase to connect to AWS Athena databases.
+driver:
+  name: athena
+  display-name: Amazon Athena
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - name: region
+      display-name: Region
+      placeholder: "us-east-1"
+      required: true
+    - name: workgroup
+      display-name: Workgroup
+      placeholder: "primary"
+    - name: s3_staging_dir
+      display-name: S3 staging directory
+      helper-text: This S3 staging directory must be in the same region you specify above.
+      placeholder: s3://your_bucket
+      required: true
+    - merge:
+      - dbname
+      - name: catalog
+        placeholder: AwsDataCatalog
+        display-name: Catalog
+        helper-text: Use a different data catalog (if you have federated queries, for example)
+        required: false
+    - name: access_key
+      display-name: Access key
+      helper-text: Leave this empty to authorize using AWS Credentials Provider Chain (Instance Profiles or IAM Roles for Tasks)
+    - merge:
+        - password
+        - name: secret_key
+          display-name: Secret key
+          helper-text: Leave this empty to authorize using AWS Credentials Provider Chain (Instance Profiles or IAM Roles for Tasks)
+    - name: hostname
+      display-name: Full hostname
+      placeholder: "Optional"
+      helper-text: "This is useful when Metabase is deployed on a VPC or AWS Secret Cloud."
+      required: false
+    - name: dbname
+      display-name: Specific database
+      helper-text: Specify a single database to enable features like DB routing
+      required: false
+    - advanced-options-start
+    - merge:
+        - additional-options
+        - display-name: Additional Athena connection string options
+          placeholder: "UseResultsetStreaming=0;LogLevel=6"
+    - name: include-user-id-and-hash
+      display-name: Include User ID and query hash in queries
+      default: true
+      type: boolean
+      visible-if:
+        advanced-options: true
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.athena
+  - step: register-jdbc-driver
+    class: com.amazon.athena.jdbc.AthenaDriver

modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml

@@ -0,0 +1,53 @@
+info:
+  name: Metabase BigQuery Cloud SDK Driver
+  version: 1.0.0-SNAPSHOT-1.131.1
+  description: Allows Metabase to connect to Google BigQuery databases using the Google Cloud SDK.
+driver:
+  name: bigquery-cloud-sdk
+  display-name: BigQuery
+  lazy-load: true
+  parent:
+    - sql
+  connection-properties:
+    - name: project-id
+      display-name: Project ID (override)
+      helper-text: Project ID to be used for authentication. You can omit this field if you are only querying datasets owned by your organization.
+      required: false
+      placeholder: 1w08oDRKPrOqBt06yxY8uiCz2sSvOp3u
+    - name: service-account-json
+      display-name: Service account JSON file
+      helper-text: This JSON file contains the credentials Metabase needs to read and query your dataset.
+      required: true
+      type: textFile
+    - name: dataset-filters
+      type: schema-filters
+      display-name: Datasets
+    - cloud-ip-address-info
+    - advanced-options-start
+    - name: use-jvm-timezone
+      display-name: Use JVM Time Zone
+      default: false
+      type: boolean
+      visible-if:
+        advanced-options: true
+    - name: include-user-id-and-hash
+      display-name: Include User ID and query hash in queries
+      default: true
+      type: boolean
+      visible-if:
+        advanced-options: true
+    - name: host
+      display-name: Alternate hostname
+      helper-text: Alternate hostname used to connect to BigQuery
+      placeholder: https://<hostname>:<port>
+      required: false
+      type: string
+      visible-if:
+        advanced-options: true
+    - default-advanced-options
+extra:
+  db-routing-info:
+    text: Route queries on this database to a different BigQuery **project** or Google Cloud account based on the person's user attribute. Each BigQuery destination must have identical schemas.
+init:
+  - step: load-namespace
+    namespace: metabase.driver.bigquery-cloud-sdk

modules/drivers/clickhouse/resources/metabase/clickhouse/metabase-plugin.yaml

@@ -0,0 +1,65 @@
+info:
+  name: Metabase ClickHouse Driver
+  version: 1.53.3
+  description: Allows Metabase to connect to ClickHouse databases.
+driver:
+  name: clickhouse
+  display-name: ClickHouse
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - host
+    - merge:
+        - port
+        - default: 8123
+    - user
+    - password
+    - name: enable-multiple-db
+      display-name: Enable multiple databases
+      type: boolean
+      default: true
+      description: Enable syncing and scanning tables from multiple ClickHouse databases except the system ones.
+    - name: dbname
+      visible-if:
+        enable-multiple-db: false
+      display-name: Default database
+      placeholder: default
+      helper-text: "Specify the default database to connect to"
+    - name: db-filters
+      visible-if:
+        enable-multiple-db: true
+      type: schema-filters
+      display-name: Databases
+      placeholder: >-
+        database_1, database_2
+      helper-text: Optionally filter which databases are visible.
+
+    - ssl
+    - ssh-tunnel
+    - advanced-options-start
+    - name: use-no-proxy
+      display-name: Disable system wide proxy settings
+      default: false
+      type: boolean
+      visible-if:
+        advanced-options: true
+    - name: clickhouse-settings
+      display-name: ClickHouse settings (comma-separated)
+      placeholder: "allow_experimental_analyzer=1,max_result_rows=100"
+      visible-if:
+        advanced-options: true
+    - name: max-open-connections
+      display-name: "Max open HTTP connections in the JDBC driver (default: 100)"
+      placeholder: 100
+      visible-if:
+        advanced-options: true
+    - merge:
+        - additional-options
+        - placeholder: "connection_timeout=1000&socket_timeout=300000"
+    - default-advanced-options
+  connection-properties-include-tunnel-config: true
+init:
+  - step: load-namespace
+    namespace: metabase.driver.clickhouse
+  - step: register-jdbc-driver
+    class: com.clickhouse.jdbc.ClickHouseDriver

modules/drivers/databricks/resources/metabase/databricks/metabase-plugin.yaml

@@ -0,0 +1,83 @@
+info:
+  name: Metabase Databricks Driver
+  version: 1.2.0
+  description: Allows Metabase to connect to Databricks SQL warehouse with personal access tokens or OAuth M2M
+dependencies:
+  - plugin: Metabase Hive Like Abstract Driver
+driver:
+  - name: databricks
+    display-name: Databricks
+    lazy-load: true
+    parent: hive-like
+    connection-properties:
+      - merge:
+        - host
+        - display-name: Host
+          required: true
+          placeholder: xxxxxxxxxx.cloud.databricks.com
+          helper-text: The Databricks host URL. Could also be adb-xxxxx.azuredatabricks.net
+      - name: http-path
+        display-name: HTTP Path
+        helper-text: Found in SQL Warehouses > Connection details
+        required: true
+        placeholder: /sql/1.0/endpoints/abcdef1234567890
+      - name: use-m2m
+        display-name: Use Machine to Machine (M2M) authentication
+        type: boolean
+        default: true
+        helper-text: Use Personal Access Token for authentication.
+      - merge:
+        - password
+        - name: token
+          display-name: Personal Access Token
+          required: true
+          helper-text: Personal Access Token for authentication. Not required if using OAuth M2M.
+          placeholder: dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+          visible-if:
+            use-m2m: false
+      - name: client-id
+        display-name: Service Principal Client ID
+        placeholder: e26ce240-0a12-454f-934f-4c646603cb61
+        helper-text: For M2M OAuth authentication, corresponds to the service principal's Client ID.
+        required: true
+        visible-if:
+          use-m2m: true
+      - merge:
+        - password
+        - name: oauth-secret
+          display-name: Service Principal OAuth Secret
+          placeholder: dosexxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+          helper-text: OAuth Secret for M2M OAuth authentication.
+          required: true
+          visible-if:
+            use-m2m: true
+      - merge:
+        - multi-level-schema
+        - display-name: Enable multiple catalogs
+      - name: catalog
+        display-name: Default Catalog
+        placeholder: default
+        helper-text: Specify the default catalog to connect to.
+        required: true
+      - name: schema-filters
+        type: schema-filters
+        display-name: Catalogs and Schemas
+        placeholder: >-
+          If multiple catalogs enabled:
+              E.x. catalog_1.schema, catalog_2\.*, catalog_3\.auth*
+
+          Otherwise:
+              E.x. schema_1,auth*
+        helper-text: Optionally filter which schemas are visible.
+      - advanced-options-start
+      - merge:
+          - additional-options
+          - placeholder: IgnoreTransactions=0
+      - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.hive-like
+  - step: load-namespace
+    namespace: metabase.driver.databricks
+  - step: register-jdbc-driver
+    class: com.databricks.client.jdbc.Driver

modules/drivers/druid-jdbc/resources/metabase/druid-jdbc/metabase-plugin.yaml

@@ -0,0 +1,55 @@
+info:
+  name: Metabase Druid JDBC Driver
+  version: 1.0.0-SNAPSHOT
+  description: Allows Metabase to connect to Druid databases using JDBC.
+driver:
+  name: druid-jdbc
+  display-name: Druid JDBC
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - merge:
+        - host
+        - placeholder: http://localhost
+          required: true
+    - merge:
+        - port
+        - display-name: Router node port
+          placeholder: 8888
+    - ssh-tunnel
+    - advanced-options-start
+    - name: json-unfolding
+      display-name: Allow unfolding of JSON columns
+      type: boolean
+      visible-if:
+        advanced-options: true
+      description: This enables unfolding JSON columns into their component fields.
+      default: true
+    - name: auth-enabled
+      display-name: Authentication
+      description: This enables the connection with basic authentication.
+      default: false
+      type: boolean
+      visible-if:
+        advanced-options: true
+    - name: auth-username
+      display-name: Username
+      description: Basic authentication username.
+      type: string
+      placeholder: ingestuser
+      visible-if:
+        auth-enabled: true
+    - name: auth-password
+      display-name: Password
+      description: Basic authentication password.
+      placeholder: '**********'
+      type: secret
+      secret-kind: password
+      visible-if:
+        auth-enabled: true
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.druid-jdbc
+  - step: register-jdbc-driver
+    class: org.apache.calcite.avatica.remote.Driver

modules/drivers/druid/resources/metabase/druid/metabase-plugin.yaml

@@ -0,0 +1,43 @@
+info:
+  name: Metabase Druid Driver
+  version: 1.0.0-SNAPSHOT
+  description: Allows Metabase to connect to Druid databases.
+driver:
+  name: druid
+  display-name: Druid
+  lazy-load: true
+  connection-properties:
+    - merge:
+        - host
+        - placeholder: http://localhost
+          required: true
+    - merge:
+        - port
+        - display-name: Broker node port
+          placeholder: 8082
+          required: true
+    - cloud-ip-address-info
+    - ssh-tunnel
+    - advanced-options-start
+    - name: auth-enabled
+      display-name: Authentication header
+      default: false
+      type: boolean
+      visible-if:
+        advanced-options: true
+    - name: auth-username
+      display-name: Username
+      type: string
+      visible-if:
+        auth-enabled: true
+    - name: auth-token
+      display-name: Token
+      type: secret
+      secret-kind: password
+      visible-if:
+        auth-enabled: true
+    - default-advanced-options
+
+init:
+  - step: load-namespace
+    namespace: metabase.driver.druid

modules/drivers/hive-like/resources/metabase/hive-like/metabase-plugin.yaml

@@ -0,0 +1,12 @@
+info:
+  name: Metabase Hive Like Abstract Driver
+  version: 1.0.0
+  description: Parent of hive like drivers as SparkSQL or Databricks
+driver:
+  - name: hive-like
+    lazy-load: true
+    abstract: true
+    parent: sql-jdbc
+init:
+  - step: load-namespace
+    namespace: metabase.driver.hive-like

modules/drivers/mongo/resources/metabase/mongo/metabase-plugin.yaml

@@ -0,0 +1,94 @@
+info:
+  name: Metabase MongoDB Driver
+  version: 1.0.0-SNAPSHOT-3.9.0
+  description: Allows Metabase to connect to MongoDB databases.
+driver:
+  name: mongo
+  display-name: MongoDB
+  lazy-load: true
+  connection-properties:
+    - name: use-conn-uri
+      type: section
+      default: false
+    - name: conn-uri
+      type: string
+      display-name: Paste your connection string
+      placeholder: 'mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[dbname][?options]]'
+      required: true
+      visible-if:
+        use-conn-uri: true
+    - merge:
+        - host
+        - visible-if:
+            use-conn-uri: false
+          required: true
+    - merge:
+        - dbname
+        - visible-if:
+            use-conn-uri: false
+    - merge:
+        - port
+        - placeholder: 27017
+          visible-if:
+            use-conn-uri: false
+    - merge:
+        - user
+        - required: false
+          visible-if:
+            use-conn-uri: false
+    - merge:
+        - password
+        - name: pass
+          visible-if:
+            use-conn-uri: false
+    - name: authdb
+      display-name: Authentication database (optional)
+      placeholder: admin
+      visible-if:
+        use-conn-uri: false
+    - cloud-ip-address-info
+    - merge:
+        - ssl
+        - visible-if:
+            use-conn-uri: false
+    - name: ssl-cert
+      type: string
+      display-name: Server SSL certificate chain (PEM)
+      visible-if:
+        ssl: true
+    - name: ssl-use-client-auth
+      display-name: Authenticate client certificate?
+      type: boolean
+      visible-if:
+        ssl: true
+    - name: client-ssl-cert
+      display-name: Client SSL certificate chain (PEM)
+      placeholder: Paste the contents of the client's SSL certificate chain here
+      type: text
+      visible-if:
+        ssl-use-client-auth: true
+    - name: client-ssl-key
+      display-name: Client SSL private key (PEM)
+      type: secret
+      secret-kind: pem-cert
+      visible-if:
+        ssl-use-client-auth: true
+    - ssh-tunnel
+    - advanced-options-start
+    - merge:
+        - additional-options
+        - display-name: Additional connection string options (optional)
+          placeholder: 'retryWrites=true&w=majority&authSource=admin&readPreference=nearest&replicaSet=test'
+          visible-if:
+            use-conn-uri: false
+            advanced-options: true
+    - name: use-srv
+      type: boolean
+      default: false
+      visible-if:
+        use-conn-uri: false
+        advanced-options: true
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.mongo

modules/drivers/oracle/resources/metabase/oracle/metabase-plugin.yaml

@@ -0,0 +1,80 @@
+info:
+  name: Metabase Oracle Driver
+  version: 1.0.0-SNAPSHOT
+  description: Allows Metabase to connect to Oracle databases.
+dependencies:
+  - class: oracle.jdbc.OracleDriver
+    message: >
+      Metabase requires the Oracle JDBC driver in order to connect to Oracle databases, but we can't ship it as part
+      of Metabase due to licensing restrictions. See
+      https://metabase.com/docs/latest/administration-guide/databases/oracle.html for more details.
+driver:
+  name: oracle
+  display-name: Oracle
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - host
+    - merge:
+        - port
+        - placeholder: 1521
+    - name: sid
+      display-name: Oracle system ID (SID)
+      placeholder: Usually something like ORCL or XE. Optional if using service name
+    - name: service-name
+      display-name: Oracle service name
+      placeholder: Optional TNS alias
+    - merge:
+        - user
+        - required: false
+    - merge:
+        - password
+        - required: false
+    - cloud-ip-address-info
+    - ssl
+    - name: ssl-use-keystore
+      display-name: Use SSL server certificate?
+      type: boolean
+      visible-if:
+        ssl: true
+    - name: ssl-keystore
+      display-name: Keystore (PKCS12 or JKS)
+      type: secret
+      secret-kind: keystore
+      placeholder: /path/to/keystore
+      visible-if:
+        ssl-use-keystore: true
+    - name: ssl-keystore-password
+      display-name: Keystore password
+      type: secret
+      secret-kind: password
+      required: false
+      visible-if:
+        ssl-use-keystore: true
+    - name: ssl-use-truststore
+      display-name: Use SSL truststore?
+      type: boolean
+      visible-if:
+        ssl: true
+    - name: ssl-truststore
+      display-name: Truststore (PKCS12 or JKS)
+      type: secret
+      secret-kind: keystore
+      placeholder: /path/to/truststore
+      visible-if:
+        ssl-use-truststore: true
+    - name: ssl-truststore-password
+      display-name: Truststore password
+      type: secret
+      secret-kind: password
+      required: false
+      visible-if:
+        ssl-use-truststore: true
+    - ssh-tunnel
+    - advanced-options-start
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.oracle
+  - step: register-jdbc-driver
+    class: oracle.jdbc.OracleDriver

modules/drivers/presto-jdbc/resources/metabase/presto-jdbc/metabase-plugin.yaml

@@ -0,0 +1,136 @@
+info:
+  name: Metabase Presto JDBC Driver
+  version: 1.0.0-350-SNAPSHOT
+  description: Allows Metabase to connect to Presto databases using the Presto JDBC driver
+driver:
+  name: presto-jdbc
+  display-name: Presto
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - merge:
+      - host
+      - required: true
+    - merge:
+        - port
+        - placeholder: 8080
+    - merge:
+        - dbname
+        - name: catalog
+          placeholder: european_birds
+          display-name: Catalog
+          helper-text: Presto Catalogs contain schemas and reference data sources via a connector.
+          required: false
+    - name: schema
+      display-name: Schema (optional)
+      helper-text: Only add tables to Metabase that come from a specific schema.
+      placeholder: just_crows
+      required: false
+    - merge:
+        - user
+        - required: false
+    - merge:
+        - password
+        - required: false
+    - cloud-ip-address-info
+    - ssl
+    - name: ssl-use-keystore
+      display-name: Use SSL server certificate?
+      type: boolean
+      visible-if:
+        ssl: true
+    - name: ssl-keystore
+      display-name: Keystore
+      type: secret
+      secret-kind: keystore
+      placeholder: /path/to/keystore.jks
+      visible-if:
+        ssl-use-keystore: true
+    - name: ssl-keystore-password
+      display-name: Keystore password
+      type: secret
+      secret-kind: password
+      required: false
+      visible-if:
+        ssl-use-keystore: true
+    - name: ssl-use-truststore
+      display-name: Use SSL truststore?
+      type: boolean
+      visible-if:
+        ssl: true
+    - name: ssl-truststore
+      display-name: Truststore
+      type: secret
+      secret-kind: keystore
+      placeholder: /path/to/truststore.jks
+      visible-if:
+        ssl-use-truststore: true
+    - name: ssl-truststore-password
+      display-name: Truststore password
+      type: secret
+      secret-kind: password
+      required: false
+      visible-if:
+        ssl-use-truststore: true
+    - advanced-options-start
+    - name: kerberos
+      type: boolean
+      display-name: Authenticate with Kerberos
+      default: false
+      visible-if:
+        advanced-options: true
+    - name: kerberos-principal
+      display-name: Kerberos principal
+      placeholder: service/instance@REALM
+      required: false
+      visible-if:
+        kerberos: true
+    - name: kerberos-remote-service-name
+      display-name: Kerberos coordinator service
+      placeholder: presto
+      required: false
+      visible-if:
+        kerberos: true
+    - name: kerberos-use-canonical-hostname
+      type: boolean
+      display-name: Use canonical hostname
+      default: false
+      required: false
+      visible-if:
+        kerberos: true
+    - name: kerberos-credential-cache-path
+      display-name: Kerberos credential cache file
+      placeholder: /tmp/kerberos-credential-cache
+      required: false
+      visible-if:
+        kerberos: true
+    - name: kerberos-keytab-path
+      display-name: Kerberos keytab file
+      placeholder: /path/to/kerberos.keytab
+      required: false
+      visible-if:
+        kerberos: true
+    - name: kerberos-config-path
+      display-name: Kerberos configuration file
+      placeholder: /etc/krb5.conf
+      required: false
+      visible-if:
+        kerberos: true
+    - name: kerberos-service-principal-pattern
+      display-name: Presto coordinator Kerberos service principal pattern
+      placeholder: ${SERVICE}@${HOST}. ${SERVICE}
+      required: false
+      visible-if:
+        kerberos: true
+    - name: additional-options
+      display-name: Additional JDBC options
+      placeholder: SSLKeyStorePath=/path/to/keystore.jks&SSLKeyStorePassword=password
+      required: false
+      visible-if:
+        advanced-options: true
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.presto-jdbc
+  - step: register-jdbc-driver
+    class: com.facebook.presto.jdbc.PrestoDriver

modules/drivers/redshift/resources/metabase/redshift/metabase-plugin.yaml

@@ -0,0 +1,40 @@
+info:
+  name: Metabase Redshift Driver
+  version: 1.1.0-SNAPSHOT-2.0.0.3
+  description: Allows Metabase to connect to Redshift databases.
+driver:
+  name: redshift
+  display-name: Amazon Redshift
+  lazy-load: true
+  parent: postgres
+  connection-properties:
+    - merge:
+        - host
+        - placeholder: my-cluster-name.abcd1234.us-east-1.redshift.amazonaws.com
+          required: true
+    - merge:
+        - port
+        - placeholder: 5439
+    - merge:
+        - dbname
+        - name: db
+          placeholder: birds_of_the_world
+    - name: schema-filters
+      type: schema-filters
+      display-name: Schemas
+    - user
+    - password
+    - cloud-ip-address-info
+    - ssh-tunnel
+    - advanced-options-start
+    - merge:
+      - additional-options
+      - placeholder: 'SocketTimeout=0'
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.postgres
+  - step: load-namespace
+    namespace: metabase.driver.redshift
+  - step: register-jdbc-driver
+    class: com.amazon.redshift.jdbc42.Driver

modules/drivers/snowflake/resources/metabase/snowflake/metabase-plugin.yaml

@@ -0,0 +1,71 @@
+info:
+  name: Metabase Snowflake Driver
+  version: 1.0.0-SNAPSHOT-3.12.7
+  description: Allows Metabase to connect to Snowflake databases.
+driver:
+  name: snowflake
+  display-name: Snowflake
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - name: use-hostname
+      type: section
+      default: false
+    - merge:
+        - host
+        - visible-if:
+            use-hostname: true
+    - name: account
+      display-name: Account name
+      helper-text: Enter your Account ID with the region that your Snowflake cluster is running on e.g. "xxxxxxxx.us-east-2.aws". Some regions don't have this suffix.
+      placeholder: xxxxxxxx.us-east-2.aws
+      required: true
+      visible-if:
+        use-hostname: false
+    - user
+    - name: use-password
+      display-name: "Authenticate with user and password"
+      description: Snowflake will soon block password authentication.
+      type: boolean
+      default: false
+    - name: private-key
+      display-name: RSA private key (PKCS#8/.p8)
+      type: secret
+      secret-kind: pem-cert
+      visible-if:
+        use-password: false
+      required: true
+    - merge:
+      - password
+      - visible-if:
+          use-password: true
+        required: true
+    - name: warehouse
+      display-name: Warehouse
+      helper-text: If your user doesn't have a default warehouse, enter the warehouse to connect to.
+      placeholder: birds_main
+      required: true
+    - merge:
+        - dbname
+        - name: db
+          required: true
+          display-name: Database name (case sensitive)
+    - name: schema-filters
+      type: schema-filters
+      display-name: Schemas
+    - name: role
+      display-name: Role (optional, required for connection impersonation)
+      helper-text: Specify a role to override the database user’s default role.
+      placeholder: user
+    - cloud-ip-address-info
+    - ssh-tunnel
+    - advanced-options-start
+    - merge:
+      - additional-options
+      - placeholder: 'queryTimeout=0'
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.snowflake
+  - step: register-jdbc-driver
+    class: net.snowflake.client.jdbc.SnowflakeDriver

modules/drivers/sparksql/resources/metabase/sparksql/metabase-plugin.yaml

@@ -0,0 +1,36 @@
+info:
+  name: Metabase Spark SQL Driver
+  version: 1.0.0-SNAPSHOT-1.2.2
+  description: Allows Metabase to connect to Spark SQL databases.
+dependencies:
+  - plugin: Metabase Hive Like Abstract Driver
+driver:
+  - name: sparksql
+    display-name: Spark SQL
+    lazy-load: true
+    parent: hive-like
+    connection-properties:
+      - merge:
+        - host
+        - required: true
+      - merge:
+          - port
+          - placeholder: 10000
+      - merge:
+          - dbname
+          - placeholder: default
+      - user
+      - password
+      - cloud-ip-address-info
+      - ssh-tunnel
+      - advanced-options-start
+      - merge:
+          - additional-options
+          - name: jdbc-flags
+            placeholder: ';transportMode=http'
+      - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.hive-like
+  - step: load-namespace
+    namespace: metabase.driver.sparksql

modules/drivers/sqlite/resources/metabase/sqlite/metabase-plugin.yaml

@@ -0,0 +1,22 @@
+info:
+  name: Metabase SQLite Driver
+  version: 1.0.0-SNAPSHOT-3.25.2
+  description: Allows Metabase to connect to SQLite databases.
+driver:
+  name: sqlite
+  display-name: SQLite
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - name: db
+      display-name: Filename
+      placeholder: /path/to/toucan_sightings.sqlite
+      required: true
+    - cloud-ip-address-info
+    - advanced-options-start
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.sqlite
+  - step: register-jdbc-driver
+    class: org.sqlite.JDBC

modules/drivers/sqlserver/resources/metabase/sqlserver/metabase-plugin.yaml

@@ -0,0 +1,40 @@
+info:
+  name: Metabase SQL Server Driver
+  version: 1.1.0-SNAPSHOT-9.2.1.jre8
+  description: Allows Metabase to connect to SQL Server databases.
+driver:
+  name: sqlserver
+  display-name: SQL Server
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - host
+    - merge:
+        - port
+        - helper-text: Leave empty to use Dynamic Ports, or input specific port. Standard port is 1433.
+    - merge:
+        - dbname
+        - name: db
+          placeholder: BirdsOfTheWorld
+    - name: instance
+      display-name: Database instance name
+      placeholder: N/A
+    - user
+    - password
+    - name: role
+      display-name: Default impersonation user (required for impersonation if the connection's user cannot be impersonated)
+      helper-text: User used to run system queries, required if the connection's user cannot be impersonated, such as the 'sa' user.
+      placeholder: user
+    - cloud-ip-address-info
+    - ssl
+    - ssh-tunnel
+    - advanced-options-start
+    - merge:
+        - additional-options
+        - placeholder: "trustServerCertificate=false"
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.sqlserver
+  - step: register-jdbc-driver
+    class: com.microsoft.sqlserver.jdbc.SQLServerDriver

modules/drivers/starburst/resources/metabase/starburst/metabase-plugin.yaml

@@ -0,0 +1,53 @@
+info:
+  name: Metabase Starburst Driver
+  version: 6.1.0
+  description: Allows Metabase to connect to Starburst Enterprise, Starburst Galaxy, and Trino query engines.
+driver:
+  name: starburst
+  display-name: Starburst (Trino)
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - host
+    - merge:
+        - port
+        - default: 8080
+    - merge:
+        - dbname
+        - name: catalog
+          placeholder: tpch
+          display-name: Catalog
+          required: true
+          helper-text: Starburst Catalogs contain schemas and reference data sources via a connector.
+    - name: schema
+      display-name: Schema (optional)
+      required: false
+      helper-text: Only add tables to Metabase that come from a specific schema.
+    - merge:
+        - user
+        - required: false
+    - merge:
+        - password
+        - required: false
+    - ssl
+    - name: roles
+      display-name: Role (optional)
+      required: false
+      helper-text: Specify a role to override the database user’s default role.
+    - name: prepared-optimized
+      display-name: Optimized prepared statements
+      type: boolean
+      required: false
+      default: false
+      description: Requires Starburst Galaxy, Starburst Enterprise (version 420-e or higher), or Trino (version 418 or higher)
+    - advanced-options-start
+    - merge:
+        - additional-options
+        - placeholder: "trustServerCertificate=false"
+    - default-advanced-options
+
+init:
+  - step: load-namespace
+    namespace: metabase.driver.starburst
+  - step: register-jdbc-driver
+    class: io.trino.jdbc.TrinoDriver

modules/drivers/vertica/resources/metabase/vertica/metabase-plugin.yaml

@@ -0,0 +1,35 @@
+info:
+  name: Metabase Vertica Driver
+  version: 1.0.0-SNAPSHOT
+  description: Allows Metabase to connect to Vertica databases.
+dependencies:
+  - class: com.vertica.jdbc.Driver
+    message: >
+      Metabase requires the Vertica JDBC driver in order to connect to Vertica databases, but we can't ship it as part
+      of Metabase due to licensing restrictions. See
+      https://metabase.com/docs/latest/administration-guide/databases/vertica.html for more details.
+driver:
+  name: vertica
+  display-name: Vertica
+  lazy-load: true
+  parent: sql-jdbc
+  connection-properties:
+    - host
+    - merge:
+        - port
+        - placeholder: 5433
+    - dbname
+    - user
+    - password
+    - cloud-ip-address-info
+    - ssh-tunnel
+    - advanced-options-start
+    - merge:
+        - additional-options
+        - placeholder: 'ConnectionLoadBalance=1'
+    - default-advanced-options
+init:
+  - step: load-namespace
+    namespace: metabase.driver.vertica
+  - step: register-jdbc-driver
+    class: com.vertica.jdbc.Driver

src/metabase/plugins/impl.clj

@@ -6,7 +6,6 @@
    [clojure.string :as str]
    [environ.core :as env]
    [metabase.classloader.core :as classloader]
-   [metabase.config.core :as config]
    [metabase.plugins.initialize :as plugins.init]
    [metabase.util.files :as u.files]
    [metabase.util.i18n :refer [trs]]
@@ -14,7 +13,7 @@
    [metabase.util.yaml :as yaml])
   (:import
    (java.io File)
-   (java.nio.file Files Path)))
+   (java.nio.file Files FileVisitOption Path Paths)))
 
 (set! *warn-on-reflection* true)
 
@@ -65,12 +64,6 @@
   []
   (:path (plugins-dir-info)))
 
-(defn- extract-system-modules! []
-  (when (io/resource "modules")
-    (let [plugins-path (plugins-dir)]
-      (u.files/with-open-path-to-resource [modules-path "modules"]
-        (u.files/copy-files! modules-path plugins-path)))))
-
 ;;; +----------------------------------------------------------------------------------------------------------------+
 ;;; |                                          loading/initializing plugins                                          |
 ;;; +----------------------------------------------------------------------------------------------------------------+
@@ -115,40 +108,36 @@
                              "spark-deps.jar is no longer needed by Metabase 0.32.0+. You can delete it from the plugins directory.")))]
     path))
 
-(when (not config/is-prod?)
-  (defn- load-local-plugin-manifest! [^Path path]
-    (some-> (slurp (str path)) yaml/parse-string plugins.init/init-plugin-with-info!))
+(defn- load-plugin-manifest! [path]
+  (some-> (slurp (str path)) yaml/parse-string plugins.init/init-plugin-with-info!))
 
-  (defn- driver-manifest-paths
-    "Return a sequence of [[java.io.File]] paths for `metabase-plugin.yaml` plugin manifests for drivers on the classpath."
-    []
-    ;; only include plugin manifests if they're on the system classpath.
-    (concat
-     (for [^File file (classpath/system-classpath)
-           :when      (and (.isDirectory file)
-                           (not (.isHidden file))
-                           (str/includes? (str file) "modules/drivers")
-                           (or (str/ends-with? (str file) "resources")
-                               (str/ends-with? (str file) "resources-ee")))
-           :let       [manifest-file (io/file file "metabase-plugin.yaml")]
-           :when      (.exists manifest-file)]
-       manifest-file)
-     ;; for hacking on 3rd-party drivers locally: set
-     ;; `-Dmb.dev.additional.driver.manifest.paths=/path/to/whatever/metabase-plugin.yaml` or
-     ;; `MB_DEV_ADDITIONAL_DRIVER_MANIFEST_PATHS=...` to have that plugin manifest get loaded during startup. Specify
-     ;; multiple plugin manifests by comma-separating them.
-     (when-let [additional-paths (env/env :mb-dev-additional-driver-manifest-paths)]
-       (map u.files/get-path (str/split additional-paths #",")))))
+(defn- driver-manifest-paths
+  "Return a sequence of paths (URIs when running from a jar, Paths in dev) for `metabase-plugin.yaml` plugin manifests
+  for drivers on the classpath. Driver manifests live at `metabase/<driver-name>/metabase-plugin.yaml` on the classpath."
+  []
+  (if (u.files/running-from-jar?)
+    (u.files/find-in-current-jar "glob:/metabase/*/metabase-plugin.yaml")
+    (let [matcher (.getPathMatcher (java.nio.file.FileSystems/getDefault) "glob:**/metabase/*/metabase-plugin.yaml")]
+      (for [^File f (classpath/system-classpath)
+            :when (.isDirectory f)
+            founds [(-> (.toURI f)
+                        (Paths/get)
+                        (Files/find 3
+                                    (fn [path _attr] (.matches matcher path))
+                                    (make-array FileVisitOption 0))
+                        (.collect (java.util.stream.Collectors/toList)))]
+            f founds]
+        f))))
 
-  (defn- load-local-plugin-manifests!
-    "Load local plugin manifest files when not running in a production mode, to simulate what would happen when loading those
-  same plugins from the uberjar. This is needed because some plugin manifests define driver methods and the like that
-  aren't defined elsewhere."
-    []
-    ;; TODO - this should probably do an actual search in case we ever add any additional directories
-    (doseq [manifest-path (driver-manifest-paths)]
-      (log/infof "Loading local plugin manifest at %s" (str manifest-path))
-      (load-local-plugin-manifest! manifest-path))))
+(defn- load-driver-plugin-manifests!
+  "Find and load driver plugin manifests from the classpath. In the uberjar, driver classes are flattened directly into
+  the jar (rather than shipped as nested JARs in a `modules/` directory) and their manifests live at
+  `metabase/<driver>/metabase-plugin.yaml`. In dev, the same manifests are found via the driver resource directories on
+  the classpath."
+  []
+  (doseq [manifest-path (driver-manifest-paths)]
+    (log/infof "Loading driver plugin manifest at %s" (str manifest-path))
+    (load-plugin-manifest! manifest-path)))
 
 (defn- has-manifest? ^Boolean [^Path path]
   (boolean (u.files/file-exists-in-archive? path "metabase-plugin.yaml")))
@@ -166,29 +155,29 @@
         (log/errorf e "Failed to initialize plugin %s" (.getFileName path))))))
 
 (defn- load! []
+  ;; Load any user-supplied plugin JARs from the plugins directory (e.g. Oracle JDBC driver).
+  ;; System/bundled drivers are no longer extracted to disk — they are flattened into the uberjar
+  ;; and their manifests are loaded from the classpath via load-driver-plugin-manifests! below.
   (log/infof "Loading plugins in %s..." (str (plugins-dir)))
-  (extract-system-modules!)
   (let [paths (plugins-paths)]
     (init-plugins! paths))
-  (when (not config/is-prod?)
-    (load-local-plugin-manifests!)))
+  (load-driver-plugin-manifests!))
 
 (defonce ^:private loaded? (atom false))
 
 (defn load-plugins!
-  "Load Metabase plugins. The are JARs shipped as part of Metabase itself, under the `resources/modules` directory (the
-  source for these JARs is under the `modules` directory); and others manually added by users to the Metabase plugins
-  directory, which defaults to `./plugins`.
+  "Load Metabase plugins. Bundled drivers are compiled directly into the uberjar and their plugin manifests are
+  discovered on the classpath at `metabase/<driver>/metabase-plugin.yaml`. User-supplied plugins (e.g. Oracle JDBC
+  driver JARs) are loaded from the plugins directory, which defaults to `./plugins`.
 
   When loading plugins, Metabase performs the following steps:
 
   *  Metabase creates the plugins directory if it does not already exist.
-  *  Any plugins that are shipped as part of Metabase itself are extracted from the Metabase uberjar (or `resources`
-     directory when running with the Clojure CLI) into the plugins directory.
   *  Each JAR in the plugins directory that *does not* include a Metabase plugin manifest is added to the classpath.
   *  For JARs that include a Metabase plugin manifest (a `metabase-plugin.yaml` file), a lazy-loading Metabase driver
      is registered; when the driver is initialized (automatically, when certain methods are called) the JAR is added
-     to the classpath and the driver namespace is loaded
+     to the classpath and the driver namespace is loaded.
+  *  Bundled driver plugin manifests are loaded from the classpath (not from disk).
 
   This function will only perform loading steps the first time it is called — it is safe to call this function more
   than once."

src/metabase/util/files.clj

@@ -14,8 +14,9 @@
    [metabase.util.log :as log])
   (:import
    (java.io FileNotFoundException)
-   (java.net URL)
-   (java.nio.file CopyOption Files FileSystem FileSystemAlreadyExistsException FileSystems
+   (java.net URI URL)
+   (java.nio.file CopyOption Files FileSystem FileSystemAlreadyExistsException
+                  FileSystems FileVisitResult
                   LinkOption OpenOption Path Paths StandardCopyOption)
    (java.nio.file.attribute FileAttribute)
    (java.util Collections)
@@ -183,3 +184,57 @@
   "Returns a java.nio.file.Path"
   [path]
   (fs/relativize (fs/absolutize ".") path))
+
+(defn running-from-jar?
+  "Returns true iff we are running from a jar.
+
+  .getResource will return a java.net.URL, and those start with \"jar:\" if and only if the app is running from a jar.
+
+  More info: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/Thread.html"
+  []
+  (= "jar" (.. (Thread/currentThread)
+               getContextClassLoader
+               (getResource ".keep-me")
+               getProtocol)))
+
+(defn get-jar-path
+  "Returns the path to the currently running jar file.
+
+  More info: https://stackoverflow.com/questions/320542/how-to-get-the-path-of-a-running-jar-file"
+  []
+  (assert (running-from-jar?) "Can only get-jar-path when running from a jar.")
+  (-> (class {})
+      (.getProtectionDomain)
+      (.getCodeSource)
+      (.getLocation)
+      (.toURI) ;; avoid problems with special characters in path.
+      (.getPath)))
+
+(defn find-in-current-jar
+  "Find matching files in the current jar. See
+  https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String- to
+  understand the syntax of [[pattern]].  Motivating use case is:
+
+  `(u.files/find-in-current-jar \"glob:/metabase/*/metabase-plugin.yaml\")`
+
+ to find the plugin manifests in the jar."
+  [pattern]
+  ;; Get the location of the current JAR
+  (let [jar-path (get-jar-path)
+        jar-uri (URI. (str "jar:file:" jar-path))]
+    (with-open [fs (FileSystems/newFileSystem jar-uri (java.util.HashMap.))]
+      (let [matcher (.getPathMatcher fs pattern)
+            root (.getPath fs "/" (into-array String []))
+            files (atom [])]
+        (Files/walkFileTree root
+                            (proxy [java.nio.file.SimpleFileVisitor] []
+                              (preVisitDirectory [dir _attrs]
+                                (if (or (str/starts-with? (str dir) "/metabase")
+                                        (= (str "/") (str dir)))
+                                  java.nio.file.FileVisitResult/CONTINUE
+                                  java.nio.file.FileVisitResult/SKIP_SUBTREE))
+                              (visitFile [file _attrs]
+                                (when (.matches matcher file)
+                                  (swap! files conj file))
+                                java.nio.file.FileVisitResult/CONTINUE)))
+        (into [] (map java.nio.file.Path/.toUri) @files)))))