mark date filter picker tests flaky (#54866)

This schema is actually used! 🎉

This reverts commit f5dbd78051.

.clj-kondo/config.edn

@@ -464,6 +464,7 @@
     metabase.sync.field-values                                    sync.field-values
     metabase.sync.interface                                       i
     metabase.sync.schedules                                       sync.schedules
+    metabase.sync.sync-metadata.crufty                            crufty
     metabase.sync.sync-metadata.fields.our-metadata               fields.our-metadata
     metabase.sync.util                                            sync-util
     metabase.sync.util-test                                       sync.util-test

.clj-kondo/config/modules/config.edn

@@ -60,7 +60,8 @@
    :uses #{config driver legacy-mbql lib models sync util}}
 
   api
-  {:api #{metabase.api.cache
+  {:api #{metabase.api.auth
+          metabase.api.cache
           metabase.api.card
           metabase.api.collection
           metabase.api.common
@@ -603,7 +604,8 @@
   enterprise/advanced-config
   {:api  #{metabase-enterprise.advanced-config.api.logs
            metabase-enterprise.advanced-config.models.pulse-channel
-           metabase-enterprise.advanced-config.file}
+           metabase-enterprise.advanced-config.file
+           metabase-enterprise.advanced-config.crufty}
    :uses #{api db driver models premium-features setup sync util}}
 
   enterprise/advanced-permissions
@@ -665,6 +667,14 @@
   {:api  #{metabase-enterprise.enhancements.init}
    :uses #{models premium-features sso util enterprise/sso}}
 
+  enterprise/gsheets
+  {:api  #{metabase-enterprise.gsheets.api}
+   :uses #{api analytics enterprise/harbormaster models premium-features server util}}
+
+  enterprise/harbormaster
+  {:api #{metabase-enterprise.harbormaster.client}
+   :uses #{api cloud-migration models util}}
+
   enterprise/impersonation
   {:api #{metabase-enterprise.impersonation.core
           metabase-enterprise.impersonation.api}
@@ -738,7 +748,7 @@
 
   enterprise/upload-management
   {:api  #{metabase-enterprise.upload-management.api}
-   :uses #{api models upload util}}}
+   :uses #{api models premium-features upload util}}}
 
  ;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals
  ;; aren't allowed in EDN just used the `(str <regex>)` version i.e. two slashes instead of one.

.github/workflows/uberjar.yml

@@ -208,7 +208,7 @@ jobs:
         push: true
 
     - name: Retag and push images if dev branch - should be cached at this point
-      if: ${{ !(startsWith(github.ref_name,'master') || startsWith(github.ref_name,'backport')) && matrix.edition == 'ee' }}
+      if: ${{ !(startsWith(github.ref_name,'master') || startsWith(github.ref_name,'backport')) && matrix.edition == 'ee-extra' }}
       uses: docker/build-push-action@v3
       with:
         context: bin/docker/.

e2e/test/scenarios/onboarding/add-initial-data.cy.spec.ts

@@ -24,7 +24,10 @@ H.describeWithSnowplow(
           testFile.fileName
         } to "Our analytics" using DWH`, () => {
           cy.visit("/");
-          cy.findByTestId("main-navbar-root").findByText("Upload CSV").click();
+          cy.findByTestId("main-navbar-root")
+            .findByText(/Add Data/)
+            .click();
+          H.popover().findByText("Upload CSV").click();
 
           H.uploadFile("#dwh-upload-csv-input", "Our analytics", testFile);
 

e2e/test/scenarios/question-reproductions/reproductions.cy.spec.ts

@@ -20,84 +20,92 @@ describe("issue 39487", () => {
     cy.viewport(1280, 1000);
   });
 
-  it("calendar has constant size when using single date picker filter (metabase#39487)", () => {
-    createTimeSeriesQuestionWithFilter([">", CREATED_AT_FIELD, "2015-01-01"]); // 5 day rows
+  it(
+    "calendar has constant size when using single date picker filter (metabase#39487)",
+    { tags: "@flaky" },
+    () => {
+      createTimeSeriesQuestionWithFilter([">", CREATED_AT_FIELD, "2015-01-01"]); // 5 day rows
 
-    cy.log("timeseries filter button");
-    cy.findByTestId("timeseries-filter-button").click();
-    checkSingleDateFilter();
+      cy.log("timeseries filter button");
+      cy.findByTestId("timeseries-filter-button").click();
+      checkSingleDateFilter();
 
-    cy.log("filter pills");
-    cy.findByTestId("filters-visibility-control").click();
-    cy.findByTestId("filter-pill").click();
-    checkSingleDateFilter();
+      cy.log("filter pills");
+      cy.findByTestId("filters-visibility-control").click();
+      cy.findByTestId("filter-pill").click();
+      checkSingleDateFilter();
 
-    cy.log("filter modal");
-    cy.button(/Filter/).click();
-    H.modal().findByText("After Jan 1, 2015").click();
-    checkSingleDateFilter();
-    H.modal().button("Close").click();
+      cy.log("filter modal");
+      cy.button(/Filter/).click();
+      H.modal().findByText("After Jan 1, 2015").click();
+      checkSingleDateFilter();
+      H.modal().button("Close").click();
 
-    cy.log("filter drill");
-    cy.findByLabelText("Switch to data").click();
-    H.tableHeaderClick("Created At: Year");
-    H.popover().findByText("Filter by this column").click();
-    H.popover().findByText("Specific dates…").click();
-    H.popover().findByText("After").click();
-    H.popover().findByRole("textbox").clear().type("2015/01/01");
-    checkSingleDateFilter();
+      cy.log("filter drill");
+      cy.findByLabelText("Switch to data").click();
+      H.tableHeaderClick("Created At: Year");
+      H.popover().findByText("Filter by this column").click();
+      H.popover().findByText("Specific dates…").click();
+      H.popover().findByText("After").click();
+      H.popover().findByRole("textbox").clear().type("2015/01/01");
+      checkSingleDateFilter();
 
-    cy.log("notebook editor");
-    H.openNotebook();
-    H.getNotebookStep("filter")
-      .findAllByTestId("notebook-cell-item")
-      .first()
-      .click();
-    checkSingleDateFilter();
-  });
+      cy.log("notebook editor");
+      H.openNotebook();
+      H.getNotebookStep("filter")
+        .findAllByTestId("notebook-cell-item")
+        .first()
+        .click();
+      checkSingleDateFilter();
+    },
+  );
 
-  it("calendar has constant size when using date range picker filter (metabase#39487)", () => {
-    createTimeSeriesQuestionWithFilter([
-      "between",
-      CREATED_AT_FIELD,
-      "2024-05-01", // 5 day rows
-      "2024-06-01", // 6 day rows
-    ]);
+  it(
+    "calendar has constant size when using date range picker filter (metabase#39487)",
+    { tags: "@flaky" },
+    () => {
+      createTimeSeriesQuestionWithFilter([
+        "between",
+        CREATED_AT_FIELD,
+        "2024-05-01", // 5 day rows
+        "2024-06-01", // 6 day rows
+      ]);
 
-    cy.log("timeseries filter button");
-    cy.findByTestId("timeseries-filter-button").click();
-    checkDateRangeFilter();
+      cy.log("timeseries filter button");
+      cy.findByTestId("timeseries-filter-button").click();
+      checkDateRangeFilter();
 
-    cy.log("filter pills");
-    cy.findByTestId("filters-visibility-control").click();
-    cy.findByTestId("filter-pill").click();
-    checkDateRangeFilter();
+      cy.log("filter pills");
+      cy.findByTestId("filters-visibility-control").click();
+      cy.findByTestId("filter-pill").click();
+      checkDateRangeFilter();
 
-    cy.log("filter modal");
-    cy.button(/Filter/).click();
-    H.modal().findByText("May 1 – Jun 1, 2024").click();
-    checkDateRangeFilter();
-    H.modal().button("Close").click();
+      cy.log("filter modal");
+      cy.button(/Filter/).click();
+      H.modal().findByText("May 1 – Jun 1, 2024").click();
+      checkDateRangeFilter();
+      H.modal().button("Close").click();
 
-    cy.log("filter drill");
-    cy.findByLabelText("Switch to data").click();
-    H.tableHeaderClick("Created At: Year");
-    H.popover().findByText("Filter by this column").click();
-    H.popover().findByText("Specific dates…").click();
-    H.popover().findAllByRole("textbox").first().clear().type("2024/05/01");
-    // eslint-disable-next-line no-unsafe-element-filtering
-    H.popover().findAllByRole("textbox").last().clear().type("2024/06/01");
-    previousButton().click();
-    checkDateRangeFilter();
+      cy.log("filter drill");
+      cy.findByLabelText("Switch to data").click();
+      H.tableHeaderClick("Created At: Year");
+      H.popover().findByText("Filter by this column").click();
+      H.popover().findByText("Specific dates…").click();
+      H.popover().findAllByRole("textbox").first().clear().type("2024/05/01");
+      // eslint-disable-next-line no-unsafe-element-filtering
+      H.popover().findAllByRole("textbox").last().clear().type("2024/06/01");
+      previousButton().click();
+      checkDateRangeFilter();
 
-    cy.log("notebook editor");
-    H.openNotebook();
-    H.getNotebookStep("filter")
-      .findAllByTestId("notebook-cell-item")
-      .first()
-      .click();
-    checkDateRangeFilter();
-  });
+      cy.log("notebook editor");
+      H.openNotebook();
+      H.getNotebookStep("filter")
+        .findAllByTestId("notebook-cell-item")
+        .first()
+        .click();
+      checkDateRangeFilter();
+    },
+  );
 
   it("date picker is scrollable when overflows (metabase#39487)", () => {
     cy.viewport(1280, 800);

enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj

@@ -1,6 +1,8 @@
 (ns metabase-enterprise.advanced-config.file.databases
   (:require
+   [clojure.set :as set]
    [clojure.spec.alpha :as s]
+   [medley.core :as m]
    [metabase-enterprise.advanced-config.file.interface :as advanced-config.file.i]
    [metabase.driver.util :as driver.u]
    [metabase.models.setting :refer [defsetting]]
@@ -26,15 +28,42 @@
 (s/def :metabase-enterprise.advanced-config.file.databases.config-file-spec/details
   map?)
 
+(defn- valid-regex-patterns? [patterns]
+  (every? (fn [pattern]
+            (try
+              (boolean (re-pattern pattern))
+              (catch Exception e (log/error e) false)))
+          patterns))
+
+(s/def :metabase-enterprise.advanced-config.file.databases.config-file-spec/settings
+  (s/and
+   map?
+   (fn cruft-patterns-are-valid? [settings]
+     (->> [(:auto-cruft-tables settings
+                               ;; we access auto.cruft.* with _'s here. Because we may expect to see underscores from
+                               ;; the yaml file, this is validated first then normalized into kebab-case after
+                               ;; validation in [[normalize-settings]].
+                               (:auto_cruft_tables settings))
+           (:auto-cruft-columns settings
+                                (:auto_cruft_columns settings))]
+          (remove nil?)
+          (every? valid-regex-patterns?)))))
+
 (s/def ::config-file-spec
   (s/keys :req-un [:metabase-enterprise.advanced-config.file.databases.config-file-spec/engine
                    :metabase-enterprise.advanced-config.file.databases.config-file-spec/name
-                   :metabase-enterprise.advanced-config.file.databases.config-file-spec/details]))
+                   :metabase-enterprise.advanced-config.file.databases.config-file-spec/details]
+          :opt-un [:metabase-enterprise.advanced-config.file.databases.config-file-spec/settings]))
 
 (defmethod advanced-config.file.i/section-spec :databases
   [_section]
   (s/spec (s/* ::config-file-spec)))
 
+(defn- normalize-settings [db]
+  (m/update-existing db :settings set/rename-keys
+                     {:auto_cruft_tables :auto-cruft-tables
+                      :auto_cruft_columns :auto-cruft-columns}))
+
 (defn- init-from-config-file!
   [database]
   (if (contains? database :delete)
@@ -55,10 +84,10 @@
       (if-let [existing-database-id (t2/select-one-pk :model/Database :engine (:engine database), :name (:name database))]
         (do
           (log/info (u/format-color :blue "Updating Database %s %s" (:engine database) (pr-str (:name database))))
-          (t2/update! :model/Database existing-database-id database))
+          (t2/update! :model/Database existing-database-id (normalize-settings database)))
         (do
           (log/info (u/format-color :green "Creating new %s Database %s" (:engine database) (pr-str (:name database))))
-          (let [db (first (t2/insert-returning-instances! :model/Database database))]
+          (let [db (first (t2/insert-returning-instances! :model/Database (normalize-settings database)))]
             (if (config-from-file-sync-databases)
               (let [sync-database! (requiring-resolve 'metabase.sync.core/sync-database!)]
                 (quick-task/submit-task! (fn [] (sync-database! db))))

enterprise/backend/src/metabase_enterprise/api/routes.clj

@@ -12,6 +12,7 @@
    [metabase-enterprise.billing.api.routes]
    [metabase-enterprise.content-verification.api.routes]
    [metabase-enterprise.data-editing.api]
+   [metabase-enterprise.gsheets.api :as gsheets.api]
    [metabase-enterprise.llm.api]
    [metabase-enterprise.query-reference-validation.api]
    [metabase-enterprise.sandbox.api.routes]
@@ -28,8 +29,10 @@
 
 (def ^:private required-feature->message
   {:advanced-permissions       (deferred-tru "Advanced Permissions")
+   :attached-dwh               (deferred-tru "Attached DWH")
    :audit-app                  (deferred-tru "Audit app")
    :collection-cleanup         (deferred-tru "Collection Cleanup")
+   :etl-connections            (deferred-tru "ETL Connections")
    :table-data-editing         (deferred-tru "Editing Table Data")
    :llm-autodescription        (deferred-tru "LLM Auto-description")
    :query-reference-validation (deferred-tru "Query Reference Validation")
@@ -61,6 +64,9 @@
    "/autodescribe"               (premium-handler 'metabase-enterprise.llm.api :llm-autodescription)
    "/billing"                    metabase-enterprise.billing.api.routes/routes
    "/data-editing"               (premium-handler metabase-enterprise.data-editing.api/routes :table-data-editing)
+   "/gsheets"                    (-> gsheets.api/routes ;; gsheets requires both features.
+                                     (premium-handler :attached-dwh)
+                                     (premium-handler :etl-connections))
    "/logs"                       (premium-handler 'metabase-enterprise.advanced-config.api.logs :audit-app)
    "/query-reference-validation" (premium-handler metabase-enterprise.query-reference-validation.api/routes :query-reference-validation)
    "/scim"                       (premium-handler metabase-enterprise.scim.routes/routes :scim)

enterprise/backend/src/metabase_enterprise/gsheets/api.clj

@@ -0,0 +1,337 @@
+(ns metabase-enterprise.gsheets.api
+  (:require
+   [java-time.api :as t]
+   [medley.core :as m]
+   [metabase-enterprise.gsheets.constants :as gsheets.constants]
+   [metabase-enterprise.gsheets.settings
+    :as gsheets.settings
+    :refer [gsheets gsheets!]]
+   [metabase-enterprise.harbormaster.client :as hm.client]
+   [metabase.analytics.core :as analytics]
+   [metabase.analytics.snowplow :as snowplow]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.util :as u]
+   [metabase.util.date-2 :as u.date]
+   [metabase.util.i18n :refer [tru]]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2]))
+
+;; # Google Sheets Integration
+;;
+;; ## Overview
+;; Google Sheets Integration allows users to connect their Google Drive folders to Metabase. We sync any Google Sheets
+;; inside of those folders, and make them available to the user in their attached data warehouse. This integration is
+;; implemented using Harbormaster, which is a service that manages instances of Metabase and also gives us a connection
+;; to Google Sheets data. When a user connects a Google Drive folder to Metabase, Harbormaster creates a connection to
+;; the folder and starts syncing the data to Metabase. This data is then available to the user in Metabase as tables.
+;;
+;; ## Steps to connect a new google drive folder
+;;
+;; - An admin clicks Add Data > Connect Google Sheets
+;; - FE sends :get "api/ee/gsheets/service-account" to get service-account email
+;;   - If the FE cannot find the service-account email, it does not show the "connect to google sheets" (ctgs) button.
+;; - user clicks ctgs button -> FE shows instructions for how to connect a folder:
+;;   - user shares their Google Drive folder w/ service-account email
+;;   - user copies their Google Drive share link into MB form and hits submit
+;; - FE sends :post "api/ee/gsheets/folder" w/ folder url string as `request.body.url`
+;;   - BE forwards request to :post "/api/v2/mb/connections" w/ body:
+;;     `{:type "gdrive" :secret {:resources ["the-url"]}}`
+;;     - on unexceptional status:
+;;       - BE sets `gsheets.status` to `"loading"`
+;;       - BE returns the gsheets shape: `{:status "loading" :folder_url "the-url" :folder-upload-time <epoch-time> :gdrive/conn-id <uuid>}}`
+;;     - on exceptional status, BE returns a message like: "Unable to setup drive folder sync. Please check that the
+;;       folder is shared with the proper service account email and sharing permissions."
+;;
+;; ## Polling
+;; - FE polls :get "api/ee/gsheets/folder" until `body.status` == `complete`
+;; - BE forwards requests to :get "/api/v2/mb/connection/<gdrive-conn-id>", filtering for the google drive connection
+;;   - If the connection doesn't exist: `{:error "google drive connection not found."}`
+;;   - If the connection exists and is NOT active, return loading state in the response
+;;   - If the connection exists and is active AND the latest sync of the attached datawarehouse is AFTER the sync time
+;;         on Harbormaster's gdrive connection, return `body.status` == "complete".
+;; - FE sees status == "complete"
+;;
+;; ### What does "complete" mean?
+;; When the status is "complete", the user can access their google sheets info inside Metabase.
+;; This means that both:
+;; 1. the gdrive connection has been marked active
+;; 2. the attached datawarehouse has completed syncing inside metabase
+;; To check this, we compare the last sync time of the attached datawarehouse with the last sync time of the gdrive.
+;;
+;; ### What if the sync takes too long or never ends?
+;; Upon the successful creation of a gdrive connection, we start a timer that will error out if the sync does not
+;; complete within [[*folder-setup-timeout-seconds*]] seconds. If the sync does not complete within this window, we
+;; reset the gsheets status to `{:status "not-connected"}` and return an error for the FE to show.
+;;
+;; ## Steps to disconnect a google drive folder
+;; - FE sends request to :delete "/folder"
+;;   - MB loops over all gdrive connection ids and deletes them
+;;
+;; ## Why do we need to sync the attached datawarehouse before considering it ready?
+;; We need to sync the attached datawarehouse to make sure that the data from the Google Drive folder is available to
+;; the user in Metabase. Once the gdrive connection's status is set to 'active' by HM, they will call MB's
+;; `api/notify/db/attached_datawarehouse` endpoint to trigger a sync. The data from the Google Drive folder is already
+;; availiable in the attached datawarehouse, and when MB finishes the sync (and puts an item into :model/TaskHistory
+;; saying so), the user can start using their Google Sheets data in Metabase.
+;;
+;; ## Why do we check for multiple gdrive connections in the delete endpoint? We check for multiple gdrive connections
+;; in the delete endpoint to make sure that we are deleting _all_ gdrive connections. As of Milestone 2 there is
+;; supposed to be only one gdrive connection, but we observed multiple being created (distributed systems amirite?) and
+;; want the delete endpoint to basically reset us to a blank slate. If there are multiple gdrive connections, we will
+;; delete all of them. If there are errors during the gdrive conn deletion we ignore them.
+
+(set! *warn-on-reflection* true)
+
+(defn error-response-in-body
+  "We've decided to return errors as a map with an `:error` key. This function is a helper to make that map.
+
+  This formats and throws an ex-info that will put the message into the body of the response."
+  ([message] (error-response-in-body message {}))
+  ([message data]
+   (throw (ex-info message (merge data {;; the `:errors true` bit informs the exception middleware to return the message
+                                        ;; in the body, even if we want to pass a stauts code. Without `:errors true`,
+                                        ;; sending a status code will elide the message from the response.
+                                        :errors true
+                                        :message message})))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; MB <-> HM APIs
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(mu/defn- hm-service-account-email :- :string
+  "Checks to see if Google service-account is setup in harbormaster, and returns the email."
+  []
+  (let [[status {:keys [body] :as response}] (hm.client/make-request :get "/api/v2/mb/connections-google/service-account")
+        email (:email body)]
+    (cond
+      email
+      email
+
+      (= :error status)
+      (error-response-in-body
+       (tru "Harbormaster returned an error.")
+       {:hm/response response :status-code 502})
+
+      :else
+      (error-response-in-body
+       (tru "Google service-account is not setup in harbormaster.")
+       {:status-code 502}))))
+
+(mr/def :gdrive/connection
+  [:map {:description "A Harbormaster Gdrive Connection"}
+   [:id :string]
+   [:type [:= "gdrive"]]
+   [:status [:enum "initializing" "syncing" "active" "error"]]
+   [:last-sync-at [:maybe :time/zoned-date-time]]
+   [:last-sync-started-at [:maybe :time/zoned-date-time]]
+   [:created-at :time/zoned-date-time]
+   [:updated-at :time/zoned-date-time]])
+
+(defn- is-gdrive?
+  "Is this connection a gdrive connection?"
+  [{:keys [type] :as _conn}] (= "gdrive" type))
+
+(defn- normalize-gdrive-conn
+  "Normalize the gdrive connection shape from harbormaster, mostly parsing times."
+  [gdc]
+  (-> gdc
+      (dissoc :hosted-instance-resource)
+      (m/update-existing :last-sync-at u.date/parse)
+      (m/update-existing :last-sync-started-at u.date/parse)
+      (m/update-existing :created-at u.date/parse)
+      (m/update-existing :updated-at u.date/parse)))
+
+(mu/defn- hm-get-gdrive-conns :- [:sequential :gdrive/connection]
+  "Get the harbormaster gdrive type connection.
+
+  If the response fails:    return an empty list.
+  If the response succeeds: return a list of all gdrive-type connections."
+  []
+  (let [[status response] (hm.client/make-request :get "/api/v2/mb/connections")]
+    (if (= status :ok)
+      (mapv normalize-gdrive-conn
+            (filter is-gdrive? (or (some-> response :body) [])))
+      [])))
+
+(mu/defn- hm-delete-conn! :- :hm-client/http-reply
+  "Delete (presumably a gdrive) connection on HM."
+  [conn-id]
+  (hm.client/make-request :delete (str "/api/v2/mb/connections/" conn-id)))
+
+(defn- reset-gsheets-status!
+  "Reset the gsheets status to not-connected and delete all known gdrive connections."
+  []
+  (u/prog1 gsheets.constants/not-connected
+    (gsheets! <>)
+    (doseq [{:keys [id]} (hm-get-gdrive-conns)]
+      (let [[delete-status _] (hm-delete-conn! id)]
+        (when-not (= delete-status :ok)
+          (log/debugf "Unable to delete gdrive connection %s." id))))))
+
+(mu/defn hm-get-gdrive-conn :- :hm-client/http-reply
+  "Get a specific gdrive connection by id."
+  [id]
+  (when-not id
+    (throw (ex-info "Cannot fetch Google Drive connection: ID is nil" {})))
+  (hm.client/make-request :get (str "/api/v2/mb/connections/" id)))
+
+(mu/defn- hm-create-gdrive-conn! :- :hm-client/http-reply
+  "Creating a gdrive connection on HM starts the sync w/ drive folder."
+  [drive-folder-url]
+  (hm.client/make-request :post "/api/v2/mb/connections" {:type "gdrive" :secret {:resources [drive-folder-url]}}))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; FE <-> MB APIs
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(api.macros/defendpoint :get "/service-account" :- [:map [:email [:maybe :string]]]
+  "Checks to see if service-account is setup or not, delegates to HM only if we haven't set it from a metabase cluster
+  before."
+  []
+  (when-not (gsheets.settings/show-google-sheets-integration)
+    (reset-gsheets-status!)
+    (error-response-in-body (tru "Google Sheets integration is not enabled.") {:status-code 402}))
+  {:email (hm-service-account-email)})
+
+(defn- seconds-from-epoch-now
+  "This is used to track how long a folder has been syncing. We set gsheets.folder-upload-time to this value when we create a
+  new gdrive connection, and on every :get /folder request we check if the current time is greater than this value plus
+  [[*folder-setup-timeout-seconds*]]. If it is, that's a timeout."
+  [] (.getEpochSecond (t/instant)))
+
+(api.macros/defendpoint :post "/folder" :- :gsheets/gsheets
+  "Hook up a new google drive folder that will be watched and have its content ETL'd into Metabase."
+  [{} {} {:keys [url]} :- [:map [:url ms/NonBlankString]]]
+  (let [[status response] (hm-create-gdrive-conn! url)]
+    (if (= status :ok)
+      (u/prog1 {:status "loading"
+                :folder_url url
+                :folder-upload-time (seconds-from-epoch-now)
+                :gdrive/conn-id (-> response :body :id)}
+        (analytics/inc! :metabase-gsheets/connection-creation-began)
+        (gsheets! <>))
+      (do
+        (reset-gsheets-status!)
+        (error-response-in-body
+         (tru "Unable to setup drive folder sync.\nPlease check that the folder is shared with the proper service account email and sharing permissions."))))))
+
+(defn- sync-complete? [{:keys [status last-dwh-sync last-gdrive-conn-sync]}]
+  (and (= status "active") ;; HM says the connection is active
+       last-dwh-sync ;; make sure it's not nil
+       last-gdrive-conn-sync ;; make sure it's not nil
+       ;; We finished a sync of the dwh from metabase After the HM conn was synced:
+       (t/after? (t/instant last-dwh-sync) (t/instant last-gdrive-conn-sync))))
+
+(defn- get-last-mb-dwh-sync-time []
+  (t2/select-one-fn :ended_at :model/TaskHistory
+                    :db_id (t2/select-one-fn :id [:model/Database :id] :is_attached_dwh true)
+                    :task "sync"
+                    :status :success
+                    {:order-by [[:ended_at :desc]]}))
+
+(def ^:dynamic ^:private *folder-setup-timeout-seconds*
+  "We want to avoid polling forever, even if harbormaster never finishes the sync, so if the sync does not happen during
+  this window, we'll error out, reset the gsheets status, and suggest trying again."
+  (* 10 60))
+
+(defn- handle-get-folder [attached-dwh]
+  (let [conn-id (or (:gdrive/conn-id (gsheets))
+                    (do (log/warn "CACHE MISS ON GSHEETS")
+                        (some-> (t2/select-one :model/Setting :key "gsheets")
+                                :value
+                                json/decode+kw
+                                :gdrive/conn-id)))
+        [sstatus {conn :body}] (try (hm-get-gdrive-conn conn-id)
+                                    ;; missing id:
+                                    (catch Exception _
+                                      (reset-gsheets-status!)
+                                      (error-response-in-body
+                                       (tru "Unable to find google drive connection, please try again.")
+                                       {:conn-id conn-id})))]
+    (if (= :ok sstatus)
+      (let [{:keys [status] last-gdrive-conn-sync :last-sync-at
+             :as   _gdrive-conn} (normalize-gdrive-conn conn)
+            last-dwh-sync       (get-last-mb-dwh-sync-time)]
+        (-> (cond
+              (sync-complete? {:status status :last-dwh-sync last-dwh-sync :last-gdrive-conn-sync last-gdrive-conn-sync})
+              (u/prog1 (assoc (gsheets) :status "complete")
+                (gsheets! <>)
+                (analytics/inc! :metabase-gsheets/connection-creation-ok)
+                (snowplow/track-event! :snowplow/simple_event
+                                       {:event "sheets_connected" :event_detail "success"}))
+
+              ;; Timeout check
+              (when-let [upload-time (:folder-upload-time (gsheets))]
+                (> (seconds-from-epoch-now) (+ upload-time *folder-setup-timeout-seconds*)))
+              (do (reset-gsheets-status!)
+                  (analytics/inc! :metabase-gsheets/connection-creation-error {:reason "timeout"})
+                  (error-response-in-body (tru "Timeout syncing google drive folder, please try again.")
+                                          {:status-code 408}))
+
+              ;; Syncing failed
+              (= "error" status)
+              (do
+                (reset-gsheets-status!)
+                (analytics/inc! :metabase-gsheets/connection-creation-error {:reason "status_error"})
+                (error-response-in-body (tru "Problem syncing google drive folder, please try again..")))
+
+              ;; Continue waiting
+              :else (gsheets))
+            (assoc :db_id (:id attached-dwh)
+                   ;; here is some debugging info that proved very helpful:
+                   #_#_:hm/conn gdrive-conn
+                   #_#_:mb/sync-info {:status                status
+                                      :last-dwh-sync         last-dwh-sync
+                                      :last-gdrive-conn-sync last-gdrive-conn-sync})))
+      (do
+        (reset-gsheets-status!)
+        (error-response-in-body
+         (tru "Unable to find google drive connection.")
+         {:status-code 404})))))
+
+(api.macros/defendpoint :get "/folder" :- :gsheets/gsheets
+  "Check the status of a newly created gsheets folder creation. This endpoint gets polled by FE to determine when to
+  stop showing the setup widget.
+
+  Returns the gsheets shape, with the attached datawarehouse db id at `:db_id`."
+  [] :- ::gsheets
+  (let [attached-dwh (t2/select-one :model/Database :is_attached_dwh true)]
+    (when-not (some? attached-dwh)
+      (snowplow/track-event! :snowplow/simple_event {:event "sheets_connected" :event_detail "fail - no dwh"})
+      (reset-gsheets-status!)
+      (error-response-in-body (tru "No attached dwh found.")))
+    (handle-get-folder attached-dwh)))
+
+(api.macros/defendpoint :delete "/folder"
+  "Disconnect the google service account. There is only one (or zero) at the time of writing."
+  []
+  (snowplow/track-event! :snowplow/simple_event {:event "sheets_disconnected"})
+  (analytics/inc! :metabase-gsheets/connection-deleted)
+  (reset-gsheets-status!))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/gsheets` routes."
+  (api/+check-superuser
+   (api.macros/ns-handler *ns*)))
+
+(comment
+
+  ;; need an "attached dwh" locally?
+  (t2/update! :model/Database 1
+              {:is_attached_dwh true
+               :settings
+               (str "{\"auto-cruft-tables\":[\".*_dlt_loads$\",\".*_dlt_pipeline_state$\",\".*_dlt_sentinel_table$\",\".*_dlt_spreadsheet_info$\",\".*_dlt_version$\"],"
+                    "\"auto-cruft-columns\":[\"^_dlt_id$\",\"^_dlt_load_id$\"]}")})
+
+  (do
+    ;; This is what the notify endpoint calls to do a sync on the attached dwh:
+    #_{:clj-kondo/ignore [:metabase/modules]}
+    (require '[metabase.sync.sync-metadata :as sync-metadata])
+
+    (sync-metadata/sync-db-metadata!
+     (t2/select-one :model/Database :is_attached_dwh true))))

enterprise/backend/src/metabase_enterprise/gsheets/constants.clj

@@ -0,0 +1,5 @@
+(ns metabase-enterprise.gsheets.constants)
+
+(def not-connected
+  "The value of the gsheets setting when the Google Sheets Folder is not setup."
+  {:status "not-connected"})

enterprise/backend/src/metabase_enterprise/gsheets/settings.clj

@@ -0,0 +1,75 @@
+(ns metabase-enterprise.gsheets.settings
+  (:require
+   [metabase-enterprise.gsheets.constants :as gsheets.constants]
+   [metabase.models.setting :as setting :refer [defsetting]]
+   [metabase.premium-features.core :as premium-features]
+   [metabase.util :as u]
+   [metabase.util.i18n :refer [deferred-tru]]
+   [metabase.util.json :as json]
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2]))
+
+(mr/def :gsheets/gsheets
+  [:multi {:dispatch :status}
+   ["not-connected" [:map]]
+
+   ["loading"
+    [:map
+     [:folder_url ms/NonBlankString]
+     ;; time in seconds from epoch:
+     [:folder-upload-time pos-int?]
+     [:gdrive/conn-id ms/UUIDString]]]
+
+   ["complete"
+    [:map
+     [:folder_url ms/NonBlankString]
+     ;; time in seconds from epoch:
+     [:folder-upload-time pos-int?]
+     [:gdrive/conn-id ms/UUIDString]]]])
+
+(defsetting show-google-sheets-integration
+  "Whether or not to show the user a button that sets up Google Sheets integration."
+  :visibility :public
+  :type :boolean
+  :export? false
+  :doc "When enabled, we show users a button to authenticate with Google to import data from Google Sheets."
+  :setter :none
+  :getter (fn []
+            (and
+             (premium-features/is-hosted?)
+             (premium-features/has-feature? :attached-dwh)
+             (premium-features/has-feature? :etl-connections)
+             ;; Need to know the store-api-url to make requests to HM
+             (some? (setting/get :store-api-url))
+             ;; Need [[api-key]] to make requests to HM
+             (some? (setting/get :api-key)))))
+
+(defsetting gsheets
+  #_"
+  This value has 3 states:
+
+  1) The Google Sheets Folder is not setup.
+  {:status \"not-connected\"}
+
+  2) We have uploaded a Folder URL to HM, but have not synced it in MB yet.
+  {:status \"loading\"
+   :folder_url \"https://drive.google.com/drive/abc\"
+   :gdrive/conn-id <uuid>
+   ;; in seconds from epoch:
+   :folder-upload-time 1738266997}
+
+  3)  Google Sheets Integration is enabled, and users can view their google sheets tables.
+  {:status \"complete\"
+   :folder_url \"https://drive.google.com/drive/abc\"}"
+  (deferred-tru "Information about Google Sheets Integration")
+  :encryption :when-encryption-key-set
+  :export? true
+  :visibility :public
+  :type :json
+  :getter (fn [] (or
+                  ;; This NEEDS to be up to date between instances on a cluster, so:
+                  ;; we are going around the settings cache:
+                  (some-> (t2/select-one :model/Setting :key "gsheets") :value json/decode+kw)
+                  (u/prog1 gsheets.constants/not-connected
+                    (setting/set-value-of-type! :json :gsheets <>)))))

enterprise/backend/src/metabase_enterprise/harbormaster/client.clj

@@ -0,0 +1,108 @@
+(ns metabase-enterprise.harbormaster.client
+  "API client for interfacing with Harbormaster on store-api-url."
+  (:require
+   [clj-http.client :as http]
+   [clojure.string :as str]
+   [medley.core :as m]
+   [metabase.api.auth :as api.auth]
+   [metabase.cloud-migration.core :as cloud-migration]
+   [metabase.util.i18n :refer [tru]]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(defn- +slash-prefix [s]
+  (cond->> s
+    (not (str/starts-with? s "/")) (str "/")))
+
+(mu/defn- ->requestor [method]
+  (case method
+    :get     http/get
+    :head    http/head
+    :post    http/post
+    :put     http/put
+    :delete  http/delete
+    :options http/options
+    :copy    http/copy
+    :move    http/move
+    :patch   http/patch))
+
+(mu/defn- get-safe-status
+  [response]
+  (when (number? (:status response))
+    (http/success? response)))
+
+(defn- ->config
+  "Returns the config needed to call [[make-request]].
+
+  `->config` either gets the store-api-url and api-key from settings or throws an exception when either are unset or
+  blank."
+  []
+  (let [store-api-url (cloud-migration/store-api-url)
+        _ (when (str/blank? store-api-url)
+            (log/error "Missing store-api-url. Cannot create hm client config.")
+            (throw (ex-info (tru "Missing store-api-url.") {:store-api-url store-api-url})))
+        api-key (api.auth/api-key)
+        _ (when (str/blank? api-key)
+            (log/error "Missing api-key. Cannot create hm client config.")
+            (throw (ex-info (tru "Missing api-key.") {:api-key api-key})))]
+    {:store-api-url store-api-url
+     :api-key api-key}))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PUBLIC API:
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(mr/def :hm-client/http-reply [:tuple [:enum :ok :error] :map])
+
+(defn- send-request [request-method-fn store-api-url url request]
+  (try
+    (request-method-fn
+     (str store-api-url (+slash-prefix url))
+     request)
+    (catch Exception e
+      (log/errorf e "Error making request to %s" url)
+      {:ex-data (ex-data e)
+       :request request
+       :url     url})))
+
+(defn- decode-respones [unparsed-response url request]
+  (if (some? (:ex-data unparsed-response)) ;; skip decoding failed responses
+    unparsed-response
+    (try
+      (m/update-existing unparsed-response :body json/decode+kw)
+      (catch Exception e
+        (log/errorf e "Error decoding response from %s, is it json?" url)
+        {:ex-data           (ex-data e)
+         :unparsed-response unparsed-response
+         :request           request
+         :url               url}))))
+
+(defn- calculate-success [response url request]
+  (try
+    (get-safe-status response)
+    (catch Exception e
+      (log/errorf e "Error decoding response from %s, is it json?" url)
+      {:response response
+       :request  request
+       :url      url
+       :ex-data  (ex-data e)})))
+
+(mu/defn make-request :- :hm-client/http-reply
+  "Makes a request to the store-api-url with the given method, path, and body.
+
+  Returns a tuple of [:ok response] if the request was successful, or [:error response] if it failed."
+  [method :- [:enum :get :head :post :put :delete :options :copy :move :patch]
+   url :- :string
+   & [body]]
+  (let [{:keys [store-api-url
+                api-key]} (->config)
+        request           (cond-> {:headers {"Authorization" (str "Bearer " api-key)
+                                             "Content-Type" "application/json"}}
+                            body (assoc :body (json/encode body)))
+        request-method-fn (->requestor method)
+        unparsed-response (send-request request-method-fn store-api-url url request)
+        response          (decode-respones unparsed-response url request)
+        success?          (calculate-success response url request)]
+    [(if success? :ok :error) response]))

enterprise/backend/src/metabase_enterprise/upload_management/api.clj

@@ -4,16 +4,27 @@
    [metabase.api.macros :as api.macros]
    [metabase.api.routes.common :refer [+auth]]
    [metabase.models.interface :as mi]
+   [metabase.premium-features.core :as premium-features]
    [metabase.upload :as upload]
    [metabase.util.i18n :refer [tru]]
    [metabase.util.malli.schema :as ms]
    [toucan2.core :as t2]))
 
+(defn- attached-dwh-tables
+  "Used for adding attached DWH tables to the list of tables visible to the user. In practice these are to manage
+  google sheets uploads."
+  []
+  (when (premium-features/has-feature? :attached-dwh)
+    (when-let [dw-db-id (t2/select-one-fn :id :model/Database :is_attached_dwh true)]
+      (when-let [dw-tables (t2/select :model/Table :db_id dw-db-id :active true)]
+        dw-tables))))
+
 (api.macros/defendpoint :get "/tables"
   "Get all `Tables` visible to the current user which were created by uploading a file."
   []
   (as-> (t2/select :model/Table, :active true, :is_upload true, {:order-by [[:name :asc]]}) tables
         ;; See https://github.com/metabase/metabase/issues/41023
+    (concat tables (attached-dwh-tables))
     (map #(update % :schema str) tables)
     (filterv mi/can-read? tables)))
 

enterprise/backend/test/metabase_enterprise/advanced_config/file/databases_test.clj

@@ -5,6 +5,7 @@
    [metabase-enterprise.advanced-config.file.databases :as advanced-config.file.databases]
    [metabase.db :as mdb]
    [metabase.driver.h2 :as h2]
+   [metabase.sync.sync-metadata :as sync-metadata]
    [metabase.test :as mt]
    [metabase.util :as u]
    [toucan2.core :as t2])
@@ -71,6 +72,57 @@
         (finally
           (t2/delete! :model/Database :name test-db-name))))))
 
+(defn- test-cruft-tables! [crufted-table-setting freq message]
+  (mt/with-temporary-setting-values [config-from-file-sync-databases nil]
+    (try
+      (let [sync-future (@#'advanced-config.file.databases/init-from-config-file!
+                         {:name    test-db-name
+                          :engine  "h2"
+                          :details (:details (mt/db))
+                          :settings {:auto-cruft-tables crufted-table-setting}})]
+        (is (future? sync-future))
+        ;; wait for the sync to finish or crash out after 5 seconds
+        (deref sync-future 5000 :timeout)
+        (is (= 1 (t2/count :model/Database :name test-db-name)))
+        (let [db (t2/select-one :model/Database :name test-db-name)
+              vis-types (t2/select-fn-vec :visibility_type :model/Table :db_id (u/the-id db))]
+          (is (= freq (frequencies vis-types))
+              message)))
+      (finally
+        (t2/delete! :model/Database :name test-db-name)))))
+
+(deftest sync-cruft-tables-test
+  (testing "tables marked crufty should be marked as such in the database"
+    (test-cruft-tables! []           {nil 8}          "No tables marked crufty")
+    (test-cruft-tables! ["^venues$"] {:cruft 1 nil 7} "VENUES table is marked crufty")
+    (test-cruft-tables! ["."]        {:cruft 8}       "All tables marked crufty")))
+
+(defn- test-cruft-columns! [crufted-field-setting freq message]
+  (mt/with-temporary-setting-values [config-from-file-sync-databases nil]
+    (try
+      (let [sync-future (@#'advanced-config.file.databases/init-from-config-file!
+                         {:name    test-db-name
+                          :engine  "h2"
+                          :details (:details (mt/db))
+                          :settings {:auto-cruft-columns crufted-field-setting}})]
+        (is (future? sync-future))
+         ;; wait for the sync to finish or crash out after 5 seconds
+        (deref sync-future 5000 :timeout)
+        (sync-metadata/sync-db-metadata! (t2/select-one :model/Database :name test-db-name))
+        (is (= 1 (t2/count :model/Database :name test-db-name)))
+        (let [db (t2/select-one :model/Database :name test-db-name)
+              tables (t2/select :model/Table :db_id (u/the-id db))
+              fields (t2/select :model/Field :table_id [:in (map :id tables)])]
+          (is (= freq (frequencies (map :visibility_type fields)))
+              message)))
+      (finally
+        (t2/delete! :model/Database :name test-db-name)))))
+
+(deftest sync-cruft-columns-test
+  (testing "columns (aka fields) marked crufty should be marked as such in the database"
+    (test-cruft-columns! [] {:normal 52} "No fields marked crufty")
+    (test-cruft-columns! ["id"] {:normal 38, :details-only 14}  "All id fields marked crufty")))
+
 (deftest disable-sync-test
   (testing "We should be able to disable sync for new Databases by specifying a Setting in the config file"
     ;; make sure we're actually testing something if it was already set to false locally.

enterprise/backend/test/metabase_enterprise/api/gsheets_test.clj

@@ -0,0 +1,211 @@
+(ns metabase-enterprise.api.gsheets-test
+  (:require
+   [clojure.java.io :as io]
+   [clojure.test :refer [deftest is testing]]
+   [java-time.api :as t]
+   [metabase-enterprise.gsheets.api :as gsheets.api]
+   [metabase-enterprise.harbormaster.client :as hm.client]
+   [metabase.test :as mt]
+   [toucan2.core :as t2])
+  (:import [java.time
+            LocalDate
+            LocalTime
+            ZoneId
+            ZonedDateTime]))
+
+(set! *warn-on-reflection* true)
+
+(deftest gsheets-calls-fail-when-missing-etl-connections
+  (mt/with-temporary-setting-values [api-key "some"]
+    (mt/with-premium-features #{:attached-dwh}
+      (is (= (str "ETL Connections is a paid feature not currently available to your instance. "
+                  "Please upgrade to use it. Learn more at metabase.com/upgrade/")
+             (:message (mt/user-http-request :crowberto :get 402 "ee/gsheets/service-account")))))))
+
+(deftest gsheets-calls-fail-when-not-activated
+  (mt/with-premium-features #{:etl-connections :attached-dwh}
+    (mt/with-temporary-setting-values [api-key nil]
+      (is (partial= {:message "Missing api-key."}
+                    (mt/user-http-request :crowberto :get 500 "ee/gsheets/service-account"))))))
+
+(deftest gsheets-calls-fail-when-missing-attached-dwh
+  (mt/with-temporary-setting-values [api-key "some"]
+    (mt/with-premium-features #{:etl-connections}
+      (is (= (str "Attached DWH is a paid feature not currently available to your instance. "
+                  "Please upgrade to use it. Learn more at metabase.com/upgrade/")
+             (:message (mt/user-http-request :crowberto :get 402 "ee/gsheets/service-account")))))))
+
+(deftest gsheets-calls-fail-when-non-superuser
+  (mt/with-premium-features #{:etl-connections :attached-dwh}
+    (mt/with-temporary-setting-values [api-key nil]
+      (is (= "You don't have permissions to do that."
+             (mt/user-http-request :rasta :get 403 "ee/gsheets/service-account"))))))
+
+(deftest gsheets-calls-fail-when-there-is-no-mb-api-key
+  (mt/with-premium-features #{:etl-connections :attached-dwh :hosting}
+    (mt/with-temporary-setting-values [api-key nil]
+      (is (= "Missing api-key."
+             (:message (mt/user-http-request :crowberto :get 500 "ee/gsheets/service-account")))))))
+
+(def happy-responses (read-string (slurp (io/resource "gsheets/mock_hm_responses.edn"))))
+
+(defn +syncing [responses]
+  (assoc responses
+         {:method :get, :url "/api/v2/mb/connections", :body nil}
+         [:ok
+          {:status 200,
+           :body [{:updated-at "2025-01-27T18:43:04Z",
+                   :hosted-instance-resource-id 7,
+                   :last-sync-at nil,
+                   :error-detail nil,
+                   :type "gdrive",
+                   :hosted-instance-id "f390ec19-bd44-48ae-991c-66817182a376",
+                   :last-sync-started-at "2025-01-27T18:43:04Z",
+                   :status "syncing",
+                   :id "049f3007-2146-4083-be38-f160c526aca7",
+                   :created-at "2025-01-27T18:43:02Z"}]}]))
+
+(defn mock-make-request
+  ([responses method url] (mock-make-request responses method url nil))
+  ([responses method url body]
+   (get responses {:method method :url url :body body}
+        [:error {:status 404}])))
+
+(deftest gsheets-calls-pass-when-activated-and-superuser
+  (mt/with-premium-features #{:etl-connections :attached-dwh :hosting}
+    (mt/with-temporary-setting-values [api-key "non-blank"]
+      (with-redefs [hm.client/make-request (partial mock-make-request happy-responses)]
+        (is (malli=
+             [:map [:email [:maybe :string]]]
+             (mt/user-http-request :crowberto :get 200 "ee/gsheets/service-account")))))))
+
+(defn- ->zdt
+  [^long date ^long time ^String zone]
+  (ZonedDateTime/of (LocalDate/of date 1 1) (-> LocalTime/MIDNIGHT (.plusSeconds time)) (ZoneId/of zone)))
+
+(deftest sync-complete?-test
+  (let [earlier-time (->zdt 2000 0 "UTC")
+        later-time (->zdt 2022 0 "UTC")]
+
+    (is (not (#'gsheets.api/sync-complete? {:status "initializing" :last-dwh-sync nil :last-gdrive-conn-sync nil}))
+        "status must be active for sync to be complete")
+
+    (is (not (#'gsheets.api/sync-complete? {:status "active" :last-dwh-sync nil :last-gdrive-conn-sync nil}))
+        "sync is not complete when we don't get a last-gdrive-conn-sync time")
+
+    (is (not (#'gsheets.api/sync-complete? {:status "active" :last-dwh-sync nil :last-gdrive-conn-sync earlier-time}))
+        "sync is not complete when we don't get a last-dwh-sync time")
+
+    (is (not (#'gsheets.api/sync-complete? {:status "active" :last-dwh-sync later-time :last-gdrive-conn-sync nil}))
+        "sync is not complete when we don't get a last-gdrive-conn-sync time")
+
+    (is (not (#'gsheets.api/sync-complete? {:status "active" :last-dwh-sync earlier-time :last-gdrive-conn-sync later-time}))
+        "sync is not complete when the last dwh sync is before the last gdrive conn sync")
+
+    (is (not (#'gsheets.api/sync-complete? {:status "active" :last-dwh-sync later-time :last-gdrive-conn-sync later-time}))
+        "sync is not complete when the last dwh sync == the last gdrive conn sync")
+
+    (is (#'gsheets.api/sync-complete? {:status "active" :last-dwh-sync later-time :last-gdrive-conn-sync earlier-time})
+        "sync is complete when we get active status and the last local sync time is before current time")))
+
+(deftest can-get-service-account-test
+  (let [[status response] (mock-make-request happy-responses
+                                             :get
+                                             "/api/v2/mb/connections-google/service-account")]
+    (is (= :ok status))
+    (is (malli= [:map [:email [:string {:min 1}]]] (:body response)))))
+
+(def ^:private
+  gdrive-link
+  "nb: if you change this, change it in test_resources/gsheets/mock_hm_responses.edn"
+  "<expected-gdrive-link>")
+
+(defmacro with-sample-db-as-dwh [& body]
+  "We need an attached dwh for these tests, so let's have the sample db fill in for us:"
+  `(try
+     (t2/update! :model/Database :id 1 {:is_attached_dwh true})
+     ~@body
+     (finally (t2/update! :model/Database :id 1 {:is_attached_dwh false}))))
+
+(deftest post-folder-test
+  (with-sample-db-as-dwh
+    (mt/with-premium-features #{:etl-connections :attached-dwh :hosting}
+      (with-redefs [hm.client/make-request (partial mock-make-request happy-responses)]
+        (is (partial=
+             {:status "loading", :folder_url gdrive-link}
+             (mt/user-http-request :crowberto :post 200 "ee/gsheets/folder" {:url gdrive-link})))))))
+
+(deftest post-folder-syncing-test
+  (mt/with-premium-features #{:etl-connections :attached-dwh :hosting}
+    (with-redefs [hm.client/make-request (partial mock-make-request (+syncing happy-responses))]
+      (is (partial= {:status "loading", :folder_url gdrive-link}
+                    (mt/user-http-request :crowberto :post 200 "ee/gsheets/folder" {:url gdrive-link}))))))
+
+(deftest get-folder-test
+  (with-sample-db-as-dwh
+    (mt/with-premium-features #{:etl-connections :attached-dwh :hosting}
+      ;; This puts us into loading state:
+      (with-redefs [hm.client/make-request (partial mock-make-request (+syncing happy-responses))]
+        (mt/user-http-request :crowberto :post 200 "ee/gsheets/folder" {:url gdrive-link}))
+      (with-redefs [hm.client/make-request (partial mock-make-request happy-responses)]
+        (dotimes [_ 10]
+          (with-redefs [gsheets.api/get-last-mb-dwh-sync-time (constantly nil)]
+            (testing (str "when the dwh has never been synced, we should be status=loading.\n"
+                          "calling it over and over will return the same result.")
+              (is (partial= {:status "loading", :folder_url gdrive-link :db_id 1}
+                            (mt/user-http-request :crowberto :get 200 "ee/gsheets/folder"))))
+            (mt/user-http-request :crowberto :get 200 "ee/gsheets/folder")))
+        (testing "when the local sync time is before the last gdrive connection sync time, we should be status=loading."
+          (with-redefs [gsheets.api/get-last-mb-dwh-sync-time (constantly (t/instant "2000-01-01T00:00:00Z"))]
+            (is (partial= {:status "loading", :folder_url gdrive-link :db_id 1}
+                          (mt/user-http-request :crowberto :get 200 "ee/gsheets/folder")))
+            (mt/user-http-request :crowberto :get 200 "ee/gsheets/folder")))
+        (testing "when the local sync time is after the last gdrive connection sync time, then we should be status=complete."
+          (with-redefs [gsheets.api/get-last-mb-dwh-sync-time (constantly (t/instant "2222-01-01T00:00:00Z"))]
+            (is (partial= {:status "complete" :folder_url gdrive-link :db_id 1}
+                          (mt/user-http-request :crowberto :get 200 "ee/gsheets/folder")))))))))
+
+(deftest get-folder-timeout-test
+  (with-sample-db-as-dwh
+    (mt/with-premium-features #{:etl-connections :attached-dwh :hosting}
+      (with-redefs [hm.client/make-request (partial mock-make-request (+syncing happy-responses))]
+        (let [resp (mt/user-http-request :crowberto :post 200 "ee/gsheets/folder" {:url gdrive-link})]
+          (with-redefs [gsheets.api/get-last-mb-dwh-sync-time (constantly nil)
+                        gsheets.api/seconds-from-epoch-now (constantly
+                                                            ;; set "now" to 1 second after now + folder upload time:
+                                                            (+ 1 @#'gsheets.api/*folder-setup-timeout-seconds*
+                                                               (:folder-upload-time resp)))]
+            (is (= {:errors true, :message "Timeout syncing google drive folder, please try again."}
+                   (mt/user-http-request :crowberto :get 408 "ee/gsheets/folder"))
+                "When we timeout, we should return an error.")))))))
+
+(deftest delete-folder-test
+  (with-sample-db-as-dwh
+    (mt/with-premium-features #{:etl-connections :attached-dwh :hosting}
+      (with-redefs [hm.client/make-request (partial mock-make-request happy-responses)]
+        (is (= {:status "not-connected"}
+               (mt/user-http-request :crowberto :delete 200 "ee/gsheets/folder")))))))
+
+(defn +empty-conn-listing [responses]
+  (assoc responses {:method :get, :url "/api/v2/mb/connections", :body nil}
+         [:ok {:status 200,
+               :body []}]))
+
+(deftest delete-folder-cannot-find
+  (with-sample-db-as-dwh
+    (mt/with-premium-features #{:etl-connections :attached-dwh :hosting}
+      (with-redefs [hm.client/make-request (partial mock-make-request (+empty-conn-listing happy-responses))]
+        (is (= {:status "not-connected"}
+               (mt/user-http-request :crowberto :delete 200 "ee/gsheets/folder")))))))
+
+(defn +failed-delete-response [responses]
+  (assoc responses
+         {:method :delete, :url "/api/v2/mb/connections/049f3007-2146-4083-be38-f160c526aca7", :body nil}
+         [:error {}]))
+
+(deftest delete-folder-fail
+  (with-sample-db-as-dwh
+    (mt/with-premium-features #{:etl-connections :attached-dwh :hosting}
+      (with-redefs [hm.client/make-request (partial mock-make-request (+failed-delete-response happy-responses))]
+        (= {:status "not-connected"}
+           (mt/user-http-request :crowberto :delete 200 "ee/gsheets/folder"))))))

enterprise/backend/test/metabase_enterprise/harbormaster/client_test.clj

@@ -0,0 +1,34 @@
+(ns metabase-enterprise.harbormaster.client-test
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [metabase-enterprise.harbormaster.client :as hm.client]
+   [metabase.models.setting :as setting]
+   [metabase.test.util :as mt]))
+
+(deftest ->config-good-test
+  (testing "Both needed values are present and pulled from settings"
+    (mt/with-temporary-setting-values
+      [api-key "mb_api_key_123"
+       store-api-url "http://store-api-url.com"]
+      (is (= {:store-api-url "http://store-api-url.com", :api-key "mb_api_key_123"}
+             (#'hm.client/->config))))))
+
+(defn- +value-for-setting [grv setting-kw value]
+  (fn
+    ([k] (grv k))
+    ([k pred parse-fn]
+     (if (= k setting-kw) value (grv k pred parse-fn)))))
+
+(deftest ->config-blank-api-key-test
+  (let [grv setting/get-raw-value]
+    ;; mt/with-temporary-setting-values is not used here because we want to test the behavior of the function when the
+    ;; api-key is blank or nil, and mt/with-temporary-setting-values will not allow us to set the api-key to blank
+    ;; or nil.
+    (with-redefs [setting/get-raw-value (+value-for-setting grv :api-key "")]
+      (is (thrown-with-msg? Exception
+                            #"Missing api-key."
+                            (#'hm.client/->config))))
+    (with-redefs [setting/get-raw-value (+value-for-setting grv :api-key nil)]
+      (is (thrown-with-msg? Exception
+                            #"Missing api-key."
+                            (#'hm.client/->config))))))

enterprise/frontend/src/metabase-enterprise/api/gsheets.ts

@@ -0,0 +1,46 @@
+import type { DatabaseId, Settings } from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+
+export const gsheetsApi = EnterpriseApi.injectEndpoints({
+  endpoints: builder => ({
+    getServiceAccount: builder.query<{ email: string }, void>({
+      query: () => ({
+        method: "GET",
+        url: "/api/ee/gsheets/service-account",
+      }),
+    }),
+    getGsheetsFolder: builder.query<
+      Settings["gsheets"] & { db_id: DatabaseId },
+      void
+    >({
+      query: () => ({
+        method: "GET",
+        url: "/api/ee/gsheets/folder",
+      }),
+    }),
+    saveGsheetsFolderLink: builder.mutation<
+      { success: boolean },
+      { url: string }
+    >({
+      query: body => ({
+        method: "POST",
+        url: "/api/ee/gsheets/folder",
+        body: body,
+      }),
+    }),
+    deleteGsheetsFolderLink: builder.mutation<{ success: boolean }, void>({
+      query: () => ({
+        method: "DELETE",
+        url: "/api/ee/gsheets/folder",
+      }),
+    }),
+  }),
+});
+
+export const {
+  useGetServiceAccountQuery,
+  useGetGsheetsFolderQuery,
+  useDeleteGsheetsFolderLinkMutation,
+  useSaveGsheetsFolderLinkMutation,
+} = gsheetsApi;

enterprise/frontend/src/metabase-enterprise/api/index.ts

@@ -1,5 +1,6 @@
 export * from "./audit-info";
 export * from "./billing-info";
+export * from "./gsheets";
 export * from "./query-validation";
 export * from "./scim";
 export * from "./upload-management";

enterprise/frontend/src/metabase-enterprise/google_sheets/GoogleSheetManagement.tsx

@@ -0,0 +1,403 @@
+import { type FormEvent, useState } from "react";
+import { useLocation } from "react-use";
+import { match } from "ts-pattern";
+import { jt, t } from "ttag";
+
+import { reloadSettings } from "metabase/admin/settings/settings";
+import { skipToken, useGetDatabaseQuery } from "metabase/api";
+import { useSetting } from "metabase/common/hooks";
+import { CopyButton } from "metabase/components/CopyButton";
+import { useDispatch, useSelector } from "metabase/lib/redux";
+import { getUserIsAdmin } from "metabase/selectors/user";
+import {
+  Box,
+  Button,
+  Center,
+  Flex,
+  Icon,
+  Menu,
+  Modal,
+  Stack,
+  Text,
+  TextInput,
+} from "metabase/ui";
+import {
+  useDeleteGsheetsFolderLinkMutation,
+  useGetServiceAccountQuery,
+  useSaveGsheetsFolderLinkMutation,
+} from "metabase-enterprise/api";
+
+import { trackSheetConnectionClick, trackSheetImportClick } from "./analytics";
+import disconnectIllustration from "./disconnect.svg?component";
+
+export function GsheetConnectButton() {
+  const url = useLocation();
+  const databaseId = /databases\/(\d+)/.exec(url.pathname ?? "")?.[1];
+
+  const { data: databaseInfo } = useGetDatabaseQuery(
+    databaseId ? { id: Number(databaseId) } : skipToken,
+  );
+
+  const isDwh = databaseInfo?.is_attached_dwh;
+
+  const [showModal, setShowModal] = useState(false);
+
+  const gSheetsSetting = useSetting("gsheets");
+  const gSheetsEnabled = useSetting("show-google-sheets-integration");
+
+  const userIsAdmin = useSelector(getUserIsAdmin);
+
+  const shouldGetServiceAccount = isDwh && gSheetsEnabled && userIsAdmin;
+
+  const { data: { email: serviceAccountEmail } = {} } =
+    useGetServiceAccountQuery(shouldGetServiceAccount ? undefined : skipToken);
+
+  const { status } = gSheetsSetting;
+
+  if (
+    !gSheetsEnabled ||
+    !gSheetsSetting ||
+    !userIsAdmin ||
+    !serviceAccountEmail ||
+    !isDwh
+  ) {
+    return null;
+  }
+
+  const buttonText = match(status)
+    .with("not-connected", () => t`Connect Google Sheets`)
+    .with("loading", () => t`Google Sheets connecting...`)
+    .with("complete", () => t`Disconnect`)
+    .otherwise(() => t`Google Sheets`);
+
+  return (
+    <Flex align="center" gap="sm">
+      {status === "complete" && (
+        <Flex align="center" gap="xs">
+          <Icon name="google_sheet" />
+          <Text>{t`Connected to Google Sheets`}</Text>
+        </Flex>
+      )}
+      <Text>{" · "}</Text>
+      <Button
+        p={0}
+        variant="subtle"
+        onClick={() => {
+          setShowModal(true);
+          trackSheetConnectionClick({ from: "db-page" });
+        }}
+        disabled={status === "loading"}
+        leftSection={
+          status === "complete" ? undefined : <Icon name="google_sheet" />
+        }
+      >
+        {buttonText}
+      </Button>
+      <GsheetConnectionModal
+        isModalOpen={showModal}
+        onClose={() => setShowModal(false)}
+        reconnect={false}
+      />
+    </Flex>
+  );
+}
+
+export function GsheetMenuItem({ onClick }: { onClick: () => void }) {
+  const gSheetsSetting = useSetting("gsheets");
+  const gSheetsEnabled = useSetting("show-google-sheets-integration");
+
+  const { status } = gSheetsSetting;
+  const userIsAdmin = useSelector(getUserIsAdmin);
+  const { data: { email: serviceAccountEmail } = {} } =
+    useGetServiceAccountQuery();
+
+  if (
+    !gSheetsEnabled ||
+    !gSheetsSetting ||
+    !userIsAdmin ||
+    !serviceAccountEmail
+  ) {
+    return null;
+  }
+
+  const handleClick = () => {
+    trackSheetConnectionClick({ from: "left-nav" });
+    onClick();
+  };
+
+  const buttonText = match(status)
+    .with("not-connected", () => t`Connect Google Sheets`)
+    .with("loading", () => t`Google Sheets`)
+    .with("complete", () => t`Google Sheets`)
+    .otherwise(() => t`Google Sheets`);
+
+  const helperText = match(status)
+    .with("not-connected", () => null)
+    .with("loading", () => t`Importing files...`)
+    .with("complete", () => t`Connected`)
+    .otherwise(() => null);
+
+  return (
+    <Menu.Item onClick={handleClick} disabled={status === "loading"}>
+      <Flex gap="sm" align="flex-start" justify="space-between" w="100%">
+        <Flex>
+          <Icon name="google_sheet" mt="xs" mr="sm" />
+          <div>
+            <Text
+              fw="bold"
+              c={status === "loading" ? "text-light" : "text-dark"}
+            >
+              {buttonText}
+            </Text>
+            {helperText && (
+              <Text size="sm" c="text-medium">
+                {helperText}
+              </Text>
+            )}
+          </div>
+        </Flex>
+        {status === "complete" && (
+          <Button variant="subtle" onClick={handleClick}>
+            {t`Add New`}
+          </Button>
+        )}
+      </Flex>
+    </Menu.Item>
+  );
+}
+
+export function GsheetConnectionModal({
+  isModalOpen,
+  onClose,
+  reconnect,
+}: {
+  isModalOpen: boolean;
+  onClose: () => void;
+  reconnect: boolean;
+}) {
+  const gSheetsSetting = useSetting("gsheets");
+  const userIsAdmin = useSelector(getUserIsAdmin);
+  const { data: { email: serviceAccountEmail } = {} } =
+    useGetServiceAccountQuery();
+
+  const gSheetsEnabled = useSetting("show-google-sheets-integration");
+
+  if (
+    !gSheetsEnabled ||
+    !gSheetsSetting ||
+    !userIsAdmin ||
+    !serviceAccountEmail
+  ) {
+    return null;
+  }
+
+  const { status, folder_url } = gSheetsSetting;
+
+  return (
+    isModalOpen &&
+    (status === "not-connected" ? (
+      <GoogleSheetsConnectModal
+        onClose={onClose}
+        serviceAccountEmail={serviceAccountEmail ?? "email not found"}
+        folderUrl={folder_url}
+      />
+    ) : (
+      <GoogleSheetsDisconnectModal onClose={onClose} reconnect={reconnect} />
+    ))
+  );
+}
+
+const ModalWrapper = ({
+  children,
+  onClose,
+}: {
+  children: React.ReactNode;
+  onClose: () => void;
+}) => (
+  <Modal opened onClose={onClose} size="lg">
+    <Flex px="lg" pb="lg" gap="md" direction="column">
+      {children}
+    </Flex>
+  </Modal>
+);
+
+function GoogleSheetsConnectModal({
+  onClose,
+  folderUrl,
+  serviceAccountEmail,
+}: {
+  onClose: () => void;
+  folderUrl: string | null;
+  serviceAccountEmail: string;
+}) {
+  const dispatch = useDispatch();
+  const [folderLink, setFolderLink] = useState(folderUrl ?? "");
+  const [errorMessage, setErrorMessage] = useState("");
+
+  const [saveFolderLink, { isLoading: isSavingFolderLink }] =
+    useSaveGsheetsFolderLinkMutation();
+
+  const onSave = async (event: FormEvent) => {
+    event.preventDefault();
+    setErrorMessage("");
+
+    const validationRegex = /(https|http)\:\/\/drive\.google\.com\/.+/;
+
+    if (!validationRegex.test(folderLink.trim())) {
+      setErrorMessage(t`Invalid Google Drive folder link`);
+      return;
+    }
+
+    trackSheetImportClick();
+    await saveFolderLink({
+      url: folderLink.trim(),
+    })
+      .unwrap()
+      .then(() => {
+        dispatch(reloadSettings());
+        onClose();
+      })
+      .catch(response => {
+        setErrorMessage(response?.data?.message ?? "Something went wrong");
+      });
+  };
+
+  return (
+    <ModalWrapper onClose={onClose}>
+      <Text size="lg" fw="bold">
+        {
+          // eslint-disable-next-line no-literal-metabase-strings -- admin only string
+          t`Share the Google Drive folder that contains your Google Sheets with Metabase`
+        }
+      </Text>
+      <Flex
+        bg="bg-light"
+        style={{ borderRadius: "0.5rem" }}
+        p="md"
+        direction="column"
+        gap="md"
+      >
+        <Box>
+          <Text>
+            1. {t`In Google Drive, right-click on the folder → Share`}
+          </Text>
+        </Box>
+        <Flex align="center" justify="space-between">
+          <Text>
+            2.{" "}
+            {jt`Enter: ${(<strong>{serviceAccountEmail ?? t`Error fetching service account email`}</strong>)}`}
+          </Text>
+          <CopyButton value={serviceAccountEmail}></CopyButton>
+        </Flex>
+        <Box>
+          <Text>
+            3.{" "}
+            {jt`Select ${(<strong>{t`Viewer`}</strong>)} permissions, and click on ${(<strong>{t`Send`}</strong>)}`}
+          </Text>
+        </Box>
+      </Flex>
+      <form onSubmit={onSave}>
+        <Box>
+          <Text
+            size="lg"
+            fw="bold"
+          >{t`Paste the sharing link for the folder`}</Text>
+          <TextInput
+            my="sm"
+            disabled={isSavingFolderLink}
+            value={folderLink}
+            onChange={e => setFolderLink(e.target.value)}
+            placeholder="https://drive.google.com/drive/folders/abc123-xyz456"
+          />
+          <Text
+            size="sm"
+            color="secondary"
+          >{t`In Google Drive, right-click on the folder → Share → Copy link`}</Text>
+        </Box>
+        <Flex justify="space-between" align="center" mt="sm" gap="md">
+          <Text c="error" lh="1.2rem">
+            {errorMessage}
+          </Text>
+          <Button
+            type="submit"
+            variant="filled"
+            loading={isSavingFolderLink}
+            disabled={folderLink.length < 3}
+            style={{ flexShrink: 0 }}
+          >
+            {t`Import Google Sheets`}
+          </Button>
+        </Flex>
+      </form>
+    </ModalWrapper>
+  );
+}
+
+function GoogleSheetsDisconnectModal({
+  onClose,
+  reconnect = false,
+}: {
+  onClose: () => void;
+  reconnect: boolean;
+}) {
+  const [errorMessage, setErrorMessage] = useState("");
+  const dispatch = useDispatch();
+
+  const [deleteFolderLink, { isLoading: isDeletingFolderLink }] =
+    useDeleteGsheetsFolderLinkMutation();
+
+  const onDelete = async () => {
+    setErrorMessage("");
+    await deleteFolderLink()
+      .unwrap()
+      .then(() => {
+        dispatch(reloadSettings());
+        if (!reconnect) {
+          onClose();
+        }
+      })
+      .catch(response => {
+        setErrorMessage(response?.data?.message ?? "Something went wrong");
+      });
+  };
+
+  return (
+    <ModalWrapper onClose={onClose}>
+      <Flex justify="center" align="center" direction="column" gap="md" p="xl">
+        <Center p="md">
+          <Box component={disconnectIllustration} />
+        </Center>
+        <Text size="lg" fw="bold">
+          {reconnect
+            ? t`To add a new Google Drive folder, the existing one needs to be disconnected first.`
+            : t`Disconnect from Google Drive?`}
+        </Text>
+        <Text>
+          {reconnect
+            ? // eslint-disable-next-line no-literal-metabase-strings -- admin only string
+              t`Only one folder can be synced with Metabase at a time. Your tables and Google Sheets will remain in place.`
+            : t`Your existing tables and Google Sheets will remain in place but they will no longer be updated automatically.`}
+        </Text>
+        <Stack mt="sm">
+          <Stack w="13rem" mx="auto">
+            <Button
+              fullWidth
+              variant="filled"
+              color="danger"
+              loading={isDeletingFolderLink}
+              onClick={onDelete}
+            >
+              {t`Disconnect`}
+            </Button>
+            <Button fullWidth variant="outline" onClick={onClose}>
+              {t`Keep connected`}
+            </Button>
+          </Stack>
+          <Text c="error" ta="center">
+            {errorMessage}
+          </Text>
+        </Stack>
+      </Flex>
+    </ModalWrapper>
+  );
+}

enterprise/frontend/src/metabase-enterprise/google_sheets/GsheetsSyncStatus.tsx

@@ -0,0 +1,160 @@
+import { useEffect, useState } from "react";
+import { usePrevious } from "react-use";
+import { match } from "ts-pattern";
+import { t } from "ttag";
+
+import { reloadSettings } from "metabase/admin/settings/settings";
+import { skipToken } from "metabase/api";
+import { useSetting } from "metabase/common/hooks";
+import { useDispatch, useSelector } from "metabase/lib/redux";
+import { getUserIsAdmin } from "metabase/selectors/user";
+import StatusLarge from "metabase/status/components/StatusLarge";
+import { useGetGsheetsFolderQuery } from "metabase-enterprise/api/gsheets";
+import type { DatabaseId, Settings } from "metabase-types/api";
+
+type GsheetsStatus = Settings["gsheets"]["status"];
+type ErrorPayload = { data?: { message: string } };
+
+export const GsheetsSyncStatus = () => {
+  const gsheetsSetting = useSetting("gsheets");
+  const { status: settingStatus } = gsheetsSetting;
+  const previousSettingStatus = usePrevious(settingStatus);
+  const dispatch = useDispatch();
+  const isAdmin = useSelector(getUserIsAdmin);
+  const [forceHide, setForceHide] = useState(
+    !isAdmin || settingStatus !== "loading",
+  );
+  const [syncError, setSyncError] = useState({ error: false, message: "" });
+  const [dbId, setDbId] = useState<DatabaseId | undefined>();
+
+  const shouldPoll = isAdmin && settingStatus === "loading";
+
+  const { currentData: folderSync, error: apiError } = useGetGsheetsFolderQuery(
+    shouldPoll ? undefined : skipToken,
+    { pollingInterval: 3000 },
+  );
+
+  // if our polling endpoint changes away from loading, refresh the settings and show error, if any
+  useEffect(() => {
+    if (folderSync?.status !== "loading" || apiError) {
+      if (apiError) {
+        console.error((apiError as ErrorPayload)?.data?.message);
+
+        setSyncError({
+          error: true,
+          // eslint-disable-next-line no-literal-metabase-strings -- admin UI
+          message: t`Please check that the folder is shared with the Metabase Service Account.`,
+        });
+      }
+
+      if (folderSync?.db_id) {
+        setDbId(folderSync.db_id);
+      }
+
+      dispatch(reloadSettings());
+    }
+  }, [folderSync, dispatch, settingStatus, apiError]);
+
+  useEffect(() => {
+    // if our setting changed to loading from something else, reset the force hide and clear any errors
+    if (settingStatus === "loading" && previousSettingStatus !== "loading") {
+      setForceHide(false);
+      setSyncError({
+        error: false,
+        message: "",
+      });
+    }
+
+    // if our setting changed to not-connected from loading and we don't have an error, force hide
+    if (
+      settingStatus === "not-connected" &&
+      previousSettingStatus === "loading" &&
+      !syncError.error
+    ) {
+      setForceHide(true);
+    }
+  }, [settingStatus, previousSettingStatus, syncError]);
+
+  if (forceHide || !isAdmin) {
+    return null;
+  }
+
+  const displayStatus = match({
+    folderSyncStatus: folderSync?.status,
+    folderSyncError: syncError.error,
+    settingStatus,
+  })
+    .returnType<GsheetsStatus>()
+    .with({ folderSyncError: true }, () => "error")
+    .with({ folderSyncStatus: "complete" }, () => "complete")
+    .with({ settingStatus: "complete" }, () => "complete")
+    .with({ folderSyncStatus: "loading" }, () => "loading")
+    .with({ settingStatus: "loading" }, () => "loading")
+    .otherwise(() => "loading");
+
+  return (
+    <GsheetsSyncStatusView
+      status={displayStatus}
+      db_id={dbId}
+      error={syncError.message ?? ""}
+      onClose={() => setForceHide(true)}
+    />
+  );
+};
+
+function GsheetsSyncStatusView({
+  status,
+  db_id,
+  error,
+  onClose,
+}: {
+  status: GsheetsStatus;
+  db_id?: DatabaseId;
+  error?: string;
+  onClose: () => void;
+}) {
+  const title = match(status)
+    .with("complete", () => t`Imported Google Sheets`)
+    .with("error", () => t`Error importing Google Sheets`)
+    .otherwise(() => t`Importing Google Sheets...`);
+
+  const itemTitle = match(status)
+    .with("complete", () => t`Start exploring`)
+    .otherwise(() => t`Google Sheets`);
+
+  if (error) {
+    console.error(error);
+  }
+
+  const description = match(status)
+    .with("complete", () => t`Files sync every 15 minutes`)
+    .with(
+      "error",
+      () =>
+        // eslint-disable-next-line no-literal-metabase-strings -- admin UI
+        t`Please check that the folder is shared with the Metabase Service Account.`,
+    )
+    .otherwise(() => undefined);
+
+  return (
+    <StatusLarge
+      status={{
+        title,
+        items: [
+          {
+            title: itemTitle,
+            href:
+              status === "complete" ? `/browse/databases/${db_id}` : undefined,
+            icon: "google_drive",
+            description,
+            isInProgress: status === "loading",
+            isCompleted: status === "complete",
+            isAborted: status === "error",
+          },
+        ],
+      }}
+      isActive
+      onDismiss={onClose}
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/google_sheets/GsheetsSyncStatus.unit.spec.tsx

@@ -0,0 +1,208 @@
+import userEvent from "@testing-library/user-event";
+
+import {
+  setupGsheetsGetFolderEndpoint,
+  setupPropertiesEndpoints,
+  setupSettingsEndpoints,
+} from "__support__/server-mocks";
+import { renderWithProviders, screen, waitFor } from "__support__/ui";
+import { reloadSettings } from "metabase/admin/settings/settings";
+import { useDispatch } from "metabase/lib/redux";
+import type { Settings } from "metabase-types/api";
+import { createMockSettings, createMockUser } from "metabase-types/api/mocks";
+import { createMockSettingsState } from "metabase-types/store/mocks";
+
+import { GsheetsSyncStatus } from "./GsheetsSyncStatus";
+
+type GsheetsStatus = Settings["gsheets"]["status"];
+
+function TestComponent() {
+  const dispatch = useDispatch();
+  return (
+    <>
+      <GsheetsSyncStatus />
+      <button onClick={() => dispatch(reloadSettings())}>
+        Test Settings Update
+      </button>
+    </>
+  );
+}
+
+const setup = ({
+  settingStatus,
+  updatedSettingStatus,
+  folderStatus,
+  isAdmin = true,
+  errorCode,
+}: {
+  settingStatus: GsheetsStatus;
+  updatedSettingStatus?: GsheetsStatus;
+  folderStatus: GsheetsStatus;
+  isAdmin?: boolean;
+  errorCode?: number;
+}) => {
+  const updatedSettings = createMockSettings({
+    gsheets: {
+      status: updatedSettingStatus ?? settingStatus,
+      folder_url: null,
+    },
+  });
+
+  setupPropertiesEndpoints(updatedSettings);
+  setupSettingsEndpoints([]);
+
+  errorCode
+    ? setupGsheetsGetFolderEndpoint({ errorCode })
+    : setupGsheetsGetFolderEndpoint({ status: folderStatus });
+
+  return renderWithProviders(<TestComponent />, {
+    storeInitialState: {
+      settings: createMockSettingsState({
+        gsheets: {
+          status: settingStatus,
+          folder_url: null,
+        },
+      }),
+      currentUser: createMockUser({ is_superuser: isAdmin }),
+    },
+  });
+};
+
+describe("GsheetsSyncStatus", () => {
+  it("should not render anything in not-connected state", () => {
+    setup({
+      settingStatus: "not-connected",
+      folderStatus: "not-connected",
+    });
+
+    expect(screen.queryByText(/Google/i)).not.toBeInTheDocument();
+  });
+
+  it("should appear when status changes from not-connected to loading", async () => {
+    setup({
+      settingStatus: "not-connected",
+      updatedSettingStatus: "loading",
+      folderStatus: "loading",
+    });
+
+    // initial state
+    expect(screen.queryByText(/Google/i)).not.toBeInTheDocument();
+
+    // trigger settings update
+    userEvent.click(await screen.findByText("Test Settings Update"));
+
+    // loading state
+    expect(
+      await screen.findByText("Importing Google Sheets..."),
+    ).toBeInTheDocument();
+  });
+
+  it("should not render anything for non-admins", () => {
+    setup({
+      settingStatus: "loading",
+      folderStatus: "loading",
+      isAdmin: false,
+    });
+
+    expect(screen.queryByText(/Google/i)).not.toBeInTheDocument();
+  });
+
+  it("should not render anything when initial state is complete", () => {
+    setup({
+      settingStatus: "complete",
+      folderStatus: "complete",
+    });
+
+    expect(screen.queryByText(/Google/i)).not.toBeInTheDocument();
+  });
+
+  it("should render loading state", () => {
+    setup({
+      settingStatus: "loading",
+      folderStatus: "loading",
+    });
+
+    expect(screen.getByText("Importing Google Sheets...")).toBeInTheDocument();
+  });
+
+  it("should close when the X is clicked", async () => {
+    setup({
+      settingStatus: "loading",
+      folderStatus: "loading",
+    });
+
+    expect(screen.getByText("Importing Google Sheets...")).toBeInTheDocument();
+
+    userEvent.click(await screen.findByLabelText("Dismiss"));
+    await waitFor(() =>
+      expect(screen.queryByText(/Google/i)).not.toBeInTheDocument(),
+    );
+  });
+
+  it("should render completed state after initial status is loading", async () => {
+    setup({
+      settingStatus: "loading",
+      folderStatus: "complete",
+    });
+
+    // initial loading state
+    expect(screen.getByText("Importing Google Sheets...")).toBeInTheDocument();
+
+    // complete state
+    expect(
+      await screen.findByText("Imported Google Sheets"),
+    ).toBeInTheDocument();
+
+    screen.getByText("Start exploring");
+    screen.getByText("Files sync every 15 minutes");
+  });
+
+  it("should show errors", async () => {
+    setup({
+      settingStatus: "loading",
+      folderStatus: "loading",
+      errorCode: 500,
+    });
+
+    // initial loading state
+    expect(screen.getByText("Importing Google Sheets...")).toBeInTheDocument();
+
+    // error state
+    expect(
+      await screen.findByText(/Error importing Google Sheets/),
+    ).toBeInTheDocument();
+  });
+
+  it("should clear error if the user tries to connect again", async () => {
+    setup({
+      settingStatus: "loading",
+      folderStatus: "not-connected",
+      errorCode: 500,
+      updatedSettingStatus: "not-connected",
+    });
+
+    // initial loading state
+    expect(screen.getByText("Importing Google Sheets...")).toBeInTheDocument();
+
+    // not-connected state
+    await waitFor(() =>
+      expect(screen.queryByText(/Google/i)).not.toBeInTheDocument(),
+    );
+  });
+
+  it("should disappear if state changes from loading to not-connected without an error", async () => {
+    setup({
+      settingStatus: "loading",
+      folderStatus: "not-connected",
+      updatedSettingStatus: "not-connected",
+    });
+
+    // initial loading state
+    expect(screen.getByText("Importing Google Sheets...")).toBeInTheDocument();
+
+    // not-connected state
+    await waitFor(() =>
+      expect(screen.queryByText(/Google/i)).not.toBeInTheDocument(),
+    );
+  });
+});

enterprise/frontend/src/metabase-enterprise/google_sheets/analytics.ts

@@ -0,0 +1,20 @@
+import { trackSimpleEvent } from "metabase/lib/analytics";
+import type { GsheetsConnectionClickedEvent } from "metabase-types/analytics";
+
+export function trackSheetImportClick() {
+  trackSimpleEvent({
+    event: "sheets_import_by_url_clicked",
+    triggered_from: "sheets-url-popup",
+  });
+}
+
+export function trackSheetConnectionClick({
+  from,
+}: {
+  from: GsheetsConnectionClickedEvent["triggered_from"];
+}) {
+  trackSimpleEvent({
+    event: "sheets_connection_clicked",
+    triggered_from: from,
+  });
+}

enterprise/frontend/src/metabase-enterprise/google_sheets/disconnect.svg

@@ -0,0 +1,8 @@
+<svg width="96" height="96" viewBox="0 0 96 96" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect width="96" height="96" rx="48" fill="#071722" fill-opacity="0.02"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M34 33.625C32.6883 33.625 31.625 34.6883 31.625 36V59.5284C31.625 61.3924 33.136 62.9034 35 62.9034H61.8361C63.7001 62.9034 65.2111 61.3924 65.2111 59.5284V41.1371C65.2111 39.8254 64.1478 38.7621 62.8361 38.7621H49.6421C49.4764 38.7621 49.3174 38.6963 49.2002 38.5791L44.2461 33.625H34ZM30.375 36C30.375 33.998 31.998 32.375 34 32.375H44.505C44.6708 32.375 44.8298 32.4408 44.947 32.5581L49.901 37.5121H62.8361C64.8382 37.5121 66.4611 39.1351 66.4611 41.1371V59.5284C66.4611 62.0827 64.3904 64.1534 61.8361 64.1534H35C32.4457 64.1534 30.375 62.0827 30.375 59.5284V36Z" fill="#92999E"/>
+<path d="M35.5504 46.1558C35.8961 44.8363 37.0884 43.916 38.4525 43.916H68.4087C69.7191 43.916 70.6754 45.1552 70.3434 46.4228L66.4496 61.2882C66.1039 62.6078 64.9116 63.5281 63.5475 63.5281H34.887C32.9214 63.5281 31.4869 61.6694 31.9849 59.7679L35.5504 46.1558Z" fill="white"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M34.9458 45.9975C35.3635 44.403 36.8042 43.291 38.4525 43.291H68.4086C70.1285 43.291 71.3838 44.9174 70.948 46.5812L67.0541 61.4466C66.6365 63.0411 65.1957 64.1531 63.5475 64.1531H34.887C32.5119 64.1531 30.7785 61.9071 31.3803 59.6095L34.9458 45.9975ZM38.4525 44.541C37.3726 44.541 36.4286 45.2696 36.155 46.3142L32.5895 59.9263C32.1952 61.4316 33.3309 62.9031 34.887 62.9031H63.5475C64.6274 62.9031 65.5713 62.1745 65.8449 61.1299L69.7388 46.2644C69.967 45.3929 69.3095 44.541 68.4086 44.541H38.4525Z" fill="#92999E"/>
+<circle cx="66" cy="60" r="9" fill="white" stroke="white" stroke-width="2"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M72 60C72 63.3137 69.3137 66 66 66C64.7042 66 63.5044 65.5892 62.5236 64.8908L70.8908 56.5236C71.5892 57.5044 72 58.7042 72 60ZM61.1093 63.4766L69.4766 55.1093C68.4958 54.4108 67.2959 54 66 54C62.6863 54 60 56.6863 60 60C60 61.2959 60.4108 62.4958 61.1093 63.4766ZM74 60C74 64.4183 70.4183 68 66 68C61.5817 68 58 64.4183 58 60C58 55.5817 61.5817 52 66 52C70.4183 52 74 55.5817 74 60Z" fill="var(--mb-color-error)"/>
+</svg>

enterprise/frontend/src/metabase-enterprise/google_sheets/index.ts

@@ -0,0 +1,2 @@
+export * from "./GoogleSheetManagement";
+export * from "./GsheetsSyncStatus";

enterprise/frontend/src/metabase-enterprise/upload_management/index.ts

@@ -1,8 +1,22 @@
 import { PLUGIN_UPLOAD_MANAGEMENT } from "metabase/plugins";
 import { hasPremiumFeature } from "metabase-enterprise/settings";
 
+import {
+  GsheetConnectButton,
+  GsheetConnectionModal,
+  GsheetMenuItem,
+  GsheetsSyncStatus,
+} from "../google_sheets";
+
 import { UploadManagementTable } from "./UploadManagementTable";
 
 if (hasPremiumFeature("upload_management")) {
   PLUGIN_UPLOAD_MANAGEMENT.UploadManagementTable = UploadManagementTable;
 }
+
+if (hasPremiumFeature("hosting") && hasPremiumFeature("attached_dwh")) {
+  PLUGIN_UPLOAD_MANAGEMENT.GsheetConnectionModal = GsheetConnectionModal;
+  PLUGIN_UPLOAD_MANAGEMENT.GsheetMenuItem = GsheetMenuItem;
+  PLUGIN_UPLOAD_MANAGEMENT.GsheetsSyncStatus = GsheetsSyncStatus;
+  PLUGIN_UPLOAD_MANAGEMENT.GsheetConnectButton = GsheetConnectButton;
+}

frontend/src/metabase-types/analytics/event.ts

@@ -81,6 +81,16 @@ export type ErrorDiagnosticModalSubmittedEvent = ValidateEvent<{
   event_detail: "download-diagnostics" | "submit-report";
 }>;
 
+export type GsheetsConnectionClickedEvent = ValidateEvent<{
+  event: "sheets_connection_clicked";
+  triggered_from: "db-page" | "left-nav";
+}>;
+
+export type GsheetsImportClickedEvent = ValidateEvent<{
+  event: "sheets_import_by_url_clicked";
+  triggered_from: "sheets-url-popup";
+}>;
+
 export type SimpleEvent =
   | CSVUploadClickedEvent
   | DatabaseAddClickedEvent
@@ -91,4 +101,6 @@ export type SimpleEvent =
   | OnboardingChecklistItemCTAClickedEvent
   | MoveToTrashEvent
   | ErrorDiagnosticModalOpenedEvent
-  | ErrorDiagnosticModalSubmittedEvent;
+  | ErrorDiagnosticModalSubmittedEvent
+  | GsheetsConnectionClickedEvent
+  | GsheetsImportClickedEvent;

frontend/src/metabase-types/api/mocks/settings.ts

@@ -208,6 +208,7 @@ export const createMockSettings = (
   "enable-xrays": false,
   engines: createMockEngines(),
   "example-dashboard-id": 1,
+  gsheets: { status: "not-connected", folder_url: null },
   "has-user-setup": true,
   "hide-embed-branding?": true,
   "instance-creation": dayjs().toISOString(),
@@ -252,6 +253,7 @@ export const createMockSettings = (
   "slack-bug-report-channel": null,
   "snowplow-enabled": false,
   "show-database-syncing-modal": false,
+  "show-google-sheets-integration": false,
   "show-homepage-data": false,
   "show-homepage-pin-message": false,
   "show-homepage-xrays": false,

frontend/src/metabase-types/api/settings.ts

@@ -362,6 +362,11 @@ interface PublicSettings {
   engines: Record<string, Engine>;
   "google-auth-client-id": string | null;
   "google-auth-enabled": boolean;
+  gsheets: {
+    status: "not-connected" | "loading" | "complete" | "error";
+    folder_url: string | null;
+    error?: string;
+  };
   "has-user-setup": boolean;
   "help-link": HelpLinkSetting;
   "help-link-custom-destination": string;
@@ -384,6 +389,7 @@ interface PublicSettings {
   "setup-token": string | null;
   "show-metabase-links": boolean;
   "show-metabot": boolean;
+  "show-google-sheets-integration": boolean;
   "site-locale": string;
   "site-url": string;
   "snowplow-enabled": boolean;

frontend/src/metabase/api/index.ts

@@ -25,6 +25,7 @@ export * from "./revision";
 export * from "./search";
 export * from "./segment";
 export * from "./session";
+export * from "./settings";
 export * from "./snippet";
 export * from "./subscription";
 export * from "./table";

frontend/src/metabase/api/settings.ts

@@ -0,0 +1,16 @@
+import type { SettingKey, SettingValue } from "metabase-types/api";
+
+import { Api } from "./api";
+
+export const settingApi = Api.injectEndpoints({
+  endpoints: builder => ({
+    getSetting: builder.query<SettingValue, SettingKey>({
+      query: settingName => ({
+        method: "GET",
+        url: `/api/setting/${settingName}`,
+      }),
+    }),
+  }),
+});
+
+export const { useGetSettingQuery } = settingApi;

frontend/src/metabase/browse/components/BrowseDataHeader.tsx

@@ -2,6 +2,7 @@ import { Link } from "react-router";
 import { t } from "ttag";
 
 import { color } from "metabase/lib/colors";
+import { PLUGIN_UPLOAD_MANAGEMENT } from "metabase/plugins";
 import { Flex, Group, Icon, Text, Title } from "metabase/ui";
 
 import {
@@ -36,12 +37,7 @@ export const BrowseDataHeader = () => {
 };
 
 const LearnAboutDataLink = () => (
-  <Flex
-    p=".75rem"
-    justify="flex-end"
-    align="center"
-    style={{ flexBasis: "40.0%", marginInlineStart: "auto" }}
-  >
+  <Flex p=".75rem" justify="flex-end" align="center" gap="md">
     <Link to="reference">
       <BrowseHeaderIconContainer>
         <LearnAboutDataIcon size={14} name="reference" />
@@ -50,5 +46,6 @@ const LearnAboutDataLink = () => (
         </Text>
       </BrowseHeaderIconContainer>
     </Link>
+    <PLUGIN_UPLOAD_MANAGEMENT.GsheetConnectButton />
   </Flex>
 );

frontend/src/metabase/nav/containers/MainNavbar/MainNavbarContainer/MainNavbarView.tsx

@@ -43,7 +43,7 @@ import {
 } from "../MainNavbar.styled";
 import { SidebarCollectionLink, SidebarLink } from "../SidebarItems";
 import { AddDatabase } from "../SidebarItems/AddDatabase";
-import { DwhUploadCSV } from "../SidebarItems/DwhUploadCSV/DwhUploadCSV";
+import { DwhUploadMenu } from "../SidebarItems/DwhUpload";
 import { trackOnboardingChecklistOpened } from "../analytics";
 import type { SelectedItem } from "../types";
 
@@ -172,7 +172,7 @@ export function MainNavbarView({
    *   - "upload" permissions for the attached DWH
    */
   const canUpload = canCurateRootCollection && canUploadToDatabase;
-  const showUploadCSVButton = hasAttachedDWHFeature && canUpload;
+  const showUploadMenu = hasAttachedDWHFeature && canUpload;
 
   const isAdditionalDatabaseAdded = getHasOwnDatabase(databases);
   const showAddDatabaseButton = isAdmin && !isAdditionalDatabaseAdded;
@@ -210,7 +210,7 @@ export function MainNavbarView({
                 {t`How to use Metabase`}
               </PaddedSidebarLinkDismissible>
             )}
-            {showUploadCSVButton && <DwhUploadCSV />}
+            {showUploadMenu && <DwhUploadMenu />}
           </SidebarSection>
 
           {bookmarks.length > 0 && (

frontend/src/metabase/nav/containers/MainNavbar/SidebarItems/DwhUpload/DwhUpload.tsx

@@ -1,5 +1,5 @@
 import { type ChangeEvent, useCallback, useRef, useState } from "react";
-import { c } from "ttag";
+import { c, t } from "ttag";
 
 import {
   type CollectionOrTableIdProps,
@@ -9,20 +9,26 @@ import type { OnFileUpload } from "metabase/collections/types";
 import { UploadInput } from "metabase/components/upload";
 import { useToggle } from "metabase/hooks/use-toggle";
 import { useDispatch } from "metabase/lib/redux";
+import { PLUGIN_UPLOAD_MANAGEMENT } from "metabase/plugins";
 import {
   type UploadFileProps,
   uploadFile as uploadFileAction,
 } from "metabase/redux/uploads";
-import { Box, Button, Icon } from "metabase/ui";
+import { Box, Button, Flex, Icon, Menu } from "metabase/ui";
 
 import { trackDWHUploadCSVClicked } from "./analytics";
 
-export const DwhUploadCSV = () => {
+export const DwhUploadMenu = () => {
   const [
     isModelUploadModalOpen,
     { turnOn: openModelUploadModal, turnOff: closeModelUploadModal },
   ] = useToggle(false);
 
+  const [
+    isGsheetModalOpen,
+    { turnOn: openGsheetModal, turnOff: closeGsheetModal },
+  ] = useToggle(false);
+
   const [uploadedFile, setUploadedFile] = useState<File | null>(null);
 
   const dispatch = useDispatch();
@@ -67,17 +73,36 @@ export const DwhUploadCSV = () => {
   const triggerUploadInput = () => uploadInputRef.current?.click();
 
   return (
-    <Box data-testid="dwh-upload-csv" mt="md" px="md">
-      <Button
-        color="brand"
-        fullWidth={true}
-        leftSection={<Icon name="upload" />}
-        onClick={triggerUploadInput}
-        radius="xl"
-        variant="outline"
-      >
-        {c("Text for a button that lets you upload a CSV file").t`Upload CSV`}
-      </Button>
+    <Box data-testid="dwh-upload" mt="md" px="md">
+      <Menu position="right">
+        <Menu.Target>
+          <Button
+            fullWidth
+            variant="outline"
+            styles={{ label: { width: "100%" } }}
+          >
+            <Flex justify="space-between" w="100%">
+              <span />
+              <Flex>
+                <Icon name="add_data" mr="sm" />
+                <Box>{t`Add Data`}</Box>
+              </Flex>
+              {/* need this on the far-right */}
+              <Icon name="chevrondown" ml="sm" />
+            </Flex>
+          </Button>
+        </Menu.Target>
+        <Menu.Dropdown miw="19rem">
+          <Menu.Item
+            fw="bold"
+            leftSection={<Icon name="upload" />}
+            onClick={triggerUploadInput}
+          >
+            {c("button label for uploading a CSV data file").t`Upload CSV`}
+          </Menu.Item>
+          <PLUGIN_UPLOAD_MANAGEMENT.GsheetMenuItem onClick={openGsheetModal} />
+        </Menu.Dropdown>
+      </Menu>
       <UploadInput
         id="dwh-upload-csv-input"
         ref={uploadInputRef}
@@ -89,6 +114,11 @@ export const DwhUploadCSV = () => {
         onClose={closeModelUploadModal}
         onUpload={handleFileUpload}
       />
+      <PLUGIN_UPLOAD_MANAGEMENT.GsheetConnectionModal
+        isModalOpen={isGsheetModalOpen}
+        onClose={closeGsheetModal}
+        reconnect={true}
+      />
     </Box>
   );
 };

frontend/src/metabase/nav/containers/MainNavbar/SidebarItems/DwhUpload/index.ts

@@ -0,0 +1 @@
+export * from "./DwhUpload";

frontend/src/metabase/nav/containers/MainNavbar/tests/enterprise.unit.spec.tsx

@@ -1,9 +1,11 @@
+import { setupGsheetsServiceAccountEndpoint } from "__support__/server-mocks";
 import { screen } from "__support__/ui";
 import { createMockUser } from "metabase-types/api/mocks";
 
 import { type SetupOpts, setup } from "./setup";
 
 const setupEnterprise = (opts?: SetupOpts) => {
+  setupGsheetsServiceAccountEndpoint();
   return setup({ ...opts, hasEnterprisePlugins: true });
 };
 

frontend/src/metabase/nav/containers/MainNavbar/tests/premium.unit.spec.tsx

@@ -1,9 +1,13 @@
+import userEvent from "@testing-library/user-event";
+
+import { setupGsheetsServiceAccountEndpoint } from "__support__/server-mocks";
 import { screen, within } from "__support__/ui";
 import { createMockUser } from "metabase-types/api/mocks";
 
 import { type SetupOpts, setup } from "./setup";
 
 function setupPremium(opts: SetupOpts) {
+  setupGsheetsServiceAccountEndpoint();
   return setup({
     hasEnterprisePlugins: true,
     hasDWHAttached: true,
@@ -12,12 +16,32 @@ function setupPremium(opts: SetupOpts) {
 }
 
 describe("nav > containers > MainNavbar (EE with token)", () => {
-  it("should render 'upload CSV' button to admins", async () => {
+  it("should render 'add data' button to admins", async () => {
     await setupPremium({
       user: createMockUser({ is_superuser: true }),
     });
     const nav = screen.getByTestId("main-navbar-root");
-    expect(within(nav).getByText("Upload CSV")).toBeInTheDocument();
+    expect(within(nav).getByText("Add Data")).toBeInTheDocument();
+  });
+
+  it("should render 'upload CSV' menu item to admins", async () => {
+    await setupPremium({
+      user: createMockUser({ is_superuser: true }),
+    });
+    const nav = screen.getByTestId("main-navbar-root");
+    await userEvent.click(within(nav).getByText("Add Data"));
+    expect(await screen.findByText("Upload CSV")).toBeInTheDocument();
+  });
+
+  it("should render gsheets upload menu item to admins", async () => {
+    await setupPremium({
+      user: createMockUser({ is_superuser: true }),
+    });
+    const nav = screen.getByTestId("main-navbar-root");
+    await userEvent.click(within(nav).getByText("Add Data"));
+    expect(
+      await screen.findByText("Connect Google Sheets"),
+    ).toBeInTheDocument();
   });
 
   it("should render 'upload CSV' button to regular users who have sufficient permissions", async () => {
@@ -27,7 +51,8 @@ describe("nav > containers > MainNavbar (EE with token)", () => {
       user: createMockUser({ is_superuser: false }),
     });
     const nav = screen.getByTestId("main-navbar-root");
-    expect(within(nav).getByText("Upload CSV")).toBeInTheDocument();
+    await userEvent.click(within(nav).getByText("Add Data"));
+    expect(await screen.findByText("Upload CSV")).toBeInTheDocument();
   });
 
   it("should not render 'upload CSV' button to regular users who lack root collection permissions", async () => {

frontend/src/metabase/nav/containers/MainNavbar/tests/setup.tsx

@@ -11,6 +11,7 @@ import {
   setupDatabasesEndpoints,
   setupSearchEndpoints,
 } from "__support__/server-mocks";
+import { mockSettings } from "__support__/settings";
 import { createMockEntitiesState } from "__support__/store";
 import {
   renderWithProviders,
@@ -30,7 +31,6 @@ import type { DashboardState } from "metabase-types/store";
 import {
   createMockDashboardState,
   createMockQueryBuilderState,
-  createMockSettingsState,
   createMockState,
 } from "metabase-types/store/mocks";
 
@@ -158,7 +158,7 @@ export async function setup({
     }),
     qb: createMockQueryBuilderState({ card: openQuestionCard }),
     entities: createMockEntitiesState({ dashboards: dashboardsForEntities }),
-    settings: createMockSettingsState({
+    settings: mockSettings({
       "uploads-settings": {
         db_id: hasDWHAttached || isUploadEnabled ? SAMPLE_DATABASE.id : null,
         schema_name: null,
@@ -167,7 +167,10 @@ export async function setup({
       "instance-creation": instanceCreationDate,
       "token-features": createMockTokenFeatures({
         attached_dwh: hasDWHAttached,
+        hosting: true,
+        upload_management: true,
       }),
+      "show-google-sheets-integration": true,
     }),
   });
 

frontend/src/metabase/plugins/index.ts

@@ -569,8 +569,19 @@ export const PLUGIN_AUDIT = {
   InsightsLink: PluginPlaceholder as ComponentType<InsightsLinkProps>,
 };
 
+type GsheetConnectionModalProps = {
+  isModalOpen: boolean;
+  onClose: () => void;
+  reconnect: boolean;
+};
+
 export const PLUGIN_UPLOAD_MANAGEMENT = {
   UploadManagementTable: PluginPlaceholder,
+  GsheetsSyncStatus: PluginPlaceholder,
+  GsheetConnectionModal:
+    PluginPlaceholder as ComponentType<GsheetConnectionModalProps>,
+  GsheetMenuItem: PluginPlaceholder as ComponentType<{ onClick: () => void }>,
+  GsheetConnectButton: PluginPlaceholder,
 };
 
 export const PLUGIN_IS_EE_BUILD = {

frontend/src/metabase/status/components/FileUploadStatus/FileUploadStatus.unit.spec.tsx

@@ -234,7 +234,7 @@ describe("FileUploadStatus", () => {
     });
 
     expect(
-      await screen.findByRole("link", { name: "Start exploring" }),
+      await screen.findByRole("link", { name: /Start exploring/ }),
     ).toHaveAttribute("href", "/model/3");
   });
 
@@ -268,7 +268,7 @@ describe("FileUploadStatus", () => {
     });
 
     expect(
-      await screen.findByRole("link", { name: "Start exploring" }),
+      await screen.findByRole("link", { name: /Start exploring/ }),
     ).toHaveAttribute("href", "/model/3");
   });
 
@@ -311,7 +311,7 @@ describe("FileUploadStatus", () => {
     });
 
     expect(
-      await screen.findByRole("link", { name: "Start exploring" }),
+      await screen.findByRole("link", { name: /Start exploring/ }),
     ).toHaveAttribute("href", "/model/3");
   });
 
@@ -358,7 +358,7 @@ describe("FileUploadStatus", () => {
     });
 
     expect(
-      await screen.findByRole("link", { name: "Start exploring" }),
+      await screen.findByRole("link", { name: /Start exploring/ }),
     ).toHaveAttribute("href", "/model/3");
     await screen.findByText("Data added to Fancy Table");
   });
@@ -405,7 +405,7 @@ describe("FileUploadStatus", () => {
     });
 
     expect(
-      await screen.findByRole("link", { name: "Start exploring" }),
+      await screen.findByRole("link", { name: /Start exploring/ }),
     ).toHaveAttribute("href", "/model/3");
     await screen.findByText("Data replaced in Fancy Table");
   });
@@ -492,7 +492,7 @@ describe("FileUploadStatus", () => {
       });
 
       expect(
-        await screen.findByRole("link", { name: "Start exploring" }),
+        await screen.findByRole("link", { name: /Start exploring/ }),
       ).toHaveAttribute("href", "/model/3");
     });
   });

frontend/src/metabase/status/components/FileUploadStatusLarge/FileUploadStatusLarge.tsx

@@ -56,6 +56,7 @@ const FileUploadLarge = ({
       id: upload.id,
       title: getName(upload),
       icon: "model",
+      href: upload.modelId ? `/model/${upload.modelId}` : undefined,
       description: Description({ upload }),
       isInProgress: isUploadInProgress(upload),
       isCompleted: isUploadCompleted(upload),
@@ -122,7 +123,7 @@ const getLoadingMessage = (time: number) => {
 
 const Description = ({ upload }: { upload: FileUpload }) => {
   if (upload.status === "complete" && upload.modelId) {
-    return <Link to={`/model/${upload.modelId}`}>Start exploring</Link>;
+    return t`Start exploring`;
   }
 
   if (upload.status === "error") {

frontend/src/metabase/status/components/StatusLarge/StatusLarge.tsx

@@ -1,6 +1,7 @@
 import { t } from "ttag";
 
 import { Ellipsified } from "metabase/core/components/Ellipsified";
+import Link from "metabase/core/components/Link";
 import type { IconName } from "metabase/ui";
 import { Icon } from "metabase/ui";
 
@@ -28,6 +29,7 @@ type Status = {
 
 type StatusItem = {
   id?: number;
+  href?: string;
   title: string | JSX.Element;
   icon: string;
   description?: string | JSX.Element;
@@ -68,7 +70,11 @@ const StatusLarge = ({
       </StatusHeader>
       <StatusBody>
         {status.items.map(item => (
-          <StatusCard item={item} isActive={isActive} key={item.id} />
+          <StatusCard
+            item={item}
+            isActive={isActive}
+            key={item.id ?? String(item.title)}
+          />
         ))}
       </StatusBody>
     </StatusRoot>
@@ -89,33 +95,38 @@ const StatusCard = ({
 
   const isVisible = useStatusVisibility(isActive || isInProgress);
 
+  const LinkWrapper = ({ children }: { children: JSX.Element }) =>
+    item?.href ? <Link to={item.href}>{children}</Link> : children;
+
   if (!isVisible) {
     return null;
   }
 
   return (
-    <StatusCardRoot key={id} hasBody={!!description}>
-      <StatusCardIcon>
-        <Icon name={icon as unknown as IconName} />
-      </StatusCardIcon>
-      <StatusCardBody>
-        <StatusCardTitle>
-          <Ellipsified>{title}</Ellipsified>
-        </StatusCardTitle>
-        <StatusCardDescription>{description}</StatusCardDescription>
-      </StatusCardBody>
-      {isInProgress && <StatusCardSpinner size={24} borderWidth={3} />}
-      {isCompleted && (
-        <StatusCardIconContainer>
-          <Icon name="check" size={12} />
-        </StatusCardIconContainer>
-      )}
-      {isAborted && (
-        <StatusCardIconContainer isError={true}>
-          <Icon name="warning" size={12} />
-        </StatusCardIconContainer>
-      )}
-    </StatusCardRoot>
+    <LinkWrapper key={id}>
+      <StatusCardRoot hasBody={!!description}>
+        <StatusCardIcon>
+          <Icon name={icon as unknown as IconName} />
+        </StatusCardIcon>
+        <StatusCardBody>
+          <StatusCardTitle>
+            <Ellipsified>{title}</Ellipsified>
+          </StatusCardTitle>
+          <StatusCardDescription>{description}</StatusCardDescription>
+        </StatusCardBody>
+        {isInProgress && <StatusCardSpinner size={24} borderWidth={3} />}
+        {isCompleted && (
+          <StatusCardIconContainer>
+            <Icon name="check" size={12} />
+          </StatusCardIconContainer>
+        )}
+        {isAborted && (
+          <StatusCardIconContainer isError={true}>
+            <Icon name="warning" size={12} />
+          </StatusCardIconContainer>
+        )}
+      </StatusCardRoot>
+    </LinkWrapper>
   );
 };
 

frontend/src/metabase/status/components/StatusListing/StatusListing.tsx

@@ -2,6 +2,7 @@ import { useBeforeUnload } from "react-use";
 import { t } from "ttag";
 
 import { useSelector } from "metabase/lib/redux";
+import { PLUGIN_UPLOAD_MANAGEMENT } from "metabase/plugins";
 import { hasActiveUploads } from "metabase/redux/uploads";
 import { getUserIsAdmin } from "metabase/selectors/user";
 import { useCheckActiveDownloadsBeforeUnload } from "metabase/status/hooks/use-check-active-downloads-before-unload";
@@ -29,6 +30,7 @@ const StatusListing = () => {
       {isAdmin && <DatabaseStatus />}
       <FileUploadStatus />
       <DownloadsStatus />
+      <PLUGIN_UPLOAD_MANAGEMENT.GsheetsSyncStatus />
     </StatusListingRoot>
   );
 };

frontend/src/metabase/ui/components/icons/Icon/icons/google_drive.svg

@@ -0,0 +1,15 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_12770_5520)">
+<path d="M2.05842 12.4663L2.67583 13.5327C2.80412 13.7572 2.98855 13.9336 3.20504 14.0619L5.41008 10.2452H1C1 10.4937 1.06415 10.7423 1.19244 10.9668L2.05842 12.4663Z" fill="#0066DA"/>
+<path d="M8 5.7549L5.79496 1.93817C5.57846 2.06646 5.39404 2.24287 5.26575 2.46738L1.19244 9.52351C1.06651 9.74318 1.00017 9.99195 1 10.2452H5.41008L8 5.7549Z" fill="#00AC47"/>
+<path d="M12.7947 14.0619C13.0112 13.9336 13.1956 13.7572 13.3239 13.5327L13.5805 13.0917L14.8073 10.9668C14.9356 10.7423 14.9998 10.4937 14.9998 10.2452H10.5894L11.5278 12.0894L12.7947 14.0619Z" fill="#EA4335"/>
+<path d="M7.99996 5.75489L10.205 1.93817C9.98851 1.80987 9.73994 1.74573 9.48335 1.74573H6.51657C6.25999 1.74573 6.01142 1.81789 5.79492 1.93817L7.99996 5.75489Z" fill="#00832D"/>
+<path d="M10.59 10.2452H5.41012L3.20508 14.0619C3.42157 14.1902 3.67014 14.2543 3.92673 14.2543H12.0733C12.3299 14.2543 12.5785 14.1822 12.795 14.0619L10.59 10.2452Z" fill="#2684FC"/>
+<path d="M12.7709 5.99544L10.7342 2.46738C10.606 2.24287 10.4215 2.06646 10.205 1.93817L8 5.7549L10.5899 10.2452H14.992C14.992 9.99659 14.9278 9.74802 14.7995 9.52351L12.7709 5.99544Z" fill="#FFBA00"/>
+</g>
+<defs>
+<clipPath id="clip0_12770_5520">
+<rect width="14" height="12.5086" fill="white" transform="translate(1 1.74573)"/>
+</clipPath>
+</defs>
+</svg>

frontend/src/metabase/ui/components/icons/Icon/icons/google_sheet.svg

@@ -0,0 +1,6 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M12.8838 5.53033L12.8839 5.53034C13.1183 5.76477 13.25 6.08271 13.25 6.41421V13C13.25 13.6904 12.6904 14.25 12 14.25H4C3.30964 14.25 2.75 13.6904 2.75 13V3C2.75 2.30964 3.30964 1.75 4 1.75H8.58579C8.91731 1.75 9.23525 1.88169 9.46966 2.11611L12.8838 5.53033ZM9 2.35354V2.25H8.5H4C3.58579 2.25 3.25 2.58579 3.25 3V13C3.25 13.4143 3.58583 13.75 4 13.75H12C12.4142 13.75 12.75 13.4142 12.75 13V6.5V6H12.6464L11.7929 5.14645L9.85356 3.20711L9 2.35354Z" fill="#4C5773" stroke="#4C5773"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M4 1.25C3.0335 1.25 2.25 2.0335 2.25 3V13C2.25 13.9665 3.0335 14.75 4 14.75H12C12.9665 14.75 13.75 13.9665 13.75 13V6.41421C13.75 5.95008 13.5656 5.50497 13.2374 5.17678L9.82322 1.76256C9.49503 1.43437 9.04992 1.25 8.58579 1.25H4Z" fill="#26A668"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M13.7003 6.00012H9.5C9.22386 6.00012 9 5.77626 9 5.50012V1.2998C9.30909 1.37511 9.59454 1.53397 9.82322 1.76265L13.2374 5.17686C13.4661 5.40555 13.625 5.69101 13.7003 6.00012Z" fill="#92D3B4"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M5.5 8.5H10.5V9.5H5.5V8.5ZM9.5 10.5H5.5V11.5H9.5V10.5Z" fill="white"/>
+</svg>

frontend/src/metabase/ui/components/icons/Icon/icons/index.ts

@@ -167,6 +167,10 @@ import globe_component from "./globe.svg?component";
 import globe_source from "./globe.svg?source";
 import google_component from "./google.svg?component";
 import google_source from "./google.svg?source";
+import google_drive_component from "./google_drive.svg?component";
+import google_drive_source from "./google_drive.svg?source";
+import google_sheet_component from "./google_sheet.svg?component";
+import google_sheet_source from "./google_sheet.svg?source";
 import grabber_component from "./grabber.svg?component";
 import grabber_source from "./grabber.svg?source";
 import grid_component from "./grid.svg?component";
@@ -753,6 +757,14 @@ export const Icons = {
     component: google_component,
     source: google_source,
   },
+  google_drive: {
+    component: google_drive_component,
+    source: google_drive_source,
+  },
+  google_sheet: {
+    component: google_sheet_component,
+    source: google_sheet_source,
+  },
   history: {
     component: history_component,
     source: history_source,

frontend/test/__support__/server-mocks/gsheets.ts

@@ -0,0 +1,36 @@
+import fetchMock from "fetch-mock";
+
+import type { Settings } from "metabase-types/api";
+
+type Props =
+  | {
+      status?: never;
+      errorCode: number;
+    }
+  | {
+      status: Settings["gsheets"]["status"];
+      errorCode?: never;
+    };
+
+export function setupGsheetsGetFolderEndpoint({ status, errorCode }: Props) {
+  if (status) {
+    fetchMock.get("path:/api/ee/gsheets/folder", () => {
+      return {
+        status,
+        db_id: 1,
+      };
+    });
+  }
+
+  if (errorCode) {
+    fetchMock.get("path:/api/ee/gsheets/folder", errorCode);
+  }
+}
+
+export function setupGsheetsServiceAccountEndpoint(
+  email = "service-account123@testing.metabase.com",
+) {
+  fetchMock.get("path:/api/ee/gsheets/service-account", () => {
+    return { email };
+  });
+}

frontend/test/__support__/server-mocks/index.ts

@@ -15,6 +15,7 @@ export * from "./dataset";
 export * from "./email";
 export * from "./field";
 export * from "./group";
+export * from "./gsheets";
 export * from "./impersonation";
 export * from "./model-indexes";
 export * from "./native-query-snippet";

src/metabase/analytics/prometheus.clj

@@ -271,7 +271,16 @@
                        {:description "Number of errors when sending channel notifications."
                         :labels [:payload-type :channel-type]})
    (prometheus/gauge :metabase-notification/concurrent-tasks
-                     {:description "Number of concurrent notification sends."})])
+                     {:description "Number of concurrent notification sends."})
+   (prometheus/counter :metabase-gsheets/connection-creation-began
+                       {:description "How many times the instance has initiated a Google Sheets connection creation."})
+   (prometheus/counter :metabase-gsheets/connection-creation-ok
+                       {:description "How many times the instance has created a Google Sheets connection."})
+   (prometheus/counter :metabase-gsheets/connection-creation-error
+                       {:description "How many failures there were when creating a Google Sheets connection."
+                        :labels [:reason]})
+   (prometheus/counter :metabase-gsheets/connection-deleted
+                       {:description "How many times the instance has deleted their Google Sheets connection."})])
 
 (defmulti known-labels
   "Implement this for a given metric to initialize it for the given set of label values."
@@ -413,5 +422,15 @@
    (prometheus/set (:registry system) metric (qualified-vals labels) amount)))
 
 (comment
+  ;; want to see what's in the registry?
   (require 'iapetos.export)
-  (spit "metrics" (iapetos.export/text-format (:registry system))))
+  (spit "metrics" (iapetos.export/text-format (:registry system)))
+
+  ;; need to restart the server to see the metrics? use:
+  (shutdown!)
+
+  ;; get the value of a metric:
+  (prometheus/value (:registry system) :metabase-gsheets/connection-creation-began)
+
+  ;; w/ a label:
+  (prometheus/value (:registry system) :metabase-gsheets/connection-creation-error [[:reason "timeout"]]))

src/metabase/analytics/snowplow.clj

@@ -58,6 +58,7 @@
    :snowplow/embed_share    "1-0-0"
    :snowplow/llm_usage      "1-0-0"
    :snowplow/serialization  "1-0-1"
+   :snowplow/simple_event   "1-0-0"
    :snowplow/cleanup        "1-0-0"})
 
 (def ^:private SnowplowSchema

src/metabase/api/auth.clj

@@ -0,0 +1,11 @@
+(ns metabase.api.auth
+  (:require
+   [metabase.models.setting :refer [defsetting]]))
+
+(defsetting api-key
+  "When set, this API key is required for all API requests."
+  :encryption :when-encryption-key-set
+  :visibility :internal
+  :doc "Middleware that enforces validation of the client via the request header X-Metabase-Apikey.
+        If the header is available, then it’s validated against MB_API_KEY.
+        When it matches, the request continues; otherwise it’s blocked with a 403 Forbidden response.")

src/metabase/api/database.clj

@@ -1149,7 +1149,8 @@
   [{:keys [id]} :- [:map
                     [:id ms/PositiveInt]]]
   (let [db (api/check-404 (t2/select-one :model/Database id))]
-    (api/check-403 (mi/can-write? db))
+    (or (api/check-403 (mi/can-write? db))
+        (:is_attached_dwh db))
     (->> db
          (driver/syncable-schemas (:engine db))
          (vec)

src/metabase/cloud_migration/core.clj

@@ -8,4 +8,6 @@
 (p/import-vars
  [metabase.cloud-migration.settings
   read-only-mode
-  read-only-mode!])
+  read-only-mode!
+
+  store-api-url])

src/metabase/models/api_key.clj

@@ -68,13 +68,13 @@
   (u.secret/secret
    (str "mb_" (crypto-random/base64 bytes-key-length))))
 
-(def ^:private string-key-length (count (u.secret/expose (generate-key))))
+(def ^:private string-key-length (delay (count (u.secret/expose (generate-key)))))
 
 (defn mask
   "Given an API key, returns a string of the same length with all but the prefix masked with `*`s"
   [key]
   (->> (concat (prefix key) (repeat "*"))
-       (take string-key-length)
+       (take @string-key-length)
        (apply str)))
 
 (defn- add-key

src/metabase/models/setting.clj

@@ -325,6 +325,7 @@
   clojure.lang.Keyword
   (resolve-setting [k]
     (or (@registered-settings k)
+        (@registered-settings (u/->kebab-case-en k))
         (throw (ex-info (tru "Unknown setting: {0}" k)
                         {::unknown-setting-error true
                          :registered-settings

src/metabase/premium_features/token_check.clj

@@ -110,7 +110,7 @@
    [:company       {:optional true} [:string {:min 1}]]])
 
 (def ^:private ^:const token-status-cache-ttl
-  "Amount of time to cache the status of a valid enterprise token before forcing a re-check."
+  "Amount of time in ms to cache the status of a valid enterprise token before forcing a re-check."
   (u/hours->ms 12))
 
 (def ^{:arglists '([token base-url site-uuid])} ^:private fetch-token-and-parse-body*

src/metabase/server/middleware/auth.clj

@@ -3,10 +3,15 @@
   is not used as part of the normal `app`; it is instead added selectively to appropriate routes."
   (:require
    [clojure.string :as str]
-   [metabase.models.setting :refer [defsetting]]
+   [metabase.api.auth :as api.auth]
+   [metabase.models.setting :as setting]
    [metabase.request.core :as request]
    [metabase.util.i18n :refer [deferred-trs]]))
 
+(comment
+  ;; needed to load the api-key setting
+  api.auth/keep-me)
+
 (def ^:private ^:const ^String static-metabase-api-key-header "x-metabase-apikey")
 
 (defn enforce-authentication
@@ -33,18 +38,11 @@
      (handler (wrap-static-api-key* request) respond raise))
    (meta handler)))
 
-(defsetting api-key
-  "When set, this API key is required for all API requests."
-  :encryption :when-encryption-key-set
-  :visibility :internal
-  :doc "Middleware that enforces validation of the client via the request header X-Metabase-Apikey.
-        If the header is available, then it’s validated against MB_API_KEY.
-        When it matches, the request continues; otherwise it’s blocked with a 403 Forbidden response.")
-
 (defn static-api-key
   "We don't want to change the name of the setting from `MB_API_KEY`, but we want to differentiate this static key from
   the API keys that can be generated by admins."
-  [] (api-key))
+  []
+  (setting/get :api-key))
 
 (def mb-api-key-doc-url
   "Url for documentation on how to set MB_API_KEY."

src/metabase/sync/interface.clj

@@ -15,7 +15,8 @@
    ;; for databases that support forcing query to include a filter (e.g: partitioned table on bigquery)
    [:database_require_filter {:optional true} [:maybe :boolean]]
    ;; `:description` in this case should be a column/remark on the Table, if there is one.
-   [:description             {:optional true} [:maybe :string]]])
+   [:description             {:optional true} [:maybe :string]]
+   [:visibility_type         {:optional true} [:maybe :string]]])
 
 (def DatabaseMetadataTable
   "Schema for the expected output of `describe-database` for a Table."
@@ -49,7 +50,8 @@
    [:database-is-auto-increment {:optional true} :boolean]
    ;; nullable for databases that don't support field partition
    [:database-partitioned       {:optional true} [:maybe :boolean]]
-   [:database-required          {:optional true} :boolean]])
+   [:database-required          {:optional true} :boolean]
+   [:visibility-type            {:optional true} [:maybe :keyword]]])
 
 (def TableMetadataField
   "Schema for a given Field as provided in [[metabase.driver/describe-table]]."

src/metabase/sync/sync_metadata/crufty.clj

@@ -0,0 +1,34 @@
+(ns metabase.sync.sync-metadata.crufty
+  "This namespace contains functions for determining whether or not a table or column name is 'crufty' during syncs.
+
+  In practice, this means that the table or column name is automatically hidden during a syncs.
+
+  We do not _unhide_ tables or columns that are hidden during syncs, so that in the case where a table or column name
+  happens to match our idea of crufty, but admins want to see it, they can manually unhide tables or columns, and
+  continue to see them."
+  (:require
+   [metabase.util :as u]))
+
+(defn- matches-any-patterns? [name patterns]
+  (some #(re-find % name) patterns))
+
+(defn- ->lower-cased-regex
+  "Converts a pattern-string into a lower-cased regex."
+  [pattern]
+  (-> pattern u/lower-case-en re-pattern))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PUBLIC API
+
+(defn name?
+  "Returns true if the table name re-find matches any of the (regex) patterns or pattern-strings (after getting passed
+  through re-pattern).
+
+  This is what does the work of deciding whether or not a table or column name is automatically hidden during syncs.
+  These are set from database.setting.auto-cruft-tables and database.setting.auto-cruft-columns, and are both vectors
+  of strings that are converted into regexes.
+
+  If you want to match a table or column name exactly, use `^my-table$` or `^my-column$`."
+  [nname regexes+pattern-strings]
+  (matches-any-patterns? (u/lower-case-en nname)
+                         (map ->lower-cased-regex regexes+pattern-strings)))

src/metabase/sync/sync_metadata/fields.clj

@@ -40,6 +40,7 @@
     for logging purposes by higher-level sync logic."
   (:require
    [metabase.driver.util :as driver.u]
+   [metabase.models.setting :refer [defsetting]]
    [metabase.models.table :as table]
    [metabase.sync.fetch-metadata :as fetch-metadata]
    [metabase.sync.interface :as i]
@@ -53,6 +54,16 @@
    [toucan2.core :as t2]
    [toucan2.util :as t2.util]))
 
+(defsetting auto-cruft-columns
+  "A list of pattern strings that get converted into additional regexes that match Fields that should automatically be
+  marked as visibility-type = `:hidden`. Not to be set directly, this setting lives in the metabase_database.settings json blob."
+  :type :json
+  :database-local :only
+  :visibility :internal
+  :default []
+  :export? true
+  :encryption :no)
+
 ;;; +----------------------------------------------------------------------------------------------------------------+
 ;;; |                                            PUTTING IT ALL TOGETHER                                             |
 ;;; +----------------------------------------------------------------------------------------------------------------+
@@ -60,13 +71,14 @@
 (mu/defn- sync-and-update! :- ms/IntGreaterThanOrEqualToZero
   "Sync Field instances (i.e., rows in the Field table in the Metabase application DB) for a Table, and update metadata
   properties (e.g. base type and comment/remark) as needed. Returns number of Fields synced."
-  [table       :- i/TableInstance
+  [database    :- i/DatabaseInstance
+   table       :- i/TableInstance
    db-metadata :- [:set i/TableMetadataField]]
   (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))
      ;; Now that tables are synced and fields created as needed make sure field properties are in sync.
-     ;; Re-fetch our metadata because there might be somethings that have changed after calling
+     ;; Re-fetch our metadata because there might be some things that have changed after calling
      ;; `sync-instances`
-     (sync-metadata/update-metadata! table db-metadata (fields.our-metadata/our-metadata table))))
+     (sync-metadata/update-metadata! database table db-metadata (fields.our-metadata/our-metadata table))))
 
 (mu/defn sync-fields! :- [:map
                           [:updated-fields ms/IntGreaterThanOrEqualToZero]
@@ -97,7 +109,7 @@
                                                                (set table-metadata)
                                                                database
                                                                table)]
-                                             (sync-and-update! table all-metadata))
+                                             (sync-and-update! database table all-metadata))
                                            (catch Exception e
                                              (log/error e)
                                              0))
@@ -123,4 +135,4 @@
            ;; Also this should be a driver method, not a sql-jdbc.sync method
            db-metadata (fetch-metadata/include-nested-fields-for-table db-metadata database table)]
        {:total-fields   (count db-metadata)
-        :updated-fields (sync-and-update! table db-metadata)}))))
+        :updated-fields (sync-and-update! database table db-metadata)}))))

src/metabase/sync/sync_metadata/fields/our_metadata.clj

@@ -34,7 +34,8 @@
           :position                  (:position field)
           :database-position         (:database_position field)
           :database-partitioned      (:database_partitioned field)
-          :database-required         (:database_required field)})
+          :database-required         (:database_required field)
+          :visibility-type           (:visibility_type field)})
        ;; make a map of parent-id -> set of child Fields
        (group-by :parent-id)
        ;; remove the parent ID because the Metadata from `describe-table` won't have it. Save the results as a set

src/metabase/sync/sync_metadata/fields/sync_metadata.clj

@@ -5,6 +5,7 @@
   (:require
    [clojure.string :as str]
    [metabase.sync.interface :as i]
+   [metabase.sync.sync-metadata.crufty :as crufty]
    [metabase.sync.sync-metadata.fields.common :as common]
    [metabase.sync.util :as sync-util]
    [metabase.util :as u]
@@ -13,10 +14,20 @@
    [metabase.util.malli.schema :as ms]
    [toucan2.core :as t2]))
 
+(defn- compute-new-visibility-type [db field-metadata]
+  (if (crufty/name? (:name field-metadata)
+                    (some-> db :settings :auto-cruft-columns))
+    :details-only
+    ;; n.b. if it was auto-crufted in the past, removing it from auto-cruft will NOT make it visible because old
+    ;; visibility-type will be :details-only. This only changes things to be hidden. If you want to make it visible
+    ;; again, you need to change the visibility-type to :normal via the fields :put api.
+    (:visibility-type field-metadata)))
+
 (mu/defn- update-field-metadata-if-needed! :- [:enum 0 1]
   "Update the metadata for a Metabase Field as needed if any of the info coming back from the DB has changed. Syncs
   base type, database type, semantic type, and comments/remarks; returns `1` if the Field was updated; `0` otherwise."
-  [table          :- i/TableInstance
+  [database       :- i/DatabaseInstance
+   table          :- i/TableInstance
    field-metadata :- i/TableMetadataField
    metabase-field :- common/TableMetadataFieldWithID]
   (let [{old-database-type              :database-type
@@ -28,7 +39,8 @@
          old-database-name              :name
          old-database-is-auto-increment :database-is-auto-increment
          old-db-partitioned             :database-partitioned
-         old-db-required                :database-required} metabase-field
+         old-db-required                :database-required
+         old-visibility-type            :visibility-type} metabase-field
         {new-database-type              :database-type
          new-base-type                  :base-type
          new-field-comment              :field-comment
@@ -37,6 +49,7 @@
          new-database-is-auto-increment :database-is-auto-increment
          new-db-partitioned             :database-partitioned
          new-db-required                :database-required} field-metadata
+        new-visibility-type             (compute-new-visibility-type database field-metadata)
         new-database-is-auto-increment  (boolean new-database-is-auto-increment)
         new-db-required                 (boolean new-db-required)
         new-database-type               (or new-database-type "NULL")
@@ -67,6 +80,7 @@
         new-db-auto-incremented? (not= old-database-is-auto-increment new-database-is-auto-increment)
         new-db-partitioned?      (not= new-db-partitioned old-db-partitioned)
         new-db-required?         (not= old-db-required new-db-required)
+        new-visibility-type?     (not= old-visibility-type new-visibility-type)
 
         ;; calculate combined updates
         updates
@@ -136,7 +150,9 @@
                       (common/field-metadata-name-for-logging table metabase-field)
                       old-db-required
                       new-db-required)
-           {:database_required new-db-required}))]
+           {:database_required new-db-required})
+         (when new-visibility-type?
+           {:visibility_type new-visibility-type}))]
     ;; if any updates need to be done, do them and return 1 (because 1 Field was updated), otherwise return 0
     (if (and (seq updates)
              (pos? (t2/update! :model/Field (u/the-id metabase-field) updates)))
@@ -147,24 +163,26 @@
 
 (mu/defn- update-nested-fields-metadata! :- ms/IntGreaterThanOrEqualToZero
   "Recursively call `update-metadata!` for all the nested Fields in a `metabase-field`."
-  [table          :- i/TableInstance
+  [database       :- i/DatabaseInstance
+   table          :- i/TableInstance
    field-metadata :- i/TableMetadataField
    metabase-field :- common/TableMetadataFieldWithID]
   (let [nested-fields-metadata (:nested-fields field-metadata)
         metabase-nested-fields (:nested-fields metabase-field)]
     (if (seq metabase-nested-fields)
-      (update-metadata! table (set nested-fields-metadata) (set metabase-nested-fields))
+      (update-metadata! database table (set nested-fields-metadata) (set metabase-nested-fields))
       0)))
 
 (mu/defn update-metadata! :- ms/IntGreaterThanOrEqualToZero
   "Make sure things like PK status and base-type are in sync with what has come back from the DB. Recursively updates
   nested Fields. Returns total number of Fields updated."
-  [table        :- i/TableInstance
+  [database     :- i/DatabaseInstance
+   table        :- i/TableInstance
    db-metadata  :- [:set i/TableMetadataField]
    our-metadata :- [:set common/TableMetadataFieldWithID]]
   (sync-util/sum-for [metabase-field our-metadata]
     ;; only update metadata for 'existing' Fields that are present in our Metadata (i.e., present in the application
     ;; database) and that are still considered active (i.e., present in DB metadata)
     (when-let [field-metadata (common/matching-field-metadata metabase-field db-metadata)]
-      (+ (update-field-metadata-if-needed! table field-metadata metabase-field)
-         (update-nested-fields-metadata! table field-metadata metabase-field)))))
+      (+ (update-field-metadata-if-needed! database table field-metadata metabase-field)
+         (update-nested-fields-metadata! database table field-metadata metabase-field)))))

src/metabase/sync/sync_metadata/tables.clj

@@ -7,8 +7,10 @@
    [metabase.lib.schema.common :as lib.schema.common]
    [metabase.models.humanization :as humanization]
    [metabase.models.interface :as mi]
+   [metabase.models.setting :refer [defsetting]]
    [metabase.sync.fetch-metadata :as fetch-metadata]
    [metabase.sync.interface :as i]
+   [metabase.sync.sync-metadata.crufty :as crufty]
    [metabase.sync.sync-metadata.metabase-metadata :as metabase-metadata]
    [metabase.sync.util :as sync-util]
    [metabase.util :as u]
@@ -75,10 +77,15 @@
     ;; MSSQL
     #"^syncobj_0x.*"})
 
-(mu/defn- is-crufty-table?
-  "Should we give newly created TABLE a `visibility_type` of `:cruft`?"
-  [table-name]
-  (some #(re-find % (u/lower-case-en table-name)) crufty-table-patterns))
+(defsetting auto-cruft-tables
+  "A list of pattern strings that get converted into additional regexes that match Tables that should automatically be
+  marked as `:cruft`."
+  :type :json
+  :database-local :only
+  :visibility :internal
+  :export? true
+  :default []
+  :encryption :no)
 
 ;;; ---------------------------------------------------- Syncing -----------------------------------------------------
 
@@ -91,10 +98,18 @@
               {:details
                (assoc (:details database) :version (:version db-metadata))}))
 
-(defn- cruft-dependent-columns [table-name]
-  ;; if this is a crufty table, mark initial sync as complete since we'll skip the subsequent sync steps
-  (let [is-crufty? (is-crufty-table? table-name)]
-    {:initial_sync_status (if is-crufty? "complete" "incomplete")
+(mu/defn- cruft-dependent-cols [{table-name :name :as table}
+                                database
+                                sync-stage :- [:enum ::reactivate ::create ::update]]
+  (let [is-crufty? (crufty/name? table-name (into crufty-table-patterns
+                                                  (some-> database :settings :auto-cruft-tables)))]
+    {:initial_sync_status (cond
+                            ;; if we're updating a table, we don't overwrite the initial sync status, so that it remain
+                            ;; "complete" during the sync. See:
+                            ;; [[metabase.sync.util-test/initial-sync-status-table-only-test]]
+                            (= sync-stage ::update) (:initial_sync_status table)
+                            is-crufty?              "complete"
+                            :else                   "incomplete")
      :visibility_type     (when is-crufty? :cruft)}))
 
 (defn create-table!
@@ -103,13 +118,14 @@
   [database table]
   (t2/insert-returning-instance!
    :model/Table
-   (merge (cruft-dependent-columns (:name table))
+   (merge (cruft-dependent-cols table database ::create)
           {:active                  true
            :db_id                   (:id database)
            :schema                  (:schema table)
            :description             (:description table)
            :database_require_filter (:database_require_filter table)
-           :display_name            (or (:display_name table) (humanization/name->human-readable-name (:name table)))
+           :display_name            (or (:display_name table)
+                                        (humanization/name->human-readable-name (:name table)))
            :name                    (:name table)})))
 
 (defn create-or-reactivate-table!
@@ -120,8 +136,11 @@
                                          :schema schema
                                          :name table-name
                                          :active false)]
-    ;; if the table already exists but is marked *inactive*, mark it as *active*
-    (t2/update! :model/Table existing-id (assoc (cruft-dependent-columns (:name table)) :active true))
+    (do (t2/select-one :model/Table existing-id)
+        ;; if the table already exists but is marked *inactive*, mark it as *active*
+        (t2/update! :model/Table existing-id (assoc
+                                              (cruft-dependent-cols table database ::reactivate)
+                                              :active true)))
     ;; otherwise create a new Table
     (create-table! database table)))
 
@@ -130,12 +149,12 @@
 (mu/defn- create-or-reactivate-tables!
   "Create `new-tables` for database, or if they already exist, mark them as active."
   [database :- i/DatabaseInstance
-   new-tables :- [:set i/DatabaseMetadataTable]]
-  (doseq [table new-tables]
+   new-table-metadatas :- [:set i/DatabaseMetadataTable]]
+  (doseq [table-metadata new-table-metadatas]
     (log/info "Found new table:"
-              (sync-util/name-for-logging (mi/instance :model/Table table))))
-  (doseq [table new-tables]
-    (create-or-reactivate-table! database table)))
+              (sync-util/name-for-logging (mi/instance :model/Table table-metadata))))
+  (doseq [table-metadata new-table-metadatas]
+    (create-or-reactivate-table! database table-metadata)))
 
 (mu/defn- retire-tables!
   "Mark any `old-tables` belonging to `database` as inactive."
@@ -153,23 +172,30 @@
                               :active true}
                 {:active false})))
 
+(def ^:private keys-to-update
+  [:description :database_require_filter :estimated_row_count :visibility_type :initial_sync_status])
+
 (mu/defn- update-table-metadata-if-needed!
   "Update the table metadata if it has changed."
   [table-metadata :- i/DatabaseMetadataTable
-   metabase-table :- (ms/InstanceOf :model/Table)]
+   metabase-table :- (ms/InstanceOf :model/Table)
+   metabase-database :- (ms/InstanceOf :model/Database)]
   (log/infof "Updating table metadata for %s" (sync-util/name-for-logging metabase-table))
-  (let [to-update-keys [:description :database_require_filter :estimated_row_count]
-        old-table      (select-keys metabase-table to-update-keys)
-        new-table      (select-keys (merge
-                                     (zipmap to-update-keys (repeat nil))
-                                     table-metadata)
-                                    to-update-keys)
-        [_ changes _]  (data/diff old-table new-table)
-        changes        (cond-> changes
-                         ;; we only update the description if the initial state is nil
-                         ;; because don't want to override the user edited description if it exists
-                         (some? (:description old-table))
-                         (dissoc changes :description))]
+  (let [old-table               (select-keys metabase-table keys-to-update)
+        new-table               (-> (zipmap keys-to-update (repeat nil))
+                                    (merge table-metadata
+                                           (cruft-dependent-cols metabase-table metabase-database
+                                                                 ::update))
+                                    (select-keys keys-to-update))
+        [_ changes _]           (data/diff old-table new-table)
+        changes                 (cond-> changes
+                                  ;; we only update the description if the initial state is nil
+                                  ;; because don't want to override the user edited description if it exists:
+                                  (some? (:description old-table))
+                                  (dissoc changes :description)
+
+                                  (= (:visibility_type new-table) (:visibility_type old-table))
+                                  (dissoc changes :visibility_type))]
     (doseq [[k v] changes]
       (log/infof "%s of %s changed from %s to %s"
                  k
@@ -181,11 +207,14 @@
 
 (mu/defn- update-tables-metadata-if-needed!
   [table-metadatas :- [:set i/DatabaseMetadataTable]
-   metabase-tables :- [:set (ms/InstanceOf :model/Table)]]
+   metabase-tables :- [:set (ms/InstanceOf :model/Table)]
+   metabase-database :- (ms/InstanceOf :model/Database)]
   (let [name+schema->table-metadata (m/index-by (juxt :name :schema) table-metadatas)
         name+schema->metabase-table (m/index-by (juxt :name :schema) metabase-tables)]
     (doseq [name+schema (set/intersection (set (keys name+schema->table-metadata)) (set (keys name+schema->metabase-table)))]
-      (update-table-metadata-if-needed! (name+schema->table-metadata name+schema) (name+schema->metabase-table name+schema)))))
+      (update-table-metadata-if-needed! (name+schema->table-metadata name+schema)
+                                        (name+schema->metabase-table name+schema)
+                                        metabase-database))))
 
 (mu/defn- table-set :- [:set i/DatabaseMetadataTable]
   "So there exist tables for the user and metabase metadata tables for internal usage by metabase.
@@ -198,7 +227,8 @@
 (mu/defn- db->our-metadata :- [:set (ms/InstanceOf :model/Table)]
   "Return information about what Tables we have for this DB in the Metabase application DB."
   [database :- i/DatabaseInstance]
-  (set (t2/select [:model/Table :id :name :schema :description :database_require_filter :estimated_row_count]
+  (set (t2/select [:model/Table :id :name :schema :description :database_require_filter :estimated_row_count
+                   :visibility_type :initial_sync_status]
                   :db_id  (u/the-id database)
                   :active true)))
 
@@ -211,34 +241,35 @@
 
   ([database :- i/DatabaseInstance db-metadata]
    ;; determine what's changed between what info we have and what's in the DB
-   (let [db-tables               (table-set db-metadata)
-         name+schema             #(select-keys % [:name :schema])
-         name+schema->db-table   (m/index-by name+schema db-tables)
-         our-metadata            (db->our-metadata database)
-         keep-name+schema-set    (fn [metadata]
-                                   (set (map name+schema metadata)))
-         [new-tables old-tables] (data/diff
-                                  (keep-name+schema-set (set (map name+schema db-tables)))
-                                  (keep-name+schema-set (set (map name+schema our-metadata))))]
+   (let [db-table-metadatas    (table-set db-metadata)
+         name+schema           #(select-keys % [:name :schema])
+         name+schema->db-table (m/index-by name+schema db-table-metadatas)
+         our-metadata          (db->our-metadata database)
+         keep-name+schema-set  (fn [metadata]
+                                 (set (map name+schema metadata)))
+         [new-table-metadatas
+          old-table-metadatas] (data/diff
+                                (keep-name+schema-set (set (map name+schema db-table-metadatas)))
+                                (keep-name+schema-set (set (map name+schema our-metadata))))]
      ;; update database metadata from database
      (when (some? (:version db-metadata))
        (sync-util/with-error-handling (format "Error creating/reactivating tables for %s"
                                               (sync-util/name-for-logging database))
          (update-database-metadata! database db-metadata)))
      ;; create new tables as needed or mark them as active again
-     (when (seq new-tables)
-       (let [new-tables-info (set (map #(get name+schema->db-table (name+schema %)) new-tables))]
+     (when (seq new-table-metadatas)
+       (let [new-tables-info (set (map #(get name+schema->db-table (name+schema %)) new-table-metadatas))]
          (sync-util/with-error-handling (format "Error creating/reactivating tables for %s"
                                                 (sync-util/name-for-logging database))
            (create-or-reactivate-tables! database new-tables-info))))
      ;; mark old tables as inactive
-     (when (seq old-tables)
+     (when (seq old-table-metadatas)
        (sync-util/with-error-handling (format "Error retiring tables for %s" (sync-util/name-for-logging database))
-         (retire-tables! database old-tables)))
+         (retire-tables! database old-table-metadatas)))
 
      (sync-util/with-error-handling (format "Error updating table metadata for %s" (sync-util/name-for-logging database))
        ;; we need to fetch the tables again because we might have retired tables in the previous steps
-       (update-tables-metadata-if-needed! db-tables (db->our-metadata database)))
+       (update-tables-metadata-if-needed! db-table-metadatas (db->our-metadata database) database))
 
-     {:updated-tables (+ (count new-tables) (count old-tables))
+     {:updated-tables (+ (count new-table-metadatas) (count old-table-metadatas))
       :total-tables   (count our-metadata)})))

src/metabase/upload.clj

@@ -891,21 +891,22 @@
 
 (defn- can-delete-error
   "Returns an ExceptionInfo object if the user cannot delete the given upload. Returns nil otherwise."
-  [table]
-  (cond
-    (not (:is_upload table))
-    (ex-info (tru "The table must be an uploaded table.")
-             {:status-code 422})
+  [table database]
+  (when-not (:is_attached_dwh database) ;; gsheets uploads: deletable, but we cannot write + they aren't is_upload
+    (cond
+      (not (:is_upload table))
+      (ex-info (tru "The table must be an uploaded table.")
+               {:status-code 422})
 
-    (not (mi/can-write? table))
-    (ex-info (tru "You don''t have permissions to do that.")
-             {:status-code 403})))
+      (not (mi/can-write? table))
+      (ex-info (tru "You don''t have permissions to do that.")
+               {:status-code 403}))))
 
 (defn- check-can-delete
   "Throws an error if the given table is not an upload, or if the user does not have permission to delete it."
-  [table]
+  [table database]
   ;; For now anyone that can update a table is allowed to delete it.
-  (when-let [error (can-delete-error table)]
+  (when-let [error (can-delete-error table database)]
     (throw error)))
 
 ;;; +--------------------------------------------------
@@ -918,7 +919,7 @@
   (let [database   (table/database table)
         driver     (driver.u/database->driver database)
         table-name (table-identifier table)]
-    (check-can-delete table)
+    (check-can-delete table database)
 
     ;; Attempt to delete the underlying data from the customer database.
     ;; We perform this before marking the table as inactive in the app db so that even if it false, the table is still

test/metabase/sync/sync_metadata/fields/sync_instances_test.clj

@@ -150,8 +150,42 @@
                                                                    :table_id      transactions-table-id
                                                                    :parent_id     food-likes-field-id
                                                                    :active        true))]
-
         ;; now sync again.
         (sync-metadata/sync-db-metadata! db)
         ;; field should become inactive
         (is (false? (t2/select-one-fn :active :model/Field :id blueberries-field-id)))))))
+
+(defn run-cruft-test [patterns freqs]
+  (mt/with-temp [:model/Database db {:engine ::toucanery/toucanery
+                                     :settings {:auto-cruft-columns patterns}}]
+    (sync-metadata/sync-db-metadata! db)
+    (let [tables (t2/select :model/Table :db_id (u/the-id db))
+          fields (mapcat (fn [{:keys [id]}] (t2/select :model/Field :table_id id)) tables)]
+      (is (= freqs
+             (frequencies (map :visibility_type fields)))))))
+
+(deftest auto-cruft-all-fields-test
+  (testing "Make sure a db's settings.auto_cruft_fields mark all fields as crufty"
+    (run-cruft-test [".*"]
+                    {:details-only 12})
+    (run-cruft-test (map str (into [] "abcdefghijklmnoqprstuvwxyz"))
+                    {:details-only 12})))
+
+(deftest auto-cruft-exact-field-name-test
+  (testing "Make sure a db's settings.auto_cruft_fields mark fields as crufty for exact table names"
+    (run-cruft-test ["^details$" "^age$"]
+                    {:normal 10 :details-only 2})))
+
+(deftest auto-cruft-fields-with-multiple-patterns-test
+  (testing "Make sure a db's settings.auto_cruft_fields mark fields as crufty against multiple patterns"
+    (run-cruft-test ["a" "b" "c"]     {:normal 4 :details-only 8})
+    (run-cruft-test ["c" "a" "t"]     {:normal 3 :details-only 9})
+    (run-cruft-test ["d" "o" "g"]     {:normal 6 :details-only 6})
+    (run-cruft-test ["b" "i" "r" "d"] {:normal 7 :details-only 5})
+    (run-cruft-test ["x" "y" "z"]     {:normal 11 :details-only 1})))
+
+(deftest auto-cruft-fields-none-are-crufted-with-missing-pattern
+  (run-cruft-test [] {:normal 12}))
+
+(deftest auto-cruft-fields-none-are-crufted-with-no-hit-pattern
+  (run-cruft-test ["^it was the best$" "^of times it was$" "^the worst of times$"] {:normal 12}))

test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj

@@ -5,6 +5,7 @@
    [metabase.sync.core :as sync]
    [metabase.sync.sync-metadata.fields.sync-metadata :as sync-metadata]
    [metabase.test :as mt]
+   [metabase.util :as u]
    [next.jdbc :as next.jdbc]
    [toucan2.core :as t2]))
 
@@ -13,12 +14,14 @@
    ;; use alphabetical field_order by default because the default, database, will update the position
    (updates-that-will-be-performed! new-metadata-from-sync metadata-in-application-db {:field_order :alphabetical}))
   ([new-metadata-from-sync metadata-in-application-db table]
-   (mt/with-temp [:model/Table table table]
+   (mt/with-temp [:model/Database db {}
+                  :model/Table table (assoc table :db_id (u/the-id db))]
      (let [update-operations (atom [])]
        (with-redefs [t2/update! (fn [model id updates]
                                   (swap! update-operations conj [(name model) id updates])
                                   (count updates))]
          (#'sync-metadata/update-field-metadata-if-needed!
+          db
           table
           new-metadata-from-sync
           metadata-in-application-db)

test/metabase/sync/sync_metadata/fields_test.clj

@@ -8,11 +8,13 @@
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.query-processor :as qp]
    [metabase.sync.core :as sync]
+   [metabase.sync.sync-metadata :as sync-metadata]
    [metabase.sync.sync-metadata.fields :as sync-fields]
    [metabase.sync.sync-metadata.fks :as sync-fks]
    [metabase.sync.util-test :as sync.util-test]
    [metabase.test :as mt]
    [metabase.test.data.one-off-dbs :as one-off-dbs]
+   [metabase.test.mock.toucanery :as toucanery]
    [metabase.util :as u]
    [toucan2.connection :as t2.connection]
    [toucan2.core :as t2]))
@@ -338,3 +340,36 @@
                                          :table_id [:in (map :id tables)])
                               distinct
                               (sort-by :name)))))))))))))
+
+(defn db->fields [db]
+  (let [tables (t2/select :model/Table :db_id (u/the-id db))]
+    (mapcat (fn [table]
+              (t2/select :model/Field :table_id (u/the-id table)))
+            tables)))
+
+(deftest auto-cruft-fields-with-an-l-test
+  (testing "Make sure a db's settings.auto-cruft-tables actually mark tables as crufty"
+    (mt/with-temp [:model/Database db {:engine ::toucanery/toucanery
+                                       :settings {:auto-cruft-columns ["l"]}}]
+      (sync-metadata/sync-db-metadata! db)
+      (is (= [["details" :details-only]]
+             (->> (db->fields db)
+                  (filter #(= :details-only (:visibility_type %)))
+                  (mapv (juxt :name :visibility_type))))))))
+
+(deftest auto-cruft-all-fields-and-they-stay-crufted-test
+  (testing "Make sure a db's settings.auto-cruft-tables actually mark tables as crufty"
+    (mt/with-temp [:model/Database db {:engine ::toucanery/toucanery
+                                       :settings {:auto-cruft-columns [".*"]}}]
+      (sync-metadata/sync-db-metadata! db)
+      (is (= {:details-only 12}
+             (->> (db->fields db)
+                  (mapv :visibility_type)
+                  frequencies)))
+      ;; remove cruft column directive:
+      (t2/update! :model/Database (u/the-id db) {:settings {:auto-cruft-columns []}})
+      (sync-metadata/sync-db-metadata! db)
+      (is (= {:details-only 12}
+             (->> (db->fields db)
+                  (mapv :visibility_type)
+                  frequencies))))))

test/metabase/sync/sync_metadata/tables_test.clj

@@ -7,15 +7,17 @@
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
    [metabase.sync.core :as sync]
+   [metabase.sync.sync-metadata :as sync-metadata]
    [metabase.sync.sync-metadata.tables :as sync-tables]
    [metabase.test :as mt]
    [metabase.test.data.interface :as tx]
    [metabase.test.data.sql :as sql.tx]
+   [metabase.test.mock.toucanery :as toucanery]
    [metabase.util :as u]
    [next.jdbc :as next.jdbc]
    [toucan2.core :as t2]))
 
-(tx/defdataset db-with-some-cruft
+(tx/defdataset db-with-a-crufty-table
   [["acquired_toucans"
     [{:field-name "species"              :base-type :type/Text}
      {:field-name "cam_has_acquired_one" :base-type :type/Boolean}]
@@ -31,9 +33,9 @@
 
 (deftest crufty-tables-test
   (testing "south_migrationhistory, being a CRUFTY table, should still be synced, but marked as such"
-    (mt/dataset metabase.sync.sync-metadata.tables-test/db-with-some-cruft
+    (mt/dataset metabase.sync.sync-metadata.tables-test/db-with-a-crufty-table
       (is (= #{{:name "SOUTH_MIGRATIONHISTORY" :visibility_type :cruft :initial_sync_status "complete"}
-               {:name "ACQUIRED_TOUCANS"       :visibility_type nil    :initial_sync_status "complete"}}
+               {:name "ACQUIRED_TOUCANS"       :visibility_type nil :initial_sync_status "complete"}}
              (set (for [table (t2/select [:model/Table :name :visibility_type :initial_sync_status] :db_id (mt/id))]
                     (into {} table))))))))
 
@@ -77,3 +79,51 @@
         (sync/sync-database! (mt/db) {:scan :schema})
         (is (= 100
                (t2/select-one-fn :estimated_row_count :model/Table (mt/id :venues))))))))
+
+(defmacro ^:private run-twice
+  "Run body twice. In this case, we are checking that creating and updating syncs the tables correctly w.r.t.
+  cruftiness."
+  [& body]
+  `(do ~@body ~@body))
+
+(deftest auto-cruft-all-tables-test
+  (testing "Make sure a db's settings.auto-cruft-tables actually mark tables as crufty"
+    (mt/with-temp [:model/Database db {:engine ::toucanery/toucanery
+                                       :settings {:auto-cruft-tables [".*"]}}]
+      (run-twice
+       (sync-metadata/sync-db-metadata! db)
+       (is (= #{:cruft}
+              (t2/select-fn-set :visibility_type
+                                :model/Table
+                                :db_id
+                                (u/the-id db))))))))
+
+(deftest auto-cruft-employee-table-test
+  (testing "Make sure a db's settings.auto-cruft-tables actually mark tables as crufty"
+    (mt/with-temp [:model/Database db {:engine ::toucanery/toucanery
+                                       :settings {:auto-cruft-tables ["employees"]}}]
+      (run-twice
+       (sync-metadata/sync-db-metadata! db)
+       (is (= #{["employees" :cruft]
+                ["transactions" nil]}
+              (t2/select-fn-set (juxt :name :visibility_type) :model/Table :db_id (u/the-id db))))))))
+
+(deftest auto-cruft-tables-with-an-l-test
+  (testing "Make sure a db's settings.auto-cruft-tables actually mark tables as crufty"
+    (mt/with-temp [:model/Database db {:engine ::toucanery/toucanery
+                                       :settings {:auto-cruft-tables ["l"]}}]
+      (run-twice
+       (sync-metadata/sync-db-metadata! db)
+       (is (= #{["employees" :cruft]
+                ["transactions" nil]}
+              (t2/select-fn-set (juxt :name :visibility_type) :model/Table :db_id (u/the-id db))))))))
+
+(deftest auto-cruft-tables-with-an-l-or-a-y-test
+  (testing "Make sure a db's settings.auto-cruft-tables actually mark tables as crufty"
+    (mt/with-temp [:model/Database db {:engine ::toucanery/toucanery
+                                       :settings {:auto-cruft-tables ["l" "y"]}}]
+      (run-twice
+       (sync-metadata/sync-db-metadata! db)
+       (is (= #{["employees" :cruft]
+                ["transactions" nil]}
+              (t2/select-fn-set (juxt :name :visibility_type) :model/Table :db_id (u/the-id db))))))))

test_resources/gsheets/mock_hm_responses.edn

@@ -0,0 +1,167 @@
+;; method + url + body -> response from [[metabase-enterprise.harbormaster.client/make-request]]
+
+{{:method :delete,
+                               ;; uuid matches the one from get connections:
+  :url "/api/v2/mb/connections/049f3007-2146-4083-be38-f160c526aca7",
+  :body nil}
+ [:ok {:cached nil,
+       :request-time 945,
+       :repeatable? false,
+       :protocol-version {:name "HTTP", :major 1, :minor 1},
+       :streaming? false,
+       ;; :http-client #object[org.apache.http.impl.client.InternalHttpClient 0x61435626 "org.apache.http.impl.client.InternalHttpClient@61435626"],
+       :chunked? false,
+       :reason-phrase "OK",
+       :headers
+       {"Date" "Mon, 27 Jan 2025 21:45:48 GMT",
+        "Connection" "close",
+        "Content-Type" "application/octet-stream",
+        "X-Frame-Options" "SAMEORIGIN",
+        "X-Content-Type-Options" "nosniff",
+        "Strict-Transport-Security" "max-age=31536000; includeSubDomains",
+        "Content-Length" "0",
+        "Server" "Jetty(11.0.24)"},
+       :orig-content-encoding nil,
+       :status 200,
+       :length 0,
+       :body nil,
+       :trace-redirects []}]
+
+ {:method :get, :url "/api/v2/mb/connections", :body nil},
+ [:ok {:cached nil,
+       :request-time 23,
+       :repeatable? false,
+       :protocol-version {:name "HTTP", :major 1, :minor 1},
+       :streaming? true,
+       ;; :http-client #object[org.apache.http.impl.client.InternalHttpClient 0x8f60f32 "org.apache.http.impl.client.InternalHttpClient@8f60f32"],
+       :chunked? false,
+       :reason-phrase "OK",
+       :headers
+       {"Date" "Mon, 27 Jan 2025 18:43:42 GMT",
+        "Connection" "close",
+        "Content-Type" "application/json;charset=utf-8",
+        "X-Frame-Options" "SAMEORIGIN",
+        "X-Content-Type-Options" "nosniff",
+        "Strict-Transport-Security" "max-age=31536000; includeSubDomains",
+        "Content-Length" "349",
+        "Server" "Jetty(11.0.24)"},
+       :orig-content-encoding nil,
+       :status 200,
+       :length 349,
+       :body
+       [{:updated-at "2025-01-27T18:43:42Z",
+         :hosted-instance-resource-id 7,
+         :last-sync-at "2025-01-27T18:43:42Z",
+         :error-detail nil,
+         :type "gdrive",
+         :hosted-instance-id "f390ec19-bd44-48ae-991c-66817182a376",
+         :last-sync-started-at "2025-01-27T18:43:04Z",
+         :status "active",
+         :id "049f3007-2146-4083-be38-f160c526aca7",
+         :created-at "2025-01-27T18:43:02Z"},]
+       :trace-redirects []}]
+ {:method :get, :url "/api/v2/mb/connections/049f3007-2146-4083-be38-f160c526aca7", :body nil},
+ [:ok {:cached nil,
+       :request-time 23,
+       :repeatable? false,
+       :protocol-version {:name "HTTP", :major 1, :minor 1},
+       :streaming? true,
+       ;; :http-client #object[org.apache.http.impl.client.InternalHttpClient 0x8f60f32 "org.apache.http.impl.client.InternalHttpClient@8f60f32"],
+       :chunked? false,
+       :reason-phrase "OK",
+       :headers
+       {"Date" "Mon, 27 Jan 2025 18:43:42 GMT",
+        "Connection" "close",
+        "Content-Type" "application/json;charset=utf-8",
+        "X-Frame-Options" "SAMEORIGIN",
+        "X-Content-Type-Options" "nosniff",
+        "Strict-Transport-Security" "max-age=31536000; includeSubDomains",
+        "Content-Length" "349",
+        "Server" "Jetty(11.0.24)"},
+       :orig-content-encoding nil,
+       :status 200,
+       :length 349,
+       :body {:updated-at "2025-01-27T18:43:42Z",
+              :hosted-instance-resource-id 7,
+              :last-sync-at "2025-01-27T18:43:42Z",
+              :error-detail nil,
+              :type "gdrive",
+              :hosted-instance-id "f390ec19-bd44-48ae-991c-66817182a376",
+              :last-sync-started-at "2025-01-27T18:43:04Z",
+              :status "active",
+              :id "049f3007-2146-4083-be38-f160c526aca7",
+              :created-at "2025-01-27T18:43:02Z"},
+       :trace-redirects []}]
+ {:method :post,
+  :url "/api/v2/mb/connections",
+  :body {:type "gdrive", :secret {:resources ["<expected-gdrive-link>"]}}}
+ [:ok {:cached nil,
+       :request-time 1941,
+       :repeatable? false,
+       :protocol-version {:name "HTTP", :major 1, :minor 1},
+       :streaming? true,
+       ;; :http-client #object[org.apache.http.impl.client.InternalHttpClient 0x5b19fc1e "org.apache.http.impl.client.InternalHttpClient@5b19fc1e"],
+       :chunked? false,
+       :reason-phrase "OK",
+       :headers
+       {"Date" "Mon, 27 Jan 2025 18:43:02 GMT",
+        "Connection" "close",
+        "Content-Type" "application/json;charset=utf-8",
+        "X-Frame-Options" "SAMEORIGIN",
+        "X-Content-Type-Options" "nosniff",
+        "Strict-Transport-Security" "max-age=31536000; includeSubDomains",
+        "Content-Length" "779",
+        "Server" "Jetty(11.0.24)"},
+       :orig-content-encoding nil,
+       :status 200,
+       :length 779,
+       :body
+       {:updated-at "2025-01-27T18:43:02Z",
+        :hosted-instance-resource-id 7,
+        :last-sync-at nil,
+        :error-detail nil,
+        :type "gdrive",
+        :hosted-instance-resource
+        {:updated-at "2025-01-27T18:43:02Z",
+         :shared-resource-id nil,
+         :resource-id
+         "arn:aws:secretsmanager:us-west-2:373519213707:secret:hosting/development/instance/f390ec19-bd44-48ae-991c-66817182a376/conn-secret/049f3007-2146-4083-be38-f160c526aca7-C6UvyD",
+         :type "conn-secret",
+         :hosted-instance-id "f390ec19-bd44-48ae-991c-66817182a376",
+         :deleted-at nil,
+         :region "us-west-2",
+         :id 7,
+         :created-at "2025-01-27T18:43:02Z",
+         :provider "k8s"},
+        :hosted-instance-id "f390ec19-bd44-48ae-991c-66817182a376",
+        :last-sync-started-at nil,
+        :status "initializing",
+        :id "049f3007-2146-4083-be38-f160c526aca7",
+        :created-at "2025-01-27T18:43:02Z"},
+       :trace-redirects []}]
+
+ {:method :get,
+  :url "/api/v2/mb/connections-google/service-account"
+  :body nil}
+ [:ok {:cached nil,
+       :request-time 12,
+       :repeatable? false,
+       :protocol-version {:name "HTTP", :major 1, :minor 1},
+       :streaming? true,
+       ;; :http-client #object[org.apache.http.impl.client.InternalHttpClient 0x4b1202cb "org.apache.http.impl.client.InternalHttpClient@4b1202cb"],
+       :chunked? false,
+       :reason-phrase "OK",
+       :headers
+       {"Date" "Mon, 27 Jan 2025 18:50:57 GMT",
+        "Connection" "close",
+        "Content-Type" "application/json;charset=utf-8",
+        "X-Frame-Options" "SAMEORIGIN",
+        "X-Content-Type-Options" "nosniff",
+        "Strict-Transport-Security" "max-age=31536000; includeSubDomains",
+        "Content-Length" "60",
+        "Server" "Jetty(11.0.24)"},
+       :orig-content-encoding nil,
+       :status 200,
+       :length 60,
+       :body {:email "service-account@elt-sync.iam.gserviceaccount.com"},
+       :trace-redirects []}]}

test_resources/serialization_baseline/settings.yaml

@@ -22,6 +22,7 @@ enable-pivoted-exports: null
 enable-sandboxes?: null
 enable-whitelabeling?: null
 enable-xrays: null
+gsheets: null
 hide-embed-branding?: null
 humanization-strategy: null
 landing-page: null