Merge branch 'master' into basic-pdf-dashboards-subscriptions-fe

To remove warnings inside `with-*` macros from etaoin.

.clj-kondo/config.edn

@@ -44,6 +44,10 @@
     (clojure.core.logic/run)
     (clojure.test/is [query= =? malli=])
     (clojure.tools.macro/macrolet)
+    (etaoin.api/with-chrome)
+    (etaoin.api/with-chrome-headless)
+    (etaoin.api/with-firefox)
+    (etaoin.api/with-firefox-headless)
     instaparse.core/transform
     (metabase.async.streaming-response/streaming-response)
     (metabase.driver.druid.query-processor-test/druid-query-returning-rows)

Dockerfile

@@ -42,7 +42,10 @@ RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew
     /opt/java/openjdk/bin/keytool -noprompt -import -trustcacerts -alias aws-rds -file /app/certs/rds-combined-ca-bundle.pem -keystore /etc/ssl/certs/java/cacerts -keypass changeit -storepass changeit && \
     curl https://cacerts.digicert.com/DigiCertGlobalRootG2.crt.pem -o /app/certs/DigiCertGlobalRootG2.crt.pem  && \
     /opt/java/openjdk/bin/keytool -noprompt -import -trustcacerts -alias azure-cert -file /app/certs/DigiCertGlobalRootG2.crt.pem -keystore /etc/ssl/certs/java/cacerts -keypass changeit -storepass changeit && \
-    mkdir -p /plugins && chmod a+rwx /plugins
+    mkdir -p /plugins && chmod a+rwx /plugins && \
+    apk add firefox && \
+    wget https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz && \
+    tar -zxf geckodriver-v0.34.0-linux64.tar.gz -C /usr/bin
 
 # add Metabase script and uberjar
 COPY --from=builder /home/node/target/uberjar/metabase.jar /app/

deps.edn

@@ -69,6 +69,7 @@
                                              :exclusions  [org.apache.poi/poi
                                                            org.apache.poi/poi-ooxml]}
   environ/environ                           {:mvn/version "1.2.0"}              ; env vars/Java properties abstraction
+  etaoin/etaoin                             {:mvn/version "1.0.40"}             ; Pure Clojure webdriver
   hiccup/hiccup                             {:mvn/version "1.0.5"}              ; HTML templating
   inflections/inflections                   {:mvn/version "0.14.1"}             ; Clojure/Script library used for prularizing words
   instaparse/instaparse                     {:mvn/version "1.4.12"}             ; Make your own parser

frontend/src/metabase/dashboard/components/SubscriptionsDashboard/SubscriptionsDashboard.tsx

@@ -0,0 +1,36 @@
+import { useEffect, useState } from "react";
+
+import { Box, Text } from "metabase/ui";
+
+import { DashboardGridConnected } from "../DashboardGrid";
+
+// import sampleDashboard from "./sampleDashboard.json";
+
+export const SubscriptionsDashboard = () => {
+  const [dashboardProps, setDashboardProps] = useState(null);
+  useEffect(() => {
+    // @ts-expect-error TODO: add types
+    window.render = (dashboardJson: string) => {
+      const dashboardProps = JSON.parse(dashboardJson);
+      setDashboardProps(dashboardProps);
+    };
+  }, []);
+
+  if (dashboardProps === null) {
+    return <div>Set dashboard using windows.render(dashboard)</div>;
+  }
+
+  return (
+    <Box p={20}>
+      <Text size="xl" fw="bold" mb={24}>
+        {dashboardProps.dashboard.name}
+      </Text>
+      <DashboardGridConnected
+        {...dashboardProps}
+        onUpdateDashCardVisualizationSettings={() => null}
+        onReplaceAllDashCardVisualizationSettings={() => null}
+        slowCards={{}}
+      />
+    </Box>
+  );
+};

frontend/src/metabase/dashboard/components/SubscriptionsDashboard/index.ts

@@ -0,0 +1 @@
+export * from "./SubscriptionsDashboard";

frontend/src/metabase/dashboard/components/SubscriptionsDashboard/sampleDashboard.json

@@ -0,0 +1,426 @@
+{
+  "dashboard": {
+    "description": null,
+    "archived": false,
+    "collection_position": null,
+    "dashcards": [
+      {
+        "size_x": 12,
+        "dashboard_tab_id": null,
+        "series": [],
+        "action_id": null,
+        "collection_authority_level": null,
+        "card": {
+          "description": null,
+          "archived": false,
+          "collection_position": null,
+          "table_id": 5,
+          "result_metadata": [
+            {
+              "description": "The type of product, valid values include: Doohicky, Gadget, Gizmo and Widget",
+              "semantic_type": "type/Category",
+              "coercion_strategy": null,
+              "name": "CATEGORY",
+              "settings": null,
+              "fk_target_field_id": null,
+              "field_ref": [
+                "field",
+                64,
+                {
+                  "base-type": "type/Text",
+                  "source-field": 37
+                }
+              ],
+              "effective_type": "type/Text",
+              "id": 64,
+              "visibility_type": "normal",
+              "display_name": "Product → Category",
+              "fingerprint": {
+                "global": {
+                  "distinct-count": 4,
+                  "nil%": 0
+                },
+                "type": {
+                  "type/Text": {
+                    "percent-json": 0,
+                    "percent-url": 0,
+                    "percent-email": 0,
+                    "percent-state": 0,
+                    "average-length": 6.375
+                  }
+                }
+              },
+              "base_type": "type/Text"
+            },
+            {
+              "display_name": "Count",
+              "semantic_type": "type/Quantity",
+              "field_ref": ["aggregation", 0],
+              "name": "count",
+              "base_type": "type/BigInteger",
+              "effective_type": "type/BigInteger",
+              "fingerprint": {
+                "global": {
+                  "distinct-count": 4,
+                  "nil%": 0
+                },
+                "type": {
+                  "type/Number": {
+                    "min": 3976,
+                    "q1": 4380,
+                    "q3": 5000,
+                    "max": 5061,
+                    "sd": 489.3103990992493,
+                    "avg": 4690
+                  }
+                }
+              }
+            },
+            {
+              "display_name": "Average of Total",
+              "settings": null,
+              "field_ref": ["aggregation", 1],
+              "name": "avg",
+              "base_type": "type/Float",
+              "effective_type": "type/Float",
+              "semantic_type": null,
+              "fingerprint": {
+                "global": {
+                  "distinct-count": 4,
+                  "nil%": 0
+                },
+                "type": {
+                  "type/Number": {
+                    "min": 74.76634648349606,
+                    "q1": 77.41891176132381,
+                    "q3": 83.02043466215214,
+                    "max": 83.71221052631937,
+                    "sd": 3.933038333878259,
+                    "avg": 80.21967321173798
+                  }
+                }
+              }
+            }
+          ],
+          "initially_published_at": null,
+          "can_write": true,
+          "database_id": 1,
+          "enable_embedding": false,
+          "collection_id": null,
+          "query_type": "query",
+          "name": "barrr",
+          "type": "question",
+          "query_average_duration": 68,
+          "creator_id": 1,
+          "moderation_reviews": [],
+          "updated_at": "2024-02-29T20:49:16.336361-03:00",
+          "made_public_by_id": null,
+          "embedding_params": null,
+          "cache_ttl": null,
+          "dataset_query": {
+            "database": 1,
+            "type": "query",
+            "query": {
+              "source-table": 5,
+              "aggregation": [
+                ["count"],
+                [
+                  "avg",
+                  [
+                    "field",
+                    39,
+                    {
+                      "base-type": "type/Float"
+                    }
+                  ]
+                ]
+              ],
+              "breakout": [
+                [
+                  "field",
+                  64,
+                  {
+                    "base-type": "type/Text",
+                    "source-field": 37
+                  }
+                ]
+              ]
+            }
+          },
+          "id": 122,
+          "parameter_mappings": [],
+          "display": "bar",
+          "entity_id": "JnWepXcIOGNjeNGDC0MkN",
+          "collection_preview": true,
+          "visualization_settings": {
+            "graph.dimensions": ["CATEGORY"],
+            "graph.metrics": ["count", "avg"]
+          },
+          "metabase_version": "v1.48.1-SNAPSHOT (6934865)",
+          "parameters": [],
+          "created_at": "2024-01-04T20:40:45.860872-03:00",
+          "public_uuid": null
+        },
+        "updated_at": "2024-02-29T20:49:20.740893-03:00",
+        "col": 0,
+        "id": 285,
+        "parameter_mappings": [],
+        "card_id": 122,
+        "entity_id": "lPqpdd5gQd3zfpGDX4bIl",
+        "visualization_settings": {},
+        "size_y": 6,
+        "dashboard_id": 1,
+        "created_at": "2024-02-29T20:49:20.740893-03:00",
+        "row": 0
+      }
+    ],
+    "param_values": null,
+    "initially_published_at": null,
+    "can_write": true,
+    "tabs": [],
+    "enable_embedding": false,
+    "collection_id": null,
+    "show_in_getting_started": false,
+    "name": "123",
+    "width": "full",
+    "caveats": null,
+    "collection_authority_level": null,
+    "creator_id": 1,
+    "updated_at": "2024-02-29T20:49:20.740893-03:00",
+    "made_public_by_id": null,
+    "embedding_params": null,
+    "cache_ttl": null,
+    "id": 1,
+    "position": null,
+    "entity_id": "NHn2mW5B1YEgGx5YC6oF5",
+    "param_fields": null,
+    "parameters": [],
+    "auto_apply_filters": true,
+    "created_at": "2024-01-02T13:56:12.752164-03:00",
+    "public_uuid": null,
+    "points_of_interest": null
+  },
+  "dashcardData": {
+    "285": {
+      "122": {
+        "data": {
+          "rows": [
+            ["Doohickey", 3976, 74.76634648349606],
+            ["Gadget", 4939, 82.32865879798491],
+            ["Gizmo", 4784, 80.07147703915157],
+            ["Widget", 5061, 83.71221052631937]
+          ],
+          "cols": [
+            {
+              "description": "The type of product, valid values include: Doohicky, Gadget, Gizmo and Widget",
+              "semantic_type": "type/Category",
+              "table_id": 8,
+              "coercion_strategy": null,
+              "name": "CATEGORY",
+              "settings": null,
+              "source": "breakout",
+              "fk_target_field_id": null,
+              "fk_field_id": 37,
+              "field_ref": [
+                "field",
+                64,
+                {
+                  "base-type": "type/Text",
+                  "source-field": 37
+                }
+              ],
+              "effective_type": "type/Text",
+              "nfc_path": null,
+              "parent_id": null,
+              "id": 64,
+              "position": 3,
+              "visibility_type": "normal",
+              "display_name": "Product → Category",
+              "fingerprint": {
+                "global": {
+                  "distinct-count": 4,
+                  "nil%": 0
+                },
+                "type": {
+                  "type/Text": {
+                    "percent-json": 0,
+                    "percent-url": 0,
+                    "percent-email": 0,
+                    "percent-state": 0,
+                    "average-length": 6.375
+                  }
+                }
+              },
+              "base_type": "type/Text",
+              "source_alias": "PRODUCTS__via__PRODUCT_ID"
+            },
+            {
+              "base_type": "type/BigInteger",
+              "name": "count",
+              "display_name": "Count",
+              "semantic_type": "type/Quantity",
+              "source": "aggregation",
+              "field_ref": ["aggregation", 0],
+              "aggregation_index": 0,
+              "effective_type": "type/BigInteger"
+            },
+            {
+              "semantic_type": null,
+              "name": "avg",
+              "settings": null,
+              "source": "aggregation",
+              "field_ref": ["aggregation", 1],
+              "effective_type": "type/Float",
+              "aggregation_index": 1,
+              "display_name": "Average of Total",
+              "base_type": "type/Float"
+            }
+          ],
+          "native_form": {
+            "query": "SELECT \"PRODUCTS__via__PRODUCT_ID\".\"CATEGORY\" AS \"PRODUCTS__via__PRODUCT_ID__CATEGORY\", COUNT(*) AS \"count\", AVG(\"PUBLIC\".\"ORDERS\".\"TOTAL\") AS \"avg\" FROM \"PUBLIC\".\"ORDERS\" LEFT JOIN \"PUBLIC\".\"PRODUCTS\" AS \"PRODUCTS__via__PRODUCT_ID\" ON \"PUBLIC\".\"ORDERS\".\"PRODUCT_ID\" = \"PRODUCTS__via__PRODUCT_ID\".\"ID\" GROUP BY \"PRODUCTS__via__PRODUCT_ID\".\"CATEGORY\" ORDER BY \"PRODUCTS__via__PRODUCT_ID\".\"CATEGORY\" ASC",
+            "params": null
+          },
+          "results_timezone": "America/Montevideo",
+          "results_metadata": {
+            "columns": [
+              {
+                "description": "The type of product, valid values include: Doohicky, Gadget, Gizmo and Widget",
+                "semantic_type": "type/Category",
+                "coercion_strategy": null,
+                "name": "CATEGORY",
+                "settings": null,
+                "fk_target_field_id": null,
+                "field_ref": [
+                  "field",
+                  64,
+                  {
+                    "base-type": "type/Text",
+                    "source-field": 37
+                  }
+                ],
+                "effective_type": "type/Text",
+                "id": 64,
+                "visibility_type": "normal",
+                "display_name": "Product → Category",
+                "fingerprint": {
+                  "global": {
+                    "distinct-count": 4,
+                    "nil%": 0
+                  },
+                  "type": {
+                    "type/Text": {
+                      "percent-json": 0,
+                      "percent-url": 0,
+                      "percent-email": 0,
+                      "percent-state": 0,
+                      "average-length": 6.375
+                    }
+                  }
+                },
+                "base_type": "type/Text"
+              },
+              {
+                "display_name": "Count",
+                "semantic_type": "type/Quantity",
+                "field_ref": ["aggregation", 0],
+                "name": "count",
+                "base_type": "type/BigInteger",
+                "effective_type": "type/BigInteger",
+                "fingerprint": {
+                  "global": {
+                    "distinct-count": 4,
+                    "nil%": 0
+                  },
+                  "type": {
+                    "type/Number": {
+                      "min": 3976,
+                      "q1": 4380,
+                      "q3": 5000,
+                      "max": 5061,
+                      "sd": 489.3103990992493,
+                      "avg": 4690
+                    }
+                  }
+                }
+              },
+              {
+                "display_name": "Average of Total",
+                "semantic_type": null,
+                "settings": null,
+                "field_ref": ["aggregation", 1],
+                "name": "avg",
+                "base_type": "type/Float",
+                "effective_type": "type/Float",
+                "fingerprint": {
+                  "global": {
+                    "distinct-count": 4,
+                    "nil%": 0
+                  },
+                  "type": {
+                    "type/Number": {
+                      "min": 74.76634648349606,
+                      "q1": 77.41891176132381,
+                      "q3": 83.02043466215214,
+                      "max": 83.71221052631937,
+                      "sd": 3.933038333878259,
+                      "avg": 80.21967321173798
+                    }
+                  }
+                }
+              }
+            ]
+          },
+          "insights": null
+        },
+        "cached": false,
+        "database_id": 1,
+        "started_at": "2024-02-29T20:49:16.271568-03:00",
+        "json_query": {
+          "constraints": {
+            "max-results": 10000,
+            "max-results-bare-rows": 2000
+          },
+          "type": "query",
+          "middleware": {
+            "js-int-to-string?": true,
+            "ignore-cached-results?": false,
+            "process-viz-settings?": false,
+            "userland-query?": true
+          },
+          "database": 1,
+          "query": {
+            "source-table": 5,
+            "aggregation": [
+              ["count"],
+              [
+                "avg",
+                [
+                  "field",
+                  39,
+                  {
+                    "base-type": "type/Float"
+                  }
+                ]
+              ]
+            ],
+            "breakout": [
+              [
+                "field",
+                64,
+                {
+                  "base-type": "type/Text",
+                  "source-field": 37
+                }
+              ]
+            ]
+          }
+        },
+        "average_execution_time": null,
+        "status": "completed",
+        "context": "question",
+        "row_count": 4,
+        "running_time": 68
+      }
+    }
+  }
+}

frontend/src/metabase/routes.jsx

@@ -61,6 +61,7 @@ import { Setup } from "metabase/setup/components/Setup";
 import getCollectionTimelineRoutes from "metabase/timelines/collections/routes";
 
 import { BrowseRedirect } from "./browse/components/BrowseRedirect";
+import { SubscriptionsDashboard } from "./dashboard/components/SubscriptionsDashboard";
 import {
   CanAccessMetabot,
   CanAccessSettings,
@@ -89,6 +90,11 @@ export const getRoutes = store => {
         }}
       />
 
+      <Route
+        path="/subscriptions_dashboard"
+        component={SubscriptionsDashboard}
+      />
+
       {/* PUBLICLY SHARED LINKS */}
       <Route path="public">
         <Route path="question/:uuid" component={PublicQuestion} />

src/metabase/email/messages.clj

@@ -6,6 +6,7 @@
    [cheshire.core :as json]
    [clojure.core.cache :as cache]
    [clojure.java.io :as io]
+   [etaoin.api :as e]
    [hiccup.core :refer [html]]
    [java-time.api :as t]
    [medley.core :as m]
@@ -27,6 +28,7 @@
    [metabase.pulse.render.image-bundle :as image-bundle]
    [metabase.pulse.render.js-svg :as js-svg]
    [metabase.pulse.render.style :as style]
+   [metabase.query-processor :as qp]
    [metabase.query-processor.store :as qp.store]
    [metabase.query-processor.streaming :as qp.streaming]
    [metabase.query-processor.streaming.interface :as qp.si]
@@ -495,8 +497,45 @@
       (for [row rows]
         [:tr {} row])])))
 
+(defn- render-dashboard-as-pdf
+  "Using a webdriver, render a dashboard as a pdf. Returns nil if it fails."
+  ^File [dashboard-with-data]
+  (try
+    (let [dash-file (doto
+                      (File/createTempFile "dashboard" ".pdf")
+                      .deleteOnExit)]
+      (e/with-firefox-headless {}
+        driver
+        (e/go driver (format "%s/subscriptions_dashboard" (public-settings/site-url)))
+        (e/js-execute driver
+                      (format "window.render(JSON.stringify(%s))"
+                              (json/generate-string dashboard-with-data)))
+        ;; We should check for element visibility here, but for now we can
+        ;; just wait a second and let everything render.
+        (e/wait driver 1)
+        (e/print-page driver dash-file))
+      dash-file)
+    (catch Exception e
+      (log/warnf "Could not render pdf dashboard. Error: %s" (.getMessage e)))))
+
+(defn- dashboard-with-data
+  "Hydrate a dashboard and evaluate each dashcard in the dashboard."
+  [dashboard-id]
+  (let [{:keys [dashcards] :as dashboard} (t2/hydrate
+                                            (t2/select-one :model/Dashboard dashboard-id)
+                                            [:dashcards
+                                             :card])
+        query-cards (filter :card dashcards)]
+    {:dashboard    dashboard
+     :dashcardData (zipmap
+                     (map :id query-cards)
+                     (map (fn [{{card-id       :id
+                                 dataset-query :dataset_query} :card}]
+                            {card-id (qp/process-query dataset-query)})
+                          query-cards))}))
+
 (defn- render-message-body
-  [notification message-type message-context timezone dashboard parts]
+  [notification message-type message-context timezone {dashboard-id :id :as dashboard} parts]
   (let [rendered-cards  (binding [render/*include-title* true]
                           (mapv #(render-part timezone %) parts))
         icon-name       (case message-type
@@ -506,10 +545,18 @@
         filters         (when dashboard
                           (render-filters notification dashboard))
         message-body    (assoc message-context :pulse (html (vec (cons :div (map :content rendered-cards))))
-                               :filters filters
-                               :iconCid (:content-id icon-attachment))
+                                               :filters filters
+                                               :iconCid (:content-id icon-attachment))
+        pdf-export      (render-dashboard-as-pdf (dashboard-with-data dashboard-id))
         attachments     (apply merge (map :attachments rendered-cards))]
-    (vec (concat [{:type "text/html; charset=utf-8" :content (stencil/render-file "metabase/email/pulse" message-body)}]
+    (vec (concat (cond-> [{:type    "text/html; charset=utf-8"
+                           :content (stencil/render-file "metabase/email/pulse" message-body)}]
+                   pdf-export
+                   (conj {:type         :inline
+                          :content-id   (gensym "metabase")
+                          :content-type "application/pdf"
+                          :content      (-> pdf-export .toURI .toURL)
+                          :description  "Dashboard exported as PDF"}))
                  (map make-message-attachment attachments)
                  [icon-attachment]
                  (part-attachments parts)))))