habemus button

.clj-kondo/config/modules/config.edn

@@ -877,6 +877,7 @@
            config
            content-verification
            dashboards
+           driver
            embedding
            events
            legacy-mbql
@@ -893,6 +894,7 @@
            request
            revisions
            search
+           sync
            util
            view-log
            warehouse-schema}}

frontend/src/metabase-lib/v1/Question.ts

@@ -227,6 +227,10 @@ class Question {
     return this.setCard(assoc(this.card(), "type", type));
   }
 
+  setTransform(t: bool) {
+    return this.setCard(assoc(this.card(), "transform", t));
+  }
+
   isPersisted() {
     return this._card && this._card.persisted;
   }

frontend/src/metabase-types/api/card.ts

@@ -25,7 +25,8 @@ import type { UserInfo } from "./user";
 import type { CardDisplayType, VisualizationDisplay } from "./visualization";
 import type { SmartScalarComparison } from "./visualization-settings";
 
-export type CardType = "model" | "question" | "metric";
+// wip
+export type CardType = "model" | "question" | "metric" ;
 
 type CreatorInfo = Pick<
   UserInfo,
@@ -43,6 +44,8 @@ export interface Card<Q extends DatasetQuery = DatasetQuery>
   type: CardType;
   public_uuid: string | null;
 
+  transform: bool | null;
+
   /* Indicates whether static embedding for this card has been published */
   enable_embedding: boolean;
   embedding_params: EmbeddingParameters | null;
@@ -380,6 +383,7 @@ export interface CreateCardRequest {
   collection_position?: number;
   result_metadata?: Field[];
   cache_ttl?: number;
+  transform?: bool;
 }
 
 export interface CreateCardFromCsvRequest {
@@ -404,6 +408,7 @@ export interface UpdateCardRequest {
   collection_position?: number;
   result_metadata?: Field[];
   cache_ttl?: number;
+  transform?: bool;
   collection_preview?: boolean;
   delete_old_dashcards?: boolean;
 }

frontend/src/metabase/entities/questions.js

@@ -244,6 +244,7 @@ const Questions = createEntity({
     "name",
     "cache_ttl",
     "type",
+    "transform",
     "dataset_query",
     "display",
     "description",

frontend/src/metabase/query_builder/actions/models.ts

@@ -31,7 +31,9 @@ export const turnQuestionIntoModel =
     }
 
     const model = question
-      .setType("model")
+        // .setType("model")
+        .setTransform(true)
+      // .setType("transform")
       .setPinned(true)
       .setDisplay("table")
       .setSettings({});
@@ -39,7 +41,7 @@ export const turnQuestionIntoModel =
 
     dispatch(
       addUndo({
-        message: t`This is a model now.`,
+        message: t`This is a transform now.`,
         actions: [apiUpdateQuestion(question, { rerunQuery: true })],
       }),
     );

frontend/src/metabase/query_builder/components/NewDatasetModal/NewDatasetModal.tsx

@@ -15,10 +15,10 @@ export function NewDatasetModal({ onClose }: { onClose: () => void }) {
   const [, { ack }] = useUserAcknowledgement("turn_into_model_modal");
 
   const onConfirm = useCallback(() => {
-    ack();
+    //ack();
     dispatch(turnQuestionIntoModel());
     onClose();
-  }, [dispatch, ack, onClose]);
+  }, [dispatch, onClose]);
 
   return (
     <ModalContent
@@ -32,27 +32,9 @@ export function NewDatasetModal({ onClose }: { onClose: () => void }) {
           key="action"
           variant="filled"
           onClick={onConfirm}
-        >{t`Turn this into a model`}</Button>,
+        >{t`Turn this into a transform`}</Button>,
       ]}
     >
-      <Box p="2rem 1rem 0">
-        <Box component="img" pt="md" src="app/img/model-illustration.svg" />
-        <Text component="h2" mt="2rem" mb="md">{t`Models`}</Text>
-        <ul>
-          <li className={NewDatasetModalS.DatasetValueProp}>
-            {t`Let you update column descriptions and customize metadata to create
-            great starting points for exploration.`}
-          </li>
-          <li className={NewDatasetModalS.DatasetValueProp}>
-            {t`Show up higher in search results and get highlighted when other
-            users start new questions to promote reuse.`}
-          </li>
-          <li className={NewDatasetModalS.DatasetValueProp}>
-            {t`Live in collections to keep them separate from messy database
-            schemas.`}
-          </li>
-        </ul>
-      </Box>
     </ModalContent>
   );
 }

frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx

@@ -150,7 +150,7 @@ export const QuestionMoreActionsMenu = ({
         data-testid={TURN_INTO_DATASET_TESTID}
         onClick={handleTurnToModel}
       >
-        {t`Turn into a model`}
+        {t`Turn into a transform`}
       </Menu.Item>
     ),
     hasCollectionPermissions && isModel && (

poc.md

@@ -0,0 +1,15 @@
+# things to do:
+
+## connect `turn-card-into-transform` with the endpoint
+## connect frontend to endpoint (adapt turn into model button)
+
+
+## assess what happens when you try to visualise the transform query
+
+## transform table
+
+## api endpoint to read from transform table
+
+# open questions
+## how do we want to return transform + model info
+## design: how do we render the transform? do we want to "transparently" hide the backing view? c.f. "sql for this transform" + query editor

resources/migrations/056_update_migrations.yaml

@@ -169,6 +169,105 @@ databaseChangeLog:
                   type: ${text.type}
                   constraints:
                     nullable: false
+
+  - changeSet:
+      id: v56.2025-07-02T11:52:19
+      author: lbrdnk
+      comment: Create table for transforms
+      changes:
+        - createTable:
+            tableName: report_card_transform
+            remarks: xxx
+            columns:
+              - column:
+                  name: id
+                  type: int
+                  autoIncrement: true
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: card_id
+                  type: int
+                  remarks: The related card
+                  constraints:
+                    nullable: false
+                    referencedTableName: report_card
+                    referencedColumnNames: id
+                    foreignKeyName: fk_transform_ref_card_id
+                    deleteCascade: true
+              # TODO: does it make sense to store report_card's result_metadata?
+              - column:
+                  name: original_dataset_query
+                  remarks: the dataset_query that is source of cards transform
+                  # TODO: Verif
+                  type: ${text.type}
+      # on rollback: all the original query definitions should be written back to the report_card
+      rollback:
+        # Backfill the original report_card dataset queries
+        - sql:
+            dbms: postgresql
+            sql: >-
+              update report_card
+              set dataset_query = rct.odq
+              from (
+                  select asdf.card_id, asdf.odq
+                  from (
+                      select
+                          card_id,
+                          LAST_VALUE(original_dataset_query) 
+                          OVER (
+                              PARTITION BY card_id
+                              ORDER BY id
+                          ) odq
+                      from report_card_transform
+                  ) asdf
+                  group by asdf.card_id, asdf.odq
+              ) as rct
+              where rct.card_id = report_card.id
+        - sql:
+            dbms: mysql,mariadb
+            sql: >-
+              update 
+                report_card rc, (
+                  select cid, odq 
+                  from (
+                    select 
+                      card_id cid, 
+                      first_value(original_dataset_query) 
+                        over (partition by card_id order by id desc) odq 
+                    from report_card_transform
+                  ) yiy 
+                  group by cid, odq
+                ) xix
+              set rc.dataset_query = xix.odq 
+              where rc.id = xix.cid
+        - sql:
+            dbms: h2
+            sql: >-
+              with rct as (
+                select card_id, odq
+                from (
+                  select
+                    card_id,
+                    FIRST_VALUE (original_dataset_query)
+                      OVER (PARTITION BY card_id
+                            ORDER BY id DESC)
+                      odq
+                  from report_card_transform
+                )
+                group by card_id, odq
+              )
+              update report_card
+              set dataset_query = (select odq from rct where report_card.id = rct.card_id)
+              where report_card.id = (select card_id from rct where report_card.id = card_id)
+        # drop the table
+        - sql:
+            #dbms: h2,msyql,postgresql
+            sql: >-
+              DROP table report_card_transform
+
+
 # >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
 
 ########################################################################################################################

src/metabase/collections/api.clj

@@ -232,6 +232,7 @@
   `no_models` is for nilling out the set because a nil model set is actually the total model set"
   #{"card"       ; SavedQuestion
     "dataset"    ; Model. TODO : update this
+    "transform"
     "metric"
     "collection"
     "dashboard"
@@ -402,6 +403,7 @@
                     [(h2x/literal (case card-type
                                     :model "dataset"
                                     :metric  "metric"
+                                    :transform "transform"
                                     "card"))
                      :model]
                     [:u.id :last_edit_user]
@@ -410,9 +412,11 @@
                     [:u.last_name :last_edit_last_name]
                     [:r.timestamp :last_edit_timestamp]
                     [:mr.status :moderated_status]]
-                    (#{:question :model} card-type)
+
+                    (#{:question :model :transform} card-type)
                     (conj :c.database_id))
        :from      [[:report_card :c]]
+       ;; TODO: audit
        :left-join [[:revision :r] [:and
                                    [:= :r.model_id :c.id]
                                    [:= :r.most_recent true]
@@ -439,7 +443,11 @@
                      :metric
                      [:= :c.type (h2x/literal "metric")]
 
+                     :transform
+                     [:= :c.type (h2x/literal "transform")]
+
                      [:= :c.type (h2x/literal "question")])]}
+      ;; TODO: audit
       (cond-> (= :model card-type)
         (-> (sql.helpers/select :c.table_id :t.is_upload :c.query_type)
             (sql.helpers/left-join [:metabase_table :t] [:= :t.id :c.table_id])))
@@ -457,6 +465,10 @@
   [_ collection options]
   (card-query :question collection options))
 
+(defmethod collection-children-query :transform
+  [_ collection options]
+  (card-query :transform collection options))
+
 (defmethod post-process-collection-children :dataset
   [_ options collection rows]
   (let [queries-before (map :dataset_query rows)
@@ -761,6 +773,7 @@
     :card       :model/Card
     :dataset    :model/Card
     :metric     :model/Card
+    :transform  :model/Card
     :dashboard  :model/Dashboard
     :pulse      :model/Pulse
     :snippet    :model/NativeQuerySnippet
@@ -950,7 +963,7 @@
   "Fetch a sequence of 'child' objects belonging to a Collection, filtered using `options`."
   [{collection-namespace :namespace, :as collection} :- collection/CollectionWithLocationAndIDOrRoot
    {:keys [models], :as options}                     :- CollectionChildrenOptions]
-  (let [valid-models (for [model-kw [:collection :dataset :metric :card :dashboard :pulse :snippet :timeline]
+  (let [valid-models (for [model-kw [:collection :transform :dataset :metric :card :dashboard :pulse :snippet :timeline]
                            ;; only fetch models that are specified by the `model` param; or everything if it's empty
                            :when    (or (empty? models) (contains? models model-kw))
                            :let     [toucan-model       (model-name->toucan-model model-kw)
@@ -991,6 +1004,8 @@
   []
   (collection-detail (api/read-check (collection/trash-collection))))
 
+ ;;models=dashboard&models=dataset&models=card&models=metric&models=snippet&models=collection&pinned_state=is_not_pinned&limit=25&offset=0&sort_column=name&sort_direction=asc
+
 (api.macros/defendpoint :get "/:id/items"
   "Fetch a specific Collection's items with the following options:
 
@@ -1013,7 +1028,7 @@
                                           [:sort_direction             {:optional true} [:maybe (into [:enum] valid-sort-directions)]]
                                           [:official_collections_first {:optional true} [:maybe ms/MaybeBooleanValue]]
                                           [:show_dashboard_questions   {:default false} [:maybe ms/BooleanValue]]]]
-  (let [model-kwds (set (map keyword (u/one-or-many models)))
+  (let [model-kwds (conj (set (map keyword (u/one-or-many models))) :transform)
         collection (api/read-check :model/Collection id)]
     (u/prog1 (collection-children collection
                                   {:show-dashboard-questions? show_dashboard_questions

src/metabase/queries/api/card.clj

@@ -1,12 +1,15 @@
 (ns metabase.queries.api.card
   "/api/card endpoints."
   (:require
+   #_[metabase.sync.sync-metadata :as sync-metadata]
+   [clojure.java.jdbc :as jdbc]
    [medley.core :as m]
    [metabase.analyze.core :as analyze]
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
    [metabase.collections.models.collection :as collection]
    [metabase.collections.models.collection.root :as collection.root]
+   [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
    [metabase.embedding.validation :as embedding.validation]
    [metabase.events.core :as events]
    [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
@@ -25,11 +28,13 @@
    [metabase.query-permissions.core :as query-perms]
    [metabase.query-processor.api :as api.dataset]
    [metabase.query-processor.card :as qp.card]
+   [metabase.query-processor.compile :as qp.compile]
    [metabase.query-processor.pivot :as qp.pivot]
    [metabase.query-processor.schema :as qp.schema]
    [metabase.request.core :as request]
    [metabase.revisions.core :as revisions]
    [metabase.search.core :as search]
+   [metabase.sync.core :as sync]
    [metabase.util :as u]
    [metabase.util.i18n :refer [deferred-tru trs tru]]
    [metabase.util.json :as json]
@@ -38,7 +43,8 @@
    [metabase.util.malli.schema :as ms]
    [ring.util.codec :as codec]
    [steffan-westcott.clj-otel.api.trace.span :as span]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import (java.sql Connection)))
 
 (set! *warn-on-reflection* true)
 
@@ -536,6 +542,7 @@
 
 (def ^:private CardUpdateSchema
   [:map
+   [:transform              {:optional true} [:maybe :boolean]]
    [:name                   {:optional true} [:maybe ms/NonBlankString]]
    [:parameters             {:optional true} [:maybe [:sequential ::parameters.schema/parameter]]]
    [:dataset_query          {:optional true} [:maybe ms/Map]]
@@ -567,6 +574,78 @@
           (api/column-will-change? :collection_id card-before-update card-updates))
       (assoc :collection_id collection-id))))
 
+;; TODO: will need to make sure it is unique, else we risk overriding
+;; existing view that are not equal. if uniqueness can't be guaranteed by
+;; hashing, we may need to store view definitions (we'll most likely need
+;; that anyway) and compare
+(defn- generate-view-name
+  ([]
+   (generate-view-name (System/currentTimeMillis)))
+  ([x]
+   (str "mb_view_" x)))
+
+;; TODO: we likely don't want CREATE OR REPLACE, and always generate a new name
+
+(defn- generate-view-definition [name query]
+  (format "CREATE OR REPLACE VIEW \"%s\" AS\n\t(%s)" name query))
+
+(defn- insert-transform
+  [{card-id :id original-dataset-query :dataset_query :as _card}]
+  (t2/insert-returning-instance! :model/Transform {:card_id card-id
+                                                   :original_dataset_query original-dataset-query}))
+
+;; TODO: We should avoid creating view from query that is view based already. Instead create view from modification
+;;       of its definition that resides in report_card_transform or :model/Transform.
+(defn create-view-for-card
+  "create view for card"
+  [driver db {:keys [dataset_query] :as card}]
+  (t2/with-transaction [_conn]
+    (let [{transform-id :id :as _transform} (insert-transform card)
+          {:keys [query]} (qp.compile/compile dataset_query)
+          view-name (generate-view-name transform-id)
+          view-definition (generate-view-definition view-name query)]
+      (log/fatalf "Transform view:\n\n%s\n"
+                  view-definition)
+      (sql-jdbc.execute/do-with-connection-with-options
+       driver db {:write? true}
+       (fn [^Connection conn]
+         #_(throw (Exception. "verif transact Transform model"))
+         (jdbc/execute! {:connection conn} view-definition)
+         view-name)))))
+
+(defn- transform-updates [{:keys [database_id] :as card}]
+  (assert (empty? (:parameters card)))
+  (let [db (t2/select-one :model/Database :id database_id)
+        driver (:engine db)
+        new-table-name (create-view-for-card driver db card)
+        ;; ?analysis
+        ;; _ (sync-metadata/sync-db-metadata! db)
+        _ (sync/sync-database! db)
+        table (t2/select-one :model/Table :name new-table-name)]
+    ;; the FE doesn't know what to do with :type :transform, so we just pretend it's still a question for the purposes of this demo
+    {;; :type :transform
+     :type :question
+     :table_id (:id table)
+     :dataset_query
+     {:database (:id db),
+      :type :query,
+      :query {:source-table (:id table)}}}))
+
+;;; (def card (t2/select-one :model/Card 115))
+;;; (turn-card-into-transform card)
+
+#_(comment
+
+    (def cu (toucan2.core/select-one :model/Card :name "q3 orders"))
+
+    (metabase.request.session/with-current-user 1
+      (update-card! 114 {:type :transform} false))
+
+    (metabase.request.session/with-current-user 1
+      (update-card! 116 {:type :transform} false))
+
+    (toucan2.core/update! :model/Card :id 116 {:type "question"}))
+
 (mu/defn update-card!
   "Updates a card - impl"
   [id :- ms/PositiveInt
@@ -574,6 +653,7 @@
            result_metadata
            type] :as card-updates} :- CardUpdateSchema
    delete-old-dashcards? :- :boolean]
+  ;; if type is transform, then let's compile it into a view
   (check-if-card-can-be-saved dataset_query type)
   (when-some [query (dataset-query->query dataset_query)]
     (try
@@ -587,7 +667,8 @@
                                 (api/updates-with-archived-directly card-before-update card-updates))
         is-model-after-update? (if (nil? type)
                                  (card/model? card-before-update)
-                                 (card/model? card-updates))]
+                                 (card/model? card-updates))
+        is-transform-after-update? (:transform card-updates)]
     ;; Do various permissions checks
     (doseq [f [check-allowed-to-move
                check-allowed-to-modify-query
@@ -617,6 +698,11 @@
                                                metadata
                                                (assoc :result_metadata           metadata
                                                       :verified-result-metadata? true))
+
+          card-updates                       (cond-> card-updates
+                                               is-transform-after-update?
+                                               (merge (transform-updates card-before-update)))
+
           card                               (-> (card/update-card! {:card-before-update    card-before-update
                                                                      :card-updates          card-updates
                                                                      :actor                 @api/*current-user*

src/metabase/queries/core.clj

@@ -6,6 +6,7 @@
    [metabase.queries.models.card.metadata]
    [metabase.queries.models.parameter-card]
    [metabase.queries.models.query]
+   [metabase.queries.models.transform]
    [potemkin :as p]))
 
 (comment metabase.queries.card/keep-me

src/metabase/queries/models/card.clj

@@ -131,6 +131,11 @@
   [card]
   (= (keyword (:type card)) :model))
 
+(defn transform?
+  "Returns true if `card` is a transform."
+  [card]
+  (= (keyword (:type card)) :transform))
+
 (defn lib-query
   "Given a card with at least its `:dataset_query` field, this returns the `metabase.lib` form of the query.
 

src/metabase/queries/models/transform.clj

@@ -0,0 +1,16 @@
+(ns metabase.queries.models.transform
+  (:require
+   [metabase.models.interface :as mi]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/Transform [_model] :report_card_transform)
+
+(t2/deftransforms :model/Transform
+  {:original_dataset_query mi/transform-metabase-query})
+
+(doto :model/Transform
+  (derive :metabase/model))
+
+(methodical/defmethod t2/primary-keys :model/Transform [_model] [:id])
+

src/metabase/queries/schema.clj

@@ -12,7 +12,7 @@
 
   Migrating all the code to use `report_card.type` will be quite an effort, we decided that we'll migrate it
   gradually."
-  #{:model :question :metric})
+  #{:model :question :metric :transform})
 
 (mr/def ::card-type
   (into [:enum {:decode/json keyword