Renamed to semantic layer

.clj-kondo/config/modules/config.edn

@@ -214,6 +214,7 @@
    :api  #{metabase.bookmarks.api}
    :uses #{api
            app-db
+           collections
            config
            premium-features
            queries
@@ -1855,6 +1856,13 @@
    :uses #{premium-features
            settings}}
 
+  enterprise/semantic-layer
+  {:team "UX West"
+   :api  #{metabase-enterprise.semantic-layer.api}
+   :uses #{api
+           collections
+           premium-features}}
+
   enterprise/llm
   {:team "Metabot"
    :api  #{metabase-enterprise.llm.api

enterprise/backend/src/metabase_enterprise/api_routes/routes.clj

@@ -33,6 +33,7 @@
    [metabase-enterprise.remote-sync.api]
    [metabase-enterprise.sandbox.api.routes]
    [metabase-enterprise.scim.routes]
+   [metabase-enterprise.semantic-layer.api]
    [metabase-enterprise.semantic-search.api]
    [metabase-enterprise.serialization.api]
    [metabase-enterprise.stale.api]
@@ -62,6 +63,7 @@
    :remote-sync                (deferred-tru "Remote Sync")
    :etl-connections            (deferred-tru "ETL Connections")
    :etl-connections-pg         (deferred-tru "ETL Connections PG replication")
+   :semantic-layer             (deferred-tru "Semantic Layer")
    :llm-autodescription        (deferred-tru "LLM Auto-description")
    :metabot-v3                 (deferred-tru "MetaBot")
    :scim                       (deferred-tru "SCIM configuration")
@@ -118,6 +120,7 @@
    "/gsheets"                      (-> gsheets.api/routes ;; gsheets requires both features.
                                        (premium-handler :attached-dwh)
                                        (premium-handler :etl-connections))
+   "/semantic-layer"               (premium-handler metabase-enterprise.semantic-layer.api/routes :semantic-layer)
    "/logs"                         (premium-handler 'metabase-enterprise.advanced-config.api.logs :audit-app)
    "/metabot-tools"                metabase-enterprise.metabot-v3.tools.api/routes
    "/metabot-v3"                   (premium-handler metabase-enterprise.metabot-v3.api/routes :metabot-v3)

enterprise/backend/src/metabase_enterprise/documents/models/document.clj

@@ -3,6 +3,7 @@
    [clojure.string :as str]
    [metabase-enterprise.documents.prose-mirror :as prose-mirror]
    [metabase.api.common :as api]
+   [metabase.collections.models.collection :as collection]
    [metabase.models.interface :as mi]
    [metabase.models.serialization :as serdes]
    [metabase.public-sharing.core :as public-sharing]
@@ -228,3 +229,11 @@
                                                                                  (:attrs %)))
                    :when (contains? model->serdes-model model)]
                {[(model->serdes-model model) link-id] {"Document" id}}))))))
+
+(t2/define-before-insert :model/Document [model]
+  (collection/check-allowed-content :document (:collection_id model))
+  model)
+
+(t2/define-before-update :model/Document [model]
+  (collection/check-allowed-content :document (:collection_id (t2/changes model)))
+  model)

enterprise/backend/src/metabase_enterprise/semantic_layer/api.clj

@@ -0,0 +1,23 @@
+(ns metabase-enterprise.semantic-layer.api
+  (:require
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.collections.core :as collections]))
+
+(set! *warn-on-reflection* true)
+
+(api.macros/defendpoint :post "/create"
+  "Creates the Semantic Layer if it doesn't exist. Returns the created collection.
+
+  Requires superuser permissions."
+  [_route
+   _query
+   _body]
+  (api/check-superuser)
+  (api/check-400 (not (collections/semantic-layer-collection)) "Semantic Layer already exists")
+  (collections/create-semantic-layer-collection!))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/semantic-layer` routes."
+  (api.macros/ns-handler *ns* +auth))

enterprise/backend/src/metabase_enterprise/semantic_layer/core.clj

@@ -0,0 +1,11 @@
+(ns metabase-enterprise.semantic-layer.core
+  (:require
+   [metabase-enterprise.semantic-layer.validation :as validation]
+   [potemkin :as p]))
+
+(comment
+  validation/keep-me)
+
+(p/import-vars
+ [metabase-enterprise.semantic-layer.validation
+  check-allowed-content])

enterprise/backend/src/metabase_enterprise/semantic_layer/validation.clj

@@ -0,0 +1,17 @@
+(ns metabase-enterprise.semantic-layer.validation
+  (:require
+   [metabase.premium-features.core :refer [defenterprise]]
+   [toucan2.core :as t2]))
+
+(defenterprise check-allowed-content
+  "Check if the collection's content matches the allowed_content.
+  Throws an exception if it does not"
+  :feature :semantic-layer
+  [content-type collection-id]
+  (when #p collection-id
+    (when-let [allowed-content #p (t2/select-one-fn :allowed_content [:model/Collection :allowed_content] :id collection-id)]
+      (when-not (content-type allowed-content)
+        (throw (ex-info "Content type not allowed in this collection" {:content-type content-type, :allowed-content allowed-content})))))
+
+  true)
+

enterprise/backend/test/metabase_enterprise/semantic_layer/validation_test.clj

@@ -0,0 +1,50 @@
+(ns metabase-enterprise.semantic-layer.validation-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+(deftest allowed-content-doesnt-block-regular-collections
+  (mt/with-premium-features #{:semantic-layer}
+    (mt/with-temp [:model/Collection regular-collection {:name "Regular Collection" :allowed_content nil}]
+      (testing "Regular collections can add anything"
+        (is (some? (t2/insert! :model/Collection (merge (mt/with-temp-defaults :model/Collection) {:location (str "/" (:id regular-collection) "/")}))))
+        (is (some? (t2/insert! :model/Card (merge (mt/with-temp-defaults :model/Card) {:type :model, :collection_id (:id regular-collection)}))))
+        (is (some? (t2/insert! :model/Card (merge (mt/with-temp-defaults :model/Card) {:type :metric, :collection_id (:id regular-collection)}))))
+        (is (some? (t2/insert! :model/Dashboard (merge (mt/with-temp-defaults :model/Dashboard) {:collection_id (:id regular-collection)}))))))))
+
+(deftest allowed-content-completely-locked-down
+  (mt/with-premium-features #{:semantic-layer}
+    (mt/with-temp [:model/Collection no-allowed-content {:name "Test No Content" :allowed_content {}}]
+      (testing "Cannot add anything when collections have no allowed content"
+        (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Content type not allowed in this collection"
+                              (t2/insert! :model/Collection (merge (mt/with-temp-defaults :model/Collection) {:location (str "/" (:id no-allowed-content) "/")}))))
+        (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Content type not allowed in this collection"
+                              (t2/insert! :model/Card (merge (mt/with-temp-defaults :model/Card) {:type :model, :collection_id (:id no-allowed-content)}))))
+        (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Content type not allowed in this collection"
+                              (t2/insert! :model/Card (merge (mt/with-temp-defaults :model/Card) {:type :metric :collection_id (:id no-allowed-content)}))))
+        (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Content type not allowed in this collection"
+                              (t2/insert! :model/Dashboard (merge (mt/with-temp-defaults :model/Dashboard) {:collection_id (:id no-allowed-content)}))))))))
+
+(deftest check-allowed-content-only-collections
+  (mt/with-premium-features #{:semantic-layer}
+    (mt/with-temp [:model/Collection only-subcollections {:name "Test Base Library" :allowed_content {"collection" true}}]
+      (testing "Only subcollections"
+        (is (some? (t2/insert! :model/Collection (merge (mt/with-temp-defaults :model/Collection) {:location (str "/" (:id only-subcollections) "/")}))))
+        (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Content type not allowed in this collection"
+                              (t2/insert! :model/Card (merge (mt/with-temp-defaults :model/Card) {:type :model, :collection_id (:id only-subcollections)}))))
+        (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Content type not allowed in this collection"
+                              (t2/insert! :model/Card (merge (mt/with-temp-defaults :model/Card) {:type :metric :collection_id (:id only-subcollections)}))))
+        (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Content type not allowed in this collection"
+                              (t2/insert! :model/Dashboard (merge (mt/with-temp-defaults :model/Dashboard) {:collection_id (:id only-subcollections)}))))))))
+
+(deftest check-allowed-content-one-type
+  (mt/with-premium-features #{:semantic-layer}
+    (mt/with-temp [:model/Collection allow-models {:name "Test Base Library" :allowed_content {"collection" true, "model" true}}]
+      (testing "Can only add allowed content types"
+        (is (some? (t2/insert! :model/Collection (merge (mt/with-temp-defaults :model/Collection) {:location (str "/" (:id allow-models) "/")}))))
+        (is (some? (t2/insert! :model/Card (merge (mt/with-temp-defaults :model/Card) {:type :model, :collection_id (:id allow-models)}))))
+        (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Content type not allowed in this collection"
+                              (t2/insert! :model/Card (merge (mt/with-temp-defaults :model/Card) {:type :metric :collection_id (:id allow-models)}))))
+        (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Content type not allowed in this collection"
+                              (t2/insert! :model/Dashboard (merge (mt/with-temp-defaults :model/Dashboard) {:collection_id (:id allow-models)}))))))))

enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj

@@ -42,20 +42,20 @@
 
           (testing "the Collections properly exported"
             (is (= (-> (into {} (t2/select-one :model/Collection :id (:id parent)))
-                       (dissoc :id :location)
+                       (dissoc :id :location :allowed_content)
                        (assoc :parent_id nil)
                        (update :created_at t/offset-date-time))
-                   (-> (yaml/from-file (io/file dump-dir "collections" parent-filename (str parent-filename ".yaml")))
-                       (dissoc :serdes/meta)
+                   (-> (into {} (yaml/from-file (io/file dump-dir "collections" parent-filename (str parent-filename ".yaml"))))
+                       (dissoc :serdes/meta :allowed_content)
                        (update :created_at t/offset-date-time))))
 
             (is (= (-> (into {} (t2/select-one :model/Collection :id (:id child)))
-                       (dissoc :id :location)
+                       (dissoc :id :location :allowed_content)
                        (assoc :parent_id (:entity_id parent))
                        (update :created_at t/offset-date-time))
                    (-> (yaml/from-file (io/file dump-dir "collections" parent-filename
                                                 child-filename (str child-filename ".yaml")))
-                       (dissoc :serdes/meta)
+                       (dissoc :serdes/meta :allowed_content)
                        (update :created_at t/offset-date-time))))))))))
 
 (deftest collection-nesting-test

resources/migrations/058_update_migrations.yaml

@@ -38,6 +38,20 @@ databaseChangeLog:
             tableName: report_card
             columnName: embedding_type
 
+  - changeSet:
+      id: v58.2025-11-06T21:49:38
+      author: nvoxland
+      comment: Some collections only allow certain types of content.
+      changes:
+        - addColumn:
+            tableName: collection
+            columns:
+              - column:
+                  name: allowed_content
+                  type: varchar(500)
+                  remarks: JSON map of the types of items that can be added to this card
+
+
 
 # >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
 

src/metabase/bookmarks/models/bookmark.clj

@@ -2,6 +2,7 @@
   (:require
    [clojure.string :as str]
    [metabase.app-db.core :as mdb]
+   [metabase.collections.models.collection :as collection]
    [metabase.config.core :as config]
    [metabase.premium-features.core :as premium-features]
    [metabase.queries.schema :as queries.schema]
@@ -169,3 +170,11 @@
   (t2/insert! :model/BookmarkOrdering (->> orderings
                                            (map #(select-keys % [:type :item_id]))
                                            (map-indexed #(assoc %2 :user_id user-id :ordering %1)))))
+
+(t2/define-before-insert :model/CollectionBookmark [bookmark]
+  (collection/check-allowed-content :collection-bookmark (:collection_id bookmark))
+  bookmark)
+
+(t2/define-before-update :model/CollectionBookmark [model]
+  (collection/check-allowed-content :collection-bookmark (:collection_id (t2/changes model)))
+  model)

src/metabase/collections/core.clj

@@ -13,6 +13,8 @@
   check-for-remote-sync-update
   check-non-remote-synced-dependencies
   check-remote-synced-dependents
+  semantic-layer-collection
+  create-semantic-layer-collection!
   moving-into-remote-synced?
   moving-from-remote-synced?
   non-remote-synced-dependencies])

src/metabase/collections/models/collection.clj

@@ -18,7 +18,7 @@
    [metabase.models.interface :as mi]
    [metabase.models.serialization :as serdes]
    [metabase.permissions.core :as perms]
-   [metabase.premium-features.core :as premium-features]
+   [metabase.premium-features.core :as premium-features :refer [defenterprise]]
    [metabase.remote-sync.core :as remote-sync]
    ;; Trying to use metabase.search would cause a circular reference ;_;
    [metabase.search.spec :as search.spec]
@@ -68,9 +68,9 @@
   "The value of the `:type` field for remote-synced collections."
   "remote-synced")
 
-(def ^:constant library-entity-id
-  "The remote-synced collection's entity ID"
-  "librarylibrarylibrary")
+(def ^:constant semantic-layer-collection-type
+  "The value of the `:type` field for semantic layer collections."
+  "semantic-layer")
 
 (defn- trash-collection* []
   (t2/select-one :model/Collection :type trash-collection-type))
@@ -144,6 +144,32 @@
   (binding [*clearing-remote-sync* true]
     (t2/update! :model/Collection :type remote-synced-collection-type {:type nil})))
 
+(defn semantic-layer-collection
+  "Get the semantic layer collection, if it exists."
+  []
+  (t2/select-one :model/Collection :type semantic-layer-collection-type :location "/"))
+
+(defn create-semantic-layer-collection!
+  "Create the semantic layer collection. Returns root semantic layer collection"
+  []
+  (when-not (nil? (semantic-layer-collection))
+    (throw (ex-info "Semantic Layer already exists" {})))
+  (u/prog1 (t2/insert-returning-instance! :model/Collection {:name     "Semantic Layer"
+                                                             :type     semantic-layer-collection-type
+                                                             :location "/"
+                                                             :allowed_content {:collection true}})
+    (let [base-location        (str "/" (:id <>) "/")]
+      (t2/insert! :model/Collection {:name            "Models"
+                                     :type            semantic-layer-collection-type
+                                     :location        base-location
+                                     :allowed_content {:collection true
+                                                       :model      true}})
+      (t2/insert! :model/Collection {:name            "Metrics"
+                                     :type            semantic-layer-collection-type
+                                     :location        base-location
+                                     :allowed_content {:collection true
+                                                       :metric     true}}))))
+
 (methodical/defmethod t2/table-name :model/Collection [_model] :collection)
 
 (methodical/defmethod t2/model-for-automagic-hydration [#_model :default #_k :collection]
@@ -152,7 +178,8 @@
 
 (t2/deftransforms :model/Collection
   {:namespace       mi/transform-keyword
-   :authority_level mi/transform-keyword})
+   :authority_level mi/transform-keyword
+   :allowed_content mi/transform-json})
 
 (defn maybe-localize-trash-name
   "If the collection is the Trash, translate the `name`. This is a public function because we can't rely on
@@ -338,6 +365,12 @@
                     (tru "A Collection placed in a remote-synced Collection must also be remote-synced."))]
           (throw (ex-info msg {:status-code 400, :errors {:location msg}})))))))
 
+(defenterprise check-allowed-content
+  "Checks contents of a collection before saving it. The OSS implementation is a no-op."
+  metabase-enterprise.semantic-layer.validation
+  [_model-type _collection-id]
+  true)
+
 ;; This function is defined later after children-location is available
 
 (def ^:private CollectionWithLocationOrRoot
@@ -1471,6 +1504,7 @@
   (assert-not-personal-collection-for-api-key collection)
   (assert-valid-namespace (merge {:namespace nil} collection))
   (assert-valid-remote-synced-parent collection)
+  (check-allowed-content :collection (when-let [location (:location (t2/changes collection))] (location-path->parent-id location)))
   (assoc collection :slug (slugify collection-name)))
 
 (defn- copy-collection-permissions!
@@ -1644,6 +1678,8 @@
     (when-not *clearing-remote-sync*
       (let [merged-collection (merge collection-before-updates collection-updates)]
         (assert-valid-remote-synced-parent merged-collection)))
+    ;; (3.6) Check that the parent collection allows this collection to be there
+    (check-allowed-content :collection (when-let [location (:location collection)] (location-path->parent-id location)))
     ;; (4) If we're moving a Collection from a location on a Personal Collection hierarchy to a location not on one,
     ;; or vice versa, we need to grant/revoke permissions as appropriate (see above for more details)
     (when (api/column-will-change? :location collection-before-updates collection-updates)
@@ -1827,14 +1863,19 @@
           :slug
           :type]
    :skip []
-   :transform {:created_at        (serdes/date)
+   :transform {:allowed_content    {:export (fn [allowed-content]
+                                              (if (nil? allowed-content)
+                                                ::serdes/skip
+                                                allowed-content))
+                                    :import identity}
+               :created_at         (serdes/date)
                ;; We only dump the parent id, and recalculate the location from that on load.
-               :location          (serdes/as :parent_id
-                                             (serdes/compose
-                                              (serdes/fk :model/Collection)
-                                              {:export location-path->parent-id
-                                               :import parent-id->location-path}))
-               :personal_owner_id (serdes/fk :model/User)}})
+               :location           (serdes/as :parent_id
+                                              (serdes/compose
+                                               (serdes/fk :model/Collection)
+                                               {:export location-path->parent-id
+                                                :import parent-id->location-path}))
+               :personal_owner_id  (serdes/fk :model/User)}})
 
 ;;; +----------------------------------------------------------------------------------------------------------------+
 ;;; |                                           Perms Checking Helper Fns                                            |

src/metabase/dashboards/models/dashboard.clj

@@ -83,6 +83,7 @@
   (let [defaults  {:parameters []}
         dashboard (lib/normalize ::dashboards.schema/dashboard (merge defaults dashboard))]
     (u/prog1 dashboard
+      (collection/check-allowed-content :dashboard (:collection_id dashboard))
       (params/assert-valid-parameters dashboard)
       (collection/check-collection-namespace :model/Dashboard (:collection_id dashboard)))))
 
@@ -96,6 +97,8 @@
   (let [changes   (t2/changes dashboard)
         dashboard (lib/normalize ::dashboards.schema/dashboard dashboard)
         changes   (lib/normalize ::dashboards.schema/dashboard changes)]
+    (collection/check-allowed-content :dashboard (:collection_id changes))
+
     (u/prog1 (maybe-populate-initially-published-at dashboard)
       (params/assert-valid-parameters dashboard)
       (when (:parameters changes)

src/metabase/native_query_snippets/models/native_query_snippet.clj

@@ -74,6 +74,7 @@
 
 (t2/define-before-insert :model/NativeQuerySnippet [snippet]
   (u/prog1 (add-template-tags snippet)
+    (collection/check-allowed-content :native-query-snippet (:collection_id snippet))
     (collection/check-collection-namespace :model/NativeQuerySnippet (:collection_id snippet))))
 
 (t2/define-after-insert :model/NativeQuerySnippet
@@ -83,6 +84,7 @@
 
 (t2/define-before-update :model/NativeQuerySnippet
   [snippet]
+  (collection/check-allowed-content :native-query-snippet (:collection_id (t2/changes snippet)))
   (u/prog1 (cond-> snippet
              (:content snippet) add-template-tags)
     ;; throw an Exception if someone tries to update creator_id

src/metabase/pulse/models/pulse.clj

@@ -71,6 +71,7 @@
                            (apply merge defaults {:collection_id collection-id}))]
     (u/prog1 notification
       (assert-valid-parameters notification)
+      (collection/check-allowed-content :pulse (:collection_id notification))
       (collection/check-collection-namespace :model/Pulse (:collection_id notification)))))
 
 (def ^:dynamic *allow-moving-dashboard-subscriptions*
@@ -82,6 +83,7 @@
   [notification]
   (let [{:keys [collection_id dashboard_id]} (t2/original notification)
         changes                              (t2/changes notification)]
+    (collection/check-allowed-content :pulse (:collection_id changes))
     (when (and dashboard_id
                (contains? notification :collection_id)
                (not= (:collection_id notification) collection_id)

src/metabase/queries/models/card.clj

@@ -710,6 +710,7 @@
 
 (t2/define-before-insert :model/Card
   [card]
+  (collection/check-allowed-content (:type card) (:collection_id card))
   (-> card
       (assoc :metabase_version config/mb-version-string
              :card_schema current-schema-version)
@@ -753,6 +754,7 @@
   [{:keys [verified-result-metadata?] :as card}]
   (let [changes (some-> card t2/changes queries.schema/normalize-card)
         card    (queries.schema/normalize-card card)]
+    (collection/check-allowed-content (:type card) (:collection_id changes))
     (-> card
         (dissoc :verified-result-metadata?)
         (assoc :card_schema current-schema-version)

src/metabase/timeline/models/timeline.clj

@@ -1,5 +1,6 @@
 (ns metabase.timeline.models.timeline
   (:require
+   [metabase.collections.models.collection :as collection]
    [metabase.collections.models.collection.root :as collection.root]
    [metabase.models.serialization :as serdes]
    [metabase.timeline.models.timeline-event :as timeline-event]
@@ -23,6 +24,14 @@
   (update timeline :icon (fn [icon]
                            (if (= icon "balloons") timeline-event/default-icon icon))))
 
+(t2/define-before-insert :model/Timeline [model]
+  (collection/check-allowed-content :timeline (:collection_id model))
+  model)
+
+(t2/define-before-update :model/Timeline [model]
+  (collection/check-allowed-content :timeline (:collection_id (t2/changes model)))
+  model)
+
 ;;;; functions
 
 (defn timelines-for-collection

test/metabase/collections/models/collection_test.clj

@@ -3306,3 +3306,15 @@
         (is (= [false false false] (map :can_delete (collection/can-delete [non-archived-collection
                                                                             non-archived-dash
                                                                             non-archived-card]))))))))
+
+(deftest create-library
+  (mt/with-discard-model-updates! [:model/Collection]
+    (testing "Can create a semantic layer if none exist"
+      (t2/update! :model/Collection :type collection/semantic-layer-collection-type {:type nil})
+      (let [base (collection/create-semantic-layer-collection!)]
+        (is (= "Semantic Layer" (:name base)))
+        (is (= ["Metrics" "Models"] (sort (map :name (collection/descendants (first (collection/descendants base)))))))))
+    (testing "Creating a Layer when one already exists throws an exception"
+      (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Semantic Layer already exists" (collection/create-semantic-layer-collection!))))
+    ;;cleanup created libraries
+    (t2/delete! :model/Collection :type collection/semantic-layer-collection-type)))