push this branch to dockerhub by default

.github/actions/build-ee-extra/action.yml

@@ -4,7 +4,7 @@ description: Build Metabase Enterprise Edition with extra features
 inputs:
   java-version:
     required: true
-    default: '21'
+    default: '11'
   iam-role:
     description: "The IAM role to assume"
     required: true

.github/workflows/uberjar.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - "master"
       - metabot-v3-second-try
+      - trying-java-11-default
     paths-ignore:
       # config files
       - ".**"
@@ -126,7 +127,7 @@ jobs:
       with:
         # Build with Java 11 so they compiled code doesn't try to use Java >11 classes... see
         # https://metaboat.slack.com/archives/C5XHN8GLW/p1731517744549619?thread_ts=1731504670.951389&cid=C5XHN8GLW
-        java-version: 11
+        java-version: 17
         iam-role: ${{ secrets.METABASE_EE_EXTRA_IAM_ROLE }}
     - name: Set up Docker Buildx
       id: buildx
@@ -199,15 +200,15 @@ jobs:
         format: sarif
         output: trivy-results.sarif
 
-    - name: Run Trivy vulnerability scanner if dev branch
-      if: ${{ !(startsWith(github.ref_name,'master') || startsWith(github.ref_name,'backport')) && matrix.edition == 'ee' }}
-      uses: aquasecurity/trivy-action@master
-      env:
-        TRIVY_OFFLINE_SCAN: true
-      with:
-        image-ref: docker.io/${{ github.repository_owner }}/metabase-dev:${{ steps.extract_branch.outputs.branch }}
-        format: sarif
-        output: trivy-results.sarif
+#    - name: Run Trivy vulnerability scanner if dev branch
+#      if: ${{ !(startsWith(github.ref_name,'master') || startsWith(github.ref_name,'backport')) && matrix.edition == 'ee' }}
+#      uses: aquasecurity/trivy-action@master
+#      env:
+#        TRIVY_OFFLINE_SCAN: true
+#      with:
+#        image-ref: docker.io/${{ github.repository_owner }}/metabase-dev:${{ steps.extract_branch.outputs.branch }}
+#        format: sarif
+#        output: trivy-results.sarif
 
     - name: Upload Trivy scan results to GitHub Security tab if master (ee)
       if: ${{ (github.ref_name == 'master') && matrix.edition == 'ee' }}

enterprise/backend/src/metabase_enterprise/search/scoring.clj

@@ -25,7 +25,7 @@
   "Return the select-item expressions used to calculate the score for each search result."
   :feature :none
   []
-  (merge fulltext.scoring/base-scorers (select-keys enterprise-scorers (additional-scorers))))
+  (merge (fulltext.scoring/base-scorers) (select-keys enterprise-scorers (additional-scorers))))
 
 ;; ------------ LEGACY ----------
 

frontend/src/metabase/palette/hooks/useCommandPalette.tsx

@@ -188,7 +188,7 @@ export const useCommandPalette = ({
               icon: icon.name,
               section: "search",
               keywords: debouncedSearchText,
-              priority: Priority.NORMAL,
+              priority: Priority.NORMAL - index,
               perform: () => {
                 trackSearchClick("item", index, "command-palette");
               },

src/metabase/api/search.clj

@@ -45,6 +45,20 @@
                 raise)))
    (meta handler)))
 
+(api/defendpoint POST "/re-init"
+  "If fulltext search is enabled, this will blow away the index table, re-create it, and re-populate it."
+  []
+  (api/check-superuser)
+  (cond
+    (not (public-settings/experimental-fulltext-search-enabled))
+    (throw (ex-info "Search index is not enabled." {:status-code 501}))
+
+    (search/supports-index?)
+    (do (search/init-index! {:force-reset? true}) {:message "done"})
+
+    :else
+    (throw (ex-info "Search index is not supported for this installation." {:status-code 501}))))
+
 (api/defendpoint POST "/force-reindex"
   "If fulltext search is enabled, this will trigger a synchronous reindexing operation."
   []
@@ -115,7 +129,7 @@
    archived                            [:maybe :boolean]
    table_db_id                         [:maybe ms/PositiveInt]
    models                              [:maybe (ms/QueryVectorOf search/SearchableModel)]
-   filter_items_in_personal_collection [:maybe [:enum "only" "exclude"]]
+   filter_items_in_personal_collection [:maybe [:enum "only" "exclude" "not-others" "all"]]
    created_at                          [:maybe ms/NonBlankString]
    created_by                          [:maybe (ms/QueryVectorOf ms/PositiveInt)]
    last_edited_at                      [:maybe ms/NonBlankString]

src/metabase/models/card.clj

@@ -1017,6 +1017,7 @@
                   :dashboardcard-count {:select [:%count.*]
                                         :from   [:report_dashboardcard]
                                         :where  [:= :report_dashboardcard.card_id :this.id]}
+                  :last-viewed-at      :last_used_at
                   :native-query        [:case [:= "native" :query_type] :dataset_query]
                   :official-collection [:= "official" :collection.authority_level]
                   :last-edited-at      :r.timestamp

src/metabase/models/dashboard.clj

@@ -672,6 +672,7 @@
                   :database-id    false
                   :last-editor-id :r.user_id
                   :last-edited-at :r.timestamp
+                  :last-viewed-at true
                   :pinned         [:> [:coalesce :collection_position [:inline 0]] [:inline 0]]
                   :view-count     true
                   :created-at     true

src/metabase/public_settings.clj

@@ -1052,7 +1052,6 @@ See [fonts](../configuring-metabase/fonts.md).")
 (defsetting experimental-search-weight-overrides
   (deferred-tru "Used to override weights used for search ranking")
   :visibility :internal
-  :cache?     false
   :encryption :no
   :export?    false
   :default    nil

src/metabase/search/config.clj

@@ -34,15 +34,21 @@
 (def ^:const stale-time-in-days
   "Results older than this number of days are all considered to be equally old. In other words, there is a ranking
   bonus for results newer than this (scaled to just how recent they are). c.f. `search.scoring/recency-score`"
-  180)
+  30)
 
 (def ^:const dashboard-count-ceiling
   "Results in more dashboards than this are all considered to be equally popular."
   10)
 
 (def ^:const view-count-scaling
-  "The larger this value, the longer it will take for the score to approach 1.0. It will never quite reach it."
-  50)
+  "A constant factor influencing how quickly the incremental score grows with view count for a given search model.
+  The larger this value, the longer it will take for the score to approach 1.0. It will never quite reach it."
+  0.2)
+
+(def ^:const view-count-scaling-percentile
+  "The percentile of the given search model's view counts, to be multiplied by [[view-count-scaling]].
+  The larger this value, the longer it will take for the score to approach 1.0. It will never quite reach it."
+  0.99)
 
 (def ^:const surrounding-match-context
   "Show this many words of context before/after matches in long search results"
@@ -81,7 +87,7 @@
 (assert (= all-models (set models-search-order)) "The models search order has to include all models")
 
 (def ^:private default-weights
-  {:pinned              2
+  {:pinned              0
    :bookmarked          2
    :recency             1.5
    :dashboard           1
@@ -193,7 +199,7 @@
    ;;
    [:created-at                          {:optional true} ms/NonBlankString]
    [:created-by                          {:optional true} [:set {:min 1} ms/PositiveInt]]
-   [:filter-items-in-personal-collection {:optional true} [:enum "only" "exclude"]]
+   [:filter-items-in-personal-collection {:optional true} [:enum "only" "exclude" "not-others" "all"]]
    [:last-edited-at                      {:optional true} ms/NonBlankString]
    [:last-edited-by                      {:optional true} [:set {:min 1} ms/PositiveInt]]
    [:limit-int                           {:optional true} ms/Int]

src/metabase/search/impl.clj

@@ -11,6 +11,7 @@
    [metabase.models.database :as database]
    [metabase.models.interface :as mi]
    [metabase.permissions.util :as perms.u]
+   [metabase.public-settings :as public-settings]
    [metabase.public-settings.premium-features :as premium-features]
    [metabase.search.api :as search.api]
    [metabase.search.config
@@ -206,7 +207,10 @@
 (defmethod supported-engine? :search.engine/in-place [_] true)
 (defmethod supported-engine? :search.engine/fulltext [_] (search.fulltext/supported-db? (mdb/db-type)))
 
-(def ^:private default-engine :search.engine/in-place)
+(defn- default-engine []
+  (if (public-settings/experimental-fulltext-search-enabled)
+    :search.engine/fulltext
+    :search.engine/in-place))
 
 (defn- known-engine? [engine]
   (let [registered? #(contains? (methods supported-engine?) %)]
@@ -224,15 +228,16 @@
 
             :else
             engine)))
-      default-engine))
+      (default-engine)))
 
 ;; This forwarding is here for tests, we should clean those up.
 
 (defn- apply-default-engine [{:keys [search-engine] :as search-ctx}]
-  (when (= default-engine search-engine)
-    (throw (ex-info "Missing implementation for default search-engine" {:search-engine search-engine})))
-  (log/debugf "Missing implementation for %s so instead using %s" search-engine default-engine)
-  (assoc search-ctx :search-engine default-engine))
+  (let [default (default-engine)]
+    (when (= default search-engine)
+      (throw (ex-info "Missing implementation for default search-engine" {:search-engine search-engine})))
+    (log/debugf "Missing implementation for %s so instead using %s" search-engine default)
+    (assoc search-ctx :search-engine default)))
 
 (defmethod search.api/results :default [search-ctx]
   (search.api/results (apply-default-engine search-ctx)))
@@ -255,7 +260,7 @@
    [:archived                            {:optional true} [:maybe :boolean]]
    [:created-at                          {:optional true} [:maybe ms/NonBlankString]]
    [:created-by                          {:optional true} [:maybe [:set ms/PositiveInt]]]
-   [:filter-items-in-personal-collection {:optional true} [:maybe [:enum "only" "exclude"]]]
+   [:filter-items-in-personal-collection {:optional true} [:maybe [:enum "only" "exclude" "not-others" "all"]]]
    [:last-edited-at                      {:optional true} [:maybe ms/NonBlankString]]
    [:last-edited-by                      {:optional true} [:maybe [:set ms/PositiveInt]]]
    [:limit                               {:optional true} [:maybe ms/Int]]

src/metabase/search/permissions.clj

@@ -62,9 +62,23 @@
                              [:= collection-id-col :collection.id])
 
       ;; TODO This is not really about permissions, it should really be handled in search.filter
-      (some? filter-items-in-personal-collection)
+      (and (some? filter-items-in-personal-collection) (not= "all" filter-items-in-personal-collection))
       (sql.helpers/where
        (case filter-items-in-personal-collection
+         "not-others"
+         `[:or
+           ;; in one of your personal collections
+           ~[:= :collection.personal_owner_id current-user-id]
+           ;; in a child of one of your collections
+           ~@(for [id (t2/select-pks-set :model/Collection :personal_owner_id [:= current-user-id])]
+               [:like :collection.location (format "/%d/%%" id)])
+           ;; in a non-personal collection
+           [:and [:= :collection.personal_owner_id nil]
+            ~@(for [id (t2/select-pks-set :model/Collection :personal_owner_id [:not= nil])]
+                [:not-like :collection.location (format "/%d/%%" id)])]
+           ;; not in a collection
+           ~[:= collection-id-col nil]]
+
          "only"
          (concat [:or]
                  ;; sub personal collections

src/metabase/search/postgres/core.clj

@@ -77,6 +77,11 @@
   (-> (merge
        (json/parse-string (:legacy_input index-row) keyword)
        (select-keys index-row [:total_score :pinned]))
+      (assoc :scores (mapv (fn [k]
+                             {:score               (get index-row k)
+                              :name                k
+                              :weight              (search.config/weight k)})
+                           (keys (search.scoring/scorers))))
       (update :created_at parse-datetime)
       (update :updated_at parse-datetime)
       (update :last_edited_at parse-datetime)))
@@ -87,12 +92,13 @@
   (when-not @#'search.index/initialized?
     (throw (ex-info "Search index is not initialized. Use [[init!]] to ensure it exists."
                     {:search-engine :postgres})))
-  (->> (let [base-query (search.index/search-query search-term search-ctx [:legacy_input])]
-         (search.permissions/add-collection-join-and-where-clauses base-query "search-index" search-ctx))
-       (search.scoring/with-scores search-ctx)
-       (search.filter/with-filters search-ctx)
-       (t2/query)
-       (map rehydrate)))
+  (let [search-ctx (update search-ctx :filter-items-in-personal-collection #(or % "not-others"))]
+    (->> (let [base-query (search.index/search-query search-term search-ctx [:legacy_input])]
+           (search.permissions/add-collection-join-and-where-clauses base-query "search-index" search-ctx))
+         (search.scoring/with-scores search-ctx)
+         (search.filter/with-filters search-ctx)
+         (t2/query)
+         (map rehydrate))))
 
 (def ^:private default-engine fulltext)
 
@@ -124,7 +130,7 @@
   "Do no scoring, whatsoever"
   [result _scoring-ctx]
   {:score  (:total_score result 1)
-   :result (assoc result :all-scores [] :relevant-scores [])})
+   :result (assoc result :all-scores (:scores result))})
 
 (defn init!
   "Ensure that the search index exists, and has been populated with all the entities."

src/metabase/search/postgres/index.clj

@@ -17,7 +17,7 @@
 
 (defonce ^:private reindexing? (atom false))
 
-(def ^:private tsv-language "simple")
+(def ^:private tsv-language "english")
 
 (defn- exists? [table-name]
   (t2/exists? :information_schema.tables :table_name (name table-name)))
@@ -53,6 +53,7 @@
              [:legacy_input :text :not-null]
              ;; scoring related
              [:dashboardcard_count :int]
+             [:last_viewed_at :timestamp]
              [:model_rank :int :not-null]
              [:official_collection :boolean]
              [:pinned :boolean]

src/metabase/search/postgres/scoring.clj

@@ -1,8 +1,11 @@
 (ns metabase.search.postgres.scoring
   (:require
+   [clojure.core.memoize :as memoize]
    [honey.sql.helpers :as sql.helpers]
    [metabase.public-settings.premium-features :refer [defenterprise]]
-   [metabase.search.config :as search.config]))
+   [metabase.search.config :as search.config]
+   [metabase.util :as u]
+   [toucan2.core :as t2]))
 
 (def ^:private seconds-in-a-day 86400)
 
@@ -22,7 +25,7 @@
   ;; 2/PI * tan^-1 (x/N)
   [:*
    [:/ [:inline 2] [:pi]]
-   [:atan [:/ [:cast [:coalesce column [:inline 0.0]] :float] [:inline scaling]]]])
+   [:atan [:/ [:cast [:coalesce column [:inline 0.0]] :float] scaling]]])
 
 (defn inverse-duration
   "Score at item based on the duration between two dates, where less is better."
@@ -42,7 +45,7 @@
   "Prefer items whose value is earlier in some list."
   [idx-col len]
   (if (pos? len)
-    [:/ [:- [:inline (dec len)] idx-col] [:inline len]]
+    [:/ [:- [:inline (dec len)] idx-col] [:inline (double len)]]
     [:inline 1]))
 
 (defn- sum-columns [column-names]
@@ -74,13 +77,33 @@
                               [:inline 1]])]
     (into [:case] (concat (mapcat (comp match-clause name) bookmarked-models) [:else [:inline 0]]))))
 
-(def base-scorers
+(defn- view-count-percentile-query [p-value]
+  (let [expr [:raw "percentile_cont(" [:lift p-value] ") WITHIN GROUP (ORDER BY view_count)"]]
+    {:select   [:model [expr :vcp]]
+     :from     [:search_index]
+     :group-by [:model]
+     :having   [:is-not expr nil]}))
+
+(def ^:private view-count-percentiles
+  (memoize/ttl (fn [p-value]
+                 (into {} (for [{:keys [model vcp]} (t2/query (view-count-percentile-query p-value))]
+                            [(keyword model) vcp])))
+               :ttl/threshold (u/hours->ms 1)))
+
+(defn- view-count-expr [const-scaling percentile]
+  (let [views (view-count-percentiles percentile)
+        cases (for [[sm v] views]
+                [[:= :model (name sm)] v])]
+    (atan-size :view_count [:* const-scaling [:greatest 1 (into [:case] cat cases)]])))
+
+(defn base-scorers
   "The default constituents of the search ranking scores."
+  []
   {:text       [:ts_rank :search_vector :query [:inline ts-rank-normalization]]
-   :view-count (atan-size :view_count search.config/view-count-scaling)
+   :view-count (view-count-expr search.config/view-count-scaling search.config/view-count-scaling-percentile)
    :pinned     (truthy :pinned)
    :bookmarked bookmark-score-expr
-   :recency    (inverse-duration :model_updated_at [:now] search.config/stale-time-in-days)
+   :recency    (inverse-duration [:coalesce :last_viewed_at :model_updated_at] [:now] search.config/stale-time-in-days)
    :dashboard  (size :dashboardcard_count search.config/dashboard-count-ceiling)
    :model      (idx-rank :model_rank (count search.config/all-models))})
 
@@ -88,7 +111,7 @@
   "Return the select-item expressions used to calculate the score for each search result."
   metabase-enterprise.search.scoring
   []
-  base-scorers)
+  (base-scorers))
 
 (defn- scorer-select-items [] (select-items (scorers)))
 

src/metabase/search/spec.clj

@@ -38,6 +38,7 @@
          :name
          :official-collection
          :dashboardcard-count
+         :last-viewed-at
          :pinned
          :verified                                          ;;  in addition to being a filter, this is also a ranker
          :view-count

src/metabase/task/search_index.clj

@@ -55,10 +55,10 @@
 
 (defn- force-scheduled-task! [^JobDetail job ^Trigger trigger]
   ;; For some reason, using the schedule-task! with a non-durable job causes it to only fire on the first trigger.
-  #_(task/schedule-task! job trigger)
-  (task/delete-task! (.getKey job) (.getKey trigger))
-  (task/add-job! job)
-  (task/add-trigger! trigger))
+  (task/schedule-task! job trigger)
+  #_(task/delete-task! (.getKey job) (.getKey trigger))
+  #_(task/add-job! job)
+  #_(task/add-trigger! trigger))
 
 (jobs/defjob ^{DisallowConcurrentExecution true
                :doc                        "Populate Search Index"}
@@ -73,7 +73,7 @@
 (defmethod task/init! ::SearchIndexReindex [_]
   (let [job         (jobs/build
                      (jobs/of-type SearchIndexReindex)
-                     (jobs/store-durably)
+                     ;(jobs/store-durably)
                      (jobs/with-identity reindex-job-key))
         trigger-key (triggers/key (str reindex-stem ".trigger"))
         trigger     (triggers/build
@@ -87,7 +87,7 @@
 (defmethod task/init! ::SearchIndexUpdate [_]
   (let [job         (jobs/build
                      (jobs/of-type SearchIndexUpdate)
-                     (jobs/store-durably)
+                     ;(jobs/store-durably)
                      (jobs/with-identity update-job-key))
         trigger-key (triggers/key (str update-stem ".trigger"))
         trigger     (triggers/build