Kimirock/metabase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract a StaticVizRenderer protocol for static-viz rendering (#77315)

Browse Source
This commit is contained in:
Alexander Polyankin
2026-07-09 11:26:13 -04:00
committed by GitHub
parent a9e1b0edf7
commit 22725a83b9
25 changed files with 515 additions and 542 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.dockerignore
View File

@@ -1,5 +1,6 @@
.git
.babel_cache
target/*
target
.circleci
.cpcache
.devcontainer
@@ -15,6 +16,7 @@ test_config/*
test_modules/*
test_resources/*
node_modules
**/node_modules
**metabase.jar
*.db

7
eslint.config.mjs
View File

@@ -547,6 +547,13 @@ const configs = [
"no-console": "off",
},
},
{
// Standalone Node service — console logging is appropriate here.
files: ["frontend/src/static-viz-server/**/*.ts"],
rules: {
"no-console": "off",
},
},
{
files: ["frontend/src/metabase/**/*"],
plugins: {

1
frontend/lint/module-boundaries.mjs
View File

@@ -179,6 +179,7 @@ const elements = [
"frontend/src/metabase/app-embed-mcp-public-path.ts",
"frontend/src/metabase/app-embed-mcp-public-path.unit.spec.ts",
"frontend/src/metabase/app-public.ts",
"frontend/src/metabase/app-static-viz.ts",
"frontend/src/metabase/AppComponent.tsx",
"frontend/src/metabase/App.styled.tsx",
"frontend/src/metabase/AppKBarProvider.tsx",

44
frontend/src/metabase-shared/color_selector.js
View File

@@ -1,44 +0,0 @@
// This runs in the Nashorn JavaScript engine and there are some limitations
//
// 1. This is not currently automatically built with the rest of the application, please run `bun run build-shared` after modifying
// 2. Avoid including unnecessary libraries as the JS engine takes a long time to parse and execute them
// 3. Related to #2, we aren't currently including `core-js` so don't use features that require it, e.x. iterables / for-of
import { makeCellBackgroundGetter } from "metabase/visualizations/lib/table_format";
global.console = {
log: print,
warn: print,
error: print,
};
function buildGetter(rowsJSON, colsJSON, settingsJSON) {
const rows = JSON.parse(rowsJSON);
const cols = JSON.parse(colsJSON);
const settings = settingsJSON ? JSON.parse(settingsJSON) : {};
try {
return makeCellBackgroundGetter(
rows,
cols,
settings?.["table.column_formatting"] ?? [],
settings?.["table.pivot"],
);
} catch (e) {
print("ERROR", e);
return () => null;
}
}
// Colors many cells in one host call, so the JVM side never holds a context-bound function value
// across a render. `cellsJSON` is an array of [value, rowIndex, colName] triples; returns a JSON
// array of color strings (or null), positionally.
global.getCellBackgroundColors = function (
rowsJSON,
colsJSON,
settingsJSON,
cellsJSON,
) {
const getter = buildGetter(rowsJSON, colsJSON, settingsJSON);
const cells = JSON.parse(cellsJSON);
return JSON.stringify(cells.map((cell) => getter(cell[0], cell[1], cell[2])));
};

14
frontend/src/metabase/app-static-viz.ts Normal file
View File

@@ -0,0 +1,14 @@
import "metabase/static-viz/polyfill";
import {
getCellBackgroundColors as getCellBackgroundColorsImpl,
renderChart as renderChartImpl,
} from "metabase/static-viz";
export function renderChart(inputJSON: string): string {
return JSON.stringify(renderChartImpl(JSON.parse(inputJSON)));
}
export function getCellBackgroundColors(inputJSON: string): string {
return JSON.stringify(getCellBackgroundColorsImpl(JSON.parse(inputJSON)));
}

103
frontend/src/metabase/static-viz/index.tsx
View File

@@ -1,6 +1,3 @@
import "./mock-environment";
import "fast-text-encoding";
import { setPlatformAPI } from "echarts/core";
import ReactDOMServer from "react-dom/server";
@@ -17,11 +14,11 @@ import { LegacyStaticChart } from "metabase/static-viz/containers/LegacyStaticCh
import type { LegacyStaticChartType } from "metabase/static-viz/containers/LegacyStaticChart/LegacyStaticChart";
import { createStaticRenderingContext } from "metabase/static-viz/lib/rendering-context";
import { measureTextEChartsAdapter } from "metabase/static-viz/lib/text";
import type { ColorPalette } from "metabase/ui/colors/types";
import { updateStartOfWeek } from "metabase/utils/i18n";
import MetabaseSettings from "metabase/utils/settings";
import { extractRemappings, isCartesianChart } from "metabase/visualizations";
import { extendCardWithDashcardSettings } from "metabase/visualizations/lib/settings/typed-utils";
import { makeCellBackgroundGetter } from "metabase/visualizations/lib/table_format";
import {
createDataSource,
getVisualizationColumns,
@@ -34,44 +31,36 @@ import type {
DashCardVisualizationSettings,
Dataset,
DatasetData,
DayOfWeekId,
FormattingSettings,
GeoJSONData,
RawSeries,
SettingKey,
TokenFeatures,
VisualizerDataSourceId,
VisualizerVizDefinition,
} from "metabase-types/api";
import type {
CellBackgroundColorsInput,
RenderChartDashcardSettings,
RenderChartInput,
RenderChartOptions,
RenderedChart,
} from "./types";
export type {
CellBackgroundColorsInput,
RenderChartInput,
RenderChartOptions,
RenderedChart,
} from "./types";
setPlatformAPI({
measureText: measureTextEChartsAdapter,
});
export type RenderChartOptions = {
tokenFeatures: TokenFeatures;
applicationColors: ColorPalette;
customFormatting: FormattingSettings;
startOfWeek: DayOfWeekId | null | undefined;
// Explicit pixel dimensions for the chart. Use fitWithinBounds to have height include
// chart legends
width?: number;
height?: number;
// When true, width/height are treated as the exact output box
fitWithinBounds?: boolean;
};
type RenderChartDashcardSettings = DashCardVisualizationSettings & {
visualization?: VisualizerVizDefinition;
};
/**
* @deprecated use RenderChart instead
*/
export function LegacyRenderChart(
type: LegacyStaticChartType,
options: unknown,
) {
function LegacyRenderChart(type: LegacyStaticChartType, options: unknown) {
return ReactDOMServer.renderToStaticMarkup(
<LegacyStaticChart type={type} options={options} />,
);
@@ -125,7 +114,7 @@ function getVisualizerRawSeries(
];
}
export function RenderChart(
function RenderChart(
rawSeries: RawSeries,
dashcardSettings: RenderChartDashcardSettings,
options: RenderChartOptions,
@@ -219,3 +208,59 @@ export function RenderChart(
/>,
);
}
export function renderChart(input: RenderChartInput): RenderedChart {
let content: string;
switch (input.kind) {
case "funnel":
content = LegacyRenderChart("funnel", {
data: input.data,
settings: input.settings,
tokenFeatures: input.tokenFeatures,
});
break;
case "gauge":
content = LegacyRenderChart("gauge", {
card: input.card,
data: input.data,
tokenFeatures: input.tokenFeatures,
});
break;
default:
content = RenderChart(
input.rawSeries,
input.dashcardSettings,
input.options,
);
}
return { type: content.startsWith("<svg") ? "svg" : "html", content };
}
function buildCellBackgroundGetter(
...args: Parameters<typeof makeCellBackgroundGetter>
) {
try {
return makeCellBackgroundGetter(...args);
} catch (e) {
console.error("Error building cell background getter", e);
return () => null;
}
}
export function getCellBackgroundColors({
rows,
cols,
settings,
cells,
}: CellBackgroundColorsInput): (string | null)[] {
const getter = buildCellBackgroundGetter(
rows,
cols,
settings?.["table.column_formatting"] ?? [],
Boolean(settings?.["table.pivot"]),
);
return cells.map(
([value, rowIndex, columnName]) =>
getter(value, rowIndex, columnName) ?? null,
);
}

10
frontend/src/metabase/static-viz/mock-environment.ts → frontend/src/metabase/static-viz/polyfill.ts
View File

@@ -1,6 +1,11 @@
import "fast-text-encoding";
import ResizeObserver from "resize-observer-polyfill";
const defineGlobal = (name: PropertyKey, value: unknown): void => {
if (Reflect.has(globalThis, name)) {
return;
}
Object.defineProperty(globalThis, name, {
value,
writable: true,
@@ -296,8 +301,3 @@ defineGlobal(
}
},
);
class MockTextEncoder {
encode(): void {}
}
defineGlobal("TextEncoder", MockTextEncoder);

80
frontend/src/metabase/static-viz/types.ts Normal file
View File

@@ -0,0 +1,80 @@
import type { ColorPalette } from "metabase/ui/colors/types";
import type {
ColumnFormattingSetting,
DashCardVisualizationSettings,
DatasetColumn,
DayOfWeekId,
FormattingSettings,
RawSeries,
RowValue,
RowValues,
TokenFeatures,
VisualizerVizDefinition,
} from "metabase-types/api";
export type RenderChartOptions = {
tokenFeatures: TokenFeatures;
applicationColors: ColorPalette;
customFormatting: FormattingSettings;
startOfWeek: DayOfWeekId | null | undefined;
// Explicit pixel dimensions for the chart. Use fitWithinBounds to have height include
// chart legends
width?: number;
height?: number;
// When true, width/height are treated as the exact output box
fitWithinBounds?: boolean;
};
export type RenderChartDashcardSettings = DashCardVisualizationSettings & {
visualization?: VisualizerVizDefinition;
};
// The default (isomorphic) path — anything StaticVisualization can render.
export type IsomorphicChartInput = {
kind?: undefined;
rawSeries: RawSeries;
dashcardSettings: RenderChartDashcardSettings;
options: RenderChartOptions;
};
// Legacy funnel/gauge charts render through LegacyStaticChart, whose options are untyped.
export type FunnelChartInput = {
kind: "funnel";
data: unknown;
settings: unknown;
tokenFeatures: TokenFeatures;
};
export type GaugeChartInput = {
kind: "gauge";
card: unknown;
data: unknown;
tokenFeatures: TokenFeatures;
};
export type RenderChartInput =
| IsomorphicChartInput
| FunnelChartInput
| GaugeChartInput;
export type RenderedChart = {
type: "svg" | "html";
content: string;
};
// One cell to color: its value, its row index, and its column name.
export type CellToColor = [
value: RowValue,
rowIndex: number,
columnName: string,
];
export type CellBackgroundColorsInput = {
rows: RowValues[];
cols: DatasetColumn[];
settings: {
"table.column_formatting"?: ColumnFormattingSetting[];
"table.pivot"?: boolean;
};
cells: CellToColor[];
};

3
frontend/src/metabase/visualizations/lib/table_format.ts
View File

@@ -1,5 +1,6 @@
// NOTE: this file is used on the frontend and backend and there are some
// limitations. See frontend/src/metabase-shared/color_selector for details
// limitations. On the backend it is bundled into the static-viz bundle and
// called via getCellBackgroundColors; see frontend/src/metabase/static-viz/index.tsx.
import Color from "color";

1
jest.config.js
View File

@@ -66,6 +66,7 @@ const baseConfig = {
"sdk-iframe-embedding-ee-plugins":
"<rootDir>/frontend/src/metabase/utils/noop.ts",
"ee-plugins": "<rootDir>/frontend/src/metabase/utils/noop.ts",
"ee-overrides": "<rootDir>/frontend/src/metabase/utils/noop.ts",
/**
* Imports which are only applicable to the embedding sdk.
* As we use SDK components in new iframe embedding, we need to import them here.

1
package.json
View File

@@ -479,7 +479,6 @@
"build-release:static-viz": "WEBPACK_BUNDLE=production rspack --config rspack.static-viz.config.js",
"build-release:embedding-sdk-package": "WEBPACK_BUNDLE=production rspack --config rspack.embedding-sdk-package.config.js",
"build-release:embedding-sdk-bundle": "WEBPACK_BUNDLE=production rspack --config rspack.embedding-sdk-bundle.config.js",
"build-shared": "bun install && rspack --config rspack.shared.config.js",
"build-static-viz": "bun install && bun run build-release:cljs && WEBPACK_BUNDLE=production rspack --config rspack.static-viz.config.js",
"build-static-viz:watch": "rspack --config rspack.static-viz.config.js --watch",
"build-static-viz:watch-wait": "bun run wait:cljs; bun run build-static-viz:watch",

1
resources/frontend_shared/color_selector.js
View File

File diff suppressed because one or more lines are too long

52
resources/frontend_shared/static_viz_interface.js
View File

@@ -1,52 +0,0 @@
const toJSArray = (a) => {
const jsArray = [];
for (let i = 0; i < a.length; i++) {
jsArray[i] = a[i];
}
return jsArray;
};
function toJSMap(m) {
const o = {};
for (let i = 0; i < m.length; i++) {
o[m[i][0]] = m[i][1];
}
return o;
}
/**
* @deprecated use javascript_visualization instead
*/
function gauge(card, data, tokenFeatures) {
return StaticViz.LegacyRenderChart("gauge", {
card: JSON.parse(card),
data: JSON.parse(data),
tokenFeatures: JSON.parse(tokenFeatures),
});
}
/**
* @deprecated use javascript_visualization instead
*/
function funnel(data, settings, tokenFeatures) {
return StaticViz.LegacyRenderChart("funnel", {
data: JSON.parse(data),
settings: JSON.parse(settings),
tokenFeatures: JSON.parse(tokenFeatures),
});
}
function javascript_visualization(rawSeries, dashcardSettings, options) {
const content = StaticViz.RenderChart(
JSON.parse(rawSeries),
JSON.parse(dashcardSettings),
JSON.parse(options),
);
const type = content.startsWith("<svg") ? "svg" : "html";
return JSON.stringify({
type,
content,
});
}

24
rspack.shared.config.js
View File

@@ -1,24 +0,0 @@
/* eslint-env node */
/* eslint-disable import/no-commonjs */
const path = require("path");
const config = require("./rspack.main.config");
module.exports = {
mode: "production",
devtool: false,
entry: {
color_selector: "./frontend/src/metabase-shared/color_selector.js",
},
module: config.module,
resolve: config.resolve,
output: {
path: path.resolve(__dirname, "resources", "frontend_shared"),
filename: "[name].js",
library: "shared",
},
optimization: {
minimize: true,
},
};

6
rspack.static-viz.config.js
View File

@@ -31,9 +31,9 @@ module.exports = (env) => {
entry: {
"lib-static-viz": {
import: "./static-viz/index.tsx",
import: "./app-static-viz.ts",
library: {
name: "StaticViz",
name: "MetabaseStaticViz",
type: "var",
},
},
@@ -43,7 +43,7 @@ module.exports = (env) => {
path: BUILD_PATH + "/app/dist",
filename: "[name].bundle.js",
publicPath: "/app/dist",
globalObject: "{}",
globalObject: "globalThis",
},
module: {

26
src/metabase/channel/render/js/color.clj
View File

@@ -1,12 +1,10 @@
(ns metabase.channel.render.js.color
"Determines the background colors of pulse table cells by delegating to the shared color-selector javascript (loaded
into every pooled static-viz context — see `metabase.channel.render.js.pool`). All the colors for a table are
computed in a single batched JS call, so no context-bound value ever escapes the pool."
"Determines the background colors of pulse table cells by delegating to the shared color-selector javascript through
[[metabase.channel.render.js.protocol]]. All the colors for a table are computed in a single batched JS call."
(:require
[metabase.channel.render.js.engine :as js.engine]
[metabase.channel.render.js.pool :as js.pool]
[metabase.channel.render.js.protocol :as js.protocol]
[metabase.channel.render.js.renderer :as renderer]
[metabase.formatter.core :as formatter]
[metabase.util.json :as json]
[metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
@@ -65,13 +63,11 @@
(empty? (or (:table.column_formatting viz-settings)
(get viz-settings "table.column_formatting"))))
(vec (repeat (count cells) nil))
(js.pool/with-static-viz-context context
(-> (js.engine/execute-fn-name context "getCellBackgroundColors"
(json/encode rows)
(json/encode cols)
(json/encode viz-settings)
(json/encode (mapv (fn [[value row-index column-name]]
(js.protocol/cell-background-colors
(renderer/renderer)
{:rows rows
:cols cols
:settings viz-settings
:cells (mapv (fn [[value row-index column-name]]
[(->js-value value) row-index column-name])
cells)))
.asString
json/decode))))
cells)})))

94
src/metabase/channel/render/js/engine.clj
View File

@@ -1,94 +0,0 @@
(ns metabase.channel.render.js.engine
"Graal polyglot context suitable for executing javascript code.
We run the js in interpreted mode and turn off the warning with the `(option \"engine.WarnInterpreterOnly\"
\"false\")`. Ideally we would compile the javascript but this is difficult when using the graal ecosystem in a non
graal jdk. See https://github.com/oracle/graaljs/blob/master/docs/user/RunOnJDK.md for more information.
Javadocs: https://www.graalvm.org/truffle/javadoc/overview-summary.html"
(:require
[clojure.java.io :as io]
[metabase.util.i18n :refer [trs]])
(:import
(org.graalvm.polyglot Context Engine HostAccess Source Value)))
(set! *warn-on-reflection* true)
;; A singleton (one shared instance, not rebuilt per context) so all contexts sharing the `Engine` have identical
;; host-access config, as GraalVM requires.
(def ^:private no-host-class-lookup
"Predicate blocking all host class lookup. A singleton so all contexts sharing an engine have identical config."
(reify java.util.function.Predicate
(test [_ _] false)))
(defn- create-engine
"Build a JS `Engine` to be shared across contexts. We run JS interpreted (no Graal compiler on a stock JDK), so
`engine.WarnInterpreterOnly` is silenced here on the engine (it's an engine-level option)."
^Engine []
;; https://github.com/oracle/graaljs/blob/master/docs/user/RunOnJDK.md
(.. (Engine/newBuilder)
(option "engine.WarnInterpreterOnly" "false")
(build)))
(defonce ^:private
^{:doc "The process-lifetime GraalVM `Engine` shared by every sandboxed JS context. The engine owns the Truffle
runtime and the parsed-source code cache, so contexts (and pool recycles) reuse one parsed copy of any
`Source` instance they share instead of each standing up their own. Intentionally never closed."}
shared-sandboxed-js-engine
(delay (create-engine)))
(defn create-context
"Create a sandboxed org.graalvm.polyglot.Context (on the process-shared engine) for evaluating javascript. No host
access, no class lookup, no filesystem I/O. All data must be passed as JSON strings and parsed in JS."
^Context []
(.. (Context/newBuilder (into-array String ["js"]))
(engine ^Engine @shared-sandboxed-js-engine)
(option "js.intl-402" "true")
(allowHostAccess HostAccess/NONE)
(allowHostClassLookup no-host-class-lookup)
(out System/out)
(err System/err)
(allowIO false)
(build)))
(defn load-js-string
"Load a string literal source into the js context."
[^Context context ^String string-src ^String src-name]
(.eval context (.buildLiteral (Source/newBuilder "js" string-src src-name))))
(defn build-source
"Build a `Source` from a classpath resource path. Evaluating the SAME `Source` instance in several contexts that
share an engine makes the engine's code cache reuse one parsed copy; a fresh instance per context makes the engine
parse (and retain) a separate copy each time — so callers that create many contexts should build once and share."
^Source [source-path]
(let [resource (io/resource source-path)]
(when (nil? resource)
(throw (ex-info (trs "Javascript resource not found: {0}" source-path)
{:source source-path})))
(.build (Source/newBuilder "js" ^java.net.URL resource))))
(defn eval-source
"Evaluate an already-built `Source` in the js context."
[^Context context ^Source source]
(.eval context source))
(defn execute-fn-name
"Executes `js-fn-name` in js context with args"
^Value [^Context context js-fn-name & args]
;; TODO: locking context is not ideal, but contexts are currently being shared with all threads and GraalVM doesn't
;; support concurrent execution for js.
;; There is a couple of idea we can try:
;; - put a thread pool around context initialization
;; - init a new context for each thread
(locking context
(let [fn-ref (.eval context "js" js-fn-name)
args (into-array Object args)]
(assert (.canExecute fn-ref) (str "cannot execute " js-fn-name))
(.execute fn-ref args))))
(defn execute-fn
"fn-ref should be an executable org.graalvm.polyglot.Value return from a js engine. Invoke this function with args."
^Value [^Value fn-ref & args]
(assert (.canExecute fn-ref) "cannot execute function reference")
(.execute fn-ref (object-array args)))

221
src/metabase/channel/render/js/graal.clj Normal file
View File

@@ -0,0 +1,221 @@
(ns metabase.channel.render.js.graal
"The GraalVM [[metabase.channel.render.js.protocol/StaticVizRenderer]]: runs the static-viz JS
in-process on a pool of sandboxed GraalVM contexts (up to two by default).
We run the JS interpreted (no Graal compiler on a stock JDK) and silence the interpreter warning with
the engine-level `engine.WarnInterpreterOnly` option. See
https://github.com/oracle/graaljs/blob/master/docs/user/RunOnJDK.md.
The pooled contexts share one `Engine` and one parsed bundle `Source`: the engine (and its
`Source`-keyed code cache) is created with the first context and closed with the last, and each context
evaluates the shared source into its own realm. So one parsed copy of the bundle is held regardless of
how many contexts there are, which makes raising the pool's max from 1 to 2 or 3 a one-line change. A
context is held exclusively per render (so renders serialize per context); the utilization controller
has min 0, so when idle the pool shrinks to 0 and the last `destroy` closes the engine (GraalVM reclaims
neither context nor engine on GC). The first render after an idle gap rebuilds them."
(:require
[clojure.java.io :as io]
[metabase.channel.render.js.protocol :as js.protocol]
[metabase.config.core :as config]
[metabase.util.i18n :refer [trs]]
[metabase.util.json :as json]
[metabase.util.malli :as mu])
(:import
(io.aleph.dirigiste IPool$Generator Pool Pools)
(java.util.concurrent TimeUnit)
(org.graalvm.polyglot Context Engine HostAccess Source Value)))
(set! *warn-on-reflection* true)
;;; --------------------------------------------- engine / context ----------------------------------------
(def ^:private no-host-class-lookup
"Predicate blocking all host class lookup."
(reify java.util.function.Predicate
(test [_ _] false)))
(defn- create-engine
"Build a JS `Engine`. We run JS interpreted (no Graal compiler on a stock JDK), so
`engine.WarnInterpreterOnly` is silenced here (it's an engine-level option)."
^Engine []
(.. (Engine/newBuilder)
(option "engine.WarnInterpreterOnly" "false")
(build)))
(defn create-context
"Create a sandboxed org.graalvm.polyglot.Context for evaluating javascript — on a fresh engine (no args)
or on the given `engine`, so several contexts can share one engine and its parsed-source code cache. No
host access, no class lookup, no filesystem I/O; all data must be passed as JSON strings and parsed in JS."
(^Context [] (create-context (create-engine)))
(^Context [^Engine engine]
(.. (Context/newBuilder (into-array String ["js"]))
(engine engine)
(option "js.intl-402" "true")
(allowHostAccess HostAccess/NONE)
(allowHostClassLookup no-host-class-lookup)
(out System/out)
(err System/err)
(allowIO false)
(build))))
(defn load-js-string
"Load a string literal source into the js context."
[^Context context ^String string-src ^String src-name]
(.eval context (.buildLiteral (Source/newBuilder "js" string-src src-name))))
(defn- build-source
"Build a `Source` from a classpath resource path."
^Source [source-path]
(let [resource (io/resource source-path)]
(when (nil? resource)
(throw (ex-info (trs "Javascript resource not found: {0}" source-path)
{:source source-path})))
(.build (Source/newBuilder "js" ^java.net.URL resource))))
(defn- eval-source
"Evaluate an already-built `Source` in the js context."
[^Context context ^Source source]
(.eval context source))
(defn execute-fn-name
"Execute the global js function named `js-fn-name` in `context` with `args`. Not thread-safe on its own
— a context is held exclusively per render by the pool (see [[call-js]])."
^Value [^Context context js-fn-name & args]
(let [fn-ref (.eval context "js" js-fn-name)]
(assert (.canExecute fn-ref) (str "cannot execute " js-fn-name))
(.execute fn-ref (into-array Object args))))
(defn execute-fn
"fn-ref should be an executable org.graalvm.polyglot.Value returned from a js engine. Invoke it with args."
^Value [^Value fn-ref & args]
(assert (.canExecute fn-ref) "cannot execute function reference")
(.execute fn-ref (object-array args)))
;;; ---------------------------------------- shared engine + contexts -------------------------------------
(def ^:private bundle-resource-path "frontend_client/app/dist/lib-static-viz.bundle.js")
(defn- assert-tests-not-initializing!
"Guard against loading the static-viz bundle as a side effect of loading namespaces: it might not have
been built yet. If it hasn't, we want a meaningful error (see the fixture in
[[metabase.channel.render.js.svg-test]]) rather than a meaningless compilation error at test-runner
startup."
[]
(when config/tests-available?
((requiring-resolve 'mb.hawk.init/assert-tests-are-not-initializing) "(mt/id ...) or (data/id ...)")))
(def ^:private engine-lock (Object.))
(def ^:private shared-engine
"Atom holding `{:engine <Engine>, :source <Source>, :refs <live context count>}`, or nil when no context
is live. Guarded by [[engine-lock]]. The engine and parsed bundle are shared by every pooled context;
they're created with the first context ([[acquire-engine!]]) and closed with the last
([[release-engine!]])."
(atom nil))
(defn- acquire-engine!
"Return the shared `{:engine, :source}`, creating them (parsing the bundle) with the first context.
Bumps the ref count."
[]
(locking engine-lock
(let [state (or @shared-engine
{:source (build-source bundle-resource-path)
:engine (create-engine)
:refs 0})]
(reset! shared-engine (update state :refs inc))
state)))
(defn- release-engine!
"Drop a ref on the shared engine, closing it once the last context is gone."
[]
(locking engine-lock
(let [{:keys [^Engine engine refs]} @shared-engine]
(if (<= refs 1)
(do (try (.close engine) (catch Exception _))
(reset! shared-engine nil))
(swap! shared-engine update :refs dec)))))
(defn- generate-context!
"Build a context on the shared engine and evaluate the bundle into it (creating the engine + parsing the
bundle if this is the first context)."
^Context []
(assert-tests-not-initializing!)
(let [{:keys [^Engine engine ^Source source]} (acquire-engine!)]
(try
(doto (create-context engine)
(eval-source source))
(catch Throwable t
(release-engine!)
(throw t)))))
(defn- destroy-context!
"Close a context and drop its ref on the shared engine (closing the engine if it was the last context)."
[^Context context]
(try (.close context true) (catch Exception _))
(release-engine!))
;;; ------------------------------------------------ context pool -----------------------------------------
(def ^:private pool-key
"Dirigiste pools are keyed; the key itself is arbitrary, it just has to be the same for every operation."
:static-viz)
(def ^:private ^Pool static-viz-context-pool
"A pool of up to two static-viz contexts (change the controller's max to run 1 or 3), each held
exclusively from acquire to release, so at most two renders run at once — one per context, on the
shared engine.
The utilization controller targets 100% utilization with a max of 2 and a min of 0, so when nothing is
rendering it shrinks to 0 and the generator's `destroy` closes the context (and, on the last one, the
shared engine). It rechecks once a minute, so an idle context lingers up to ~1 minute before being
reaped (keeping it warm through short gaps between renders). The other constructor args (queue size,
sampling interval) don't matter much."
(let [max-pool-size 2
max-queued-acquires 65000
sample-period-ms (.toMillis TimeUnit/MILLISECONDS 25)
control-period-ms (.toMillis TimeUnit/MINUTES 1)]
(Pool. (reify IPool$Generator
(generate [_ _]
(generate-context!))
(destroy [_ _ context]
(destroy-context! context)))
(Pools/utilizationController 1.0 max-pool-size max-pool-size)
max-queued-acquires
sample-period-ms
control-period-ms
TimeUnit/MILLISECONDS)))
(defn- do-with-static-viz-context
"Borrow a pooled static-viz context and call `f` with it, held exclusively for the call (never let it —
or a context-bound `Value` — escape). In dev, builds and closes a throwaway context per call so a fresh
`bun run build-static-viz` is picked up without a REPL restart."
[f]
(if config/is-dev?
(let [context (generate-context!)]
(try (f context)
(finally (destroy-context! context))))
(let [context (.acquire static-viz-context-pool pool-key)]
(try (f context)
(finally (.release static-viz-context-pool pool-key context))))))
;;; ------------------------------------------------ backend ----------------------------------------------
(mu/defn- call-js :- :string
"Execute static-viz bundle function `fn-name` (a `MetabaseStaticViz.*` global) with the already-JSON-encoded
string `args` on the pooled context."
[fn-name :- :string
args :- [:sequential :string]]
(do-with-static-viz-context
(fn [^Context context]
(.asString ^Value (apply execute-fn-name context (str "MetabaseStaticViz." fn-name) args)))))
(defn renderer
"The GraalVM [[metabase.channel.render.js.protocol/StaticVizRenderer]] — runs the static-viz JS
in-process on the pooled GraalVM context. Each method JSON-encodes its `input` map for the bundle and
decodes the bundle's JSON result back into Clojure data."
[]
(reify js.protocol/StaticVizRenderer
(chart [_ input]
(json/decode+kw (call-js "renderChart" [(json/encode input)])))
(cell-background-colors [_ input]
(json/decode (call-js "getCellBackgroundColors" [(json/encode input)])))))

160
src/metabase/channel/render/js/pool.clj
View File

@@ -1,160 +0,0 @@
(ns metabase.channel.render.js.pool
"The shared pool of sandboxed GraalVM JS contexts used for static-viz rendering — charts
(`metabase.channel.render.js.svg`) and table-cell color selection (`metabase.channel.render.js.color`). Every
pooled context has the static-viz bundle, the static-viz interface, and the color-selector sources loaded.
### Memory model (GraalJS 25, interpreter mode — measured, see GHY-4077)
- The engine plus the parsed sources cost ~220 MB retained, steady, paid once per process. All contexts share the
process-lifetime engine and evaluate the same `Source` instances (one `defonce` per resource), so the engine's
code cache — which is keyed on the `Source` instance — holds ONE parsed copy of the static-viz bundle. Building a
fresh `Source` per context (the previous behavior) made the engine retain a separate parsed copy for every pool
generation, and made each generation reparse the 27 MB bundle: a multi-hundred-MB allocation spike every context
recycle.
- Evaluating the sources in a fresh context (realm setup against the warm code cache) churns through ~500 MB of
allocation in ~1 s. That is collectable garbage, not footprint: each extra live context retains only ~16 MB. The
very first context in the JVM additionally pays the cold parse — ~1.2 GB / ~5 s.
- A render itself allocates on the order of ~1 MB per row of chart data, also transient.
### Why context creation is serialized
Creation being ~500 MB of churn a pop, letting the pool grow 1→3 concurrently during a render burst stacks
~1–1.5 GB of transient allocation on top of the in-flight renders — on a 2 GB heap (the Cloud default) that
outruns the collector and drives GC thrash / OOM. So creation takes a private monitor
(see [[context-creation-lock]]). A time-based throttle (\"no two creations within X seconds\") is the wrong shape:
a burst of concurrent renders genuinely needs multiple contexts, so we let the pool grow to meet demand — it just
warms one context per ~1 s instead of all at once, bounding the worst case to a single creation's churn. Waiting
acquires simply queue on the pool. Pooled contexts expire after ~10 minutes with jitter so members created
together don't all regenerate in the same window.
Consumers borrow a context within [[with-static-viz-context]] and return plain data — never the context or a
context-bound `Value`. Never acquire a context while already holding one: with a small pool, nested acquires can
deadlock."
(:require
[metabase.channel.render.js.engine :as js.engine]
[metabase.config.core :as config])
(:import
(io.aleph.dirigiste IPool$Controller IPool$Generator Pool Pools Stats)
(java.util.concurrent TimeUnit)
(org.graalvm.polyglot Context)))
(set! *warn-on-reflection* true)
;; Each source is built exactly once per process (defonce + delay) and shared by every context: the engine's code
;; cache is keyed on the `Source` instance, so this is what makes it hold ONE parsed copy of each resource. Building
;; a `Source` anywhere else reintroduces the leak/reparse this exists to fix — after editing a resource in dev,
;; re-evaluate its defonce (or restart the REPL) to pick the change up.
;; goes through webpack; changes require a `bun run build-static-viz`
(defonce ^:private bundle-source
(delay (js.engine/build-source "frontend_client/app/dist/lib-static-viz.bundle.js")))
;; does not go through a build step, edit freely
(defonce ^:private interface-source
(delay (js.engine/build-source "frontend_shared/static_viz_interface.js")))
;; built by `bun run build-shared`
(defonce ^:private color-selector-source
(delay (js.engine/build-source "frontend_shared/color_selector.js")))
(defn- static-viz-sources []
[@bundle-source @interface-source @color-selector-source])
(defn- assert-tests-not-initializing! []
;; make sure people don't try to load the static viz bundle as a side-effect of loading namespaces, because it might
;; not have been built! If it's not built, we want to be able to give people a meaningful error (see the fixture
;; in [[metabase.channel.render.js.svg-test]]) rather than have the test runner fail to start with a meaningless
;; compilation error.
(when config/tests-available?
((requiring-resolve 'mb.hawk.init/assert-tests-are-not-initializing) "(mt/id ...) or (data/id ...)")))
(defn- create-static-viz-context
"A fresh sandboxed context with the static-viz sources loaded."
^Context []
(assert-tests-not-initializing!)
(let [ctx (js.engine/create-context)]
(doseq [source (static-viz-sources)]
(js.engine/eval-source ctx source))
ctx))
(def ^:private pool-key
"Dirigiste pools are keyed. The key itself is arbitrary — it just has to be the same for every pool operation."
:engines)
(def ^:private context-creation-lock
"Monitor serializing pooled context creation. Even with the shared-`Source` code cache, evaluating the bundle in a
fresh context allocates ~500 MB, so letting the pool grow 1→3 concurrently during a render burst stacks ~1–1.5 GB
of transient allocation on top of the in-flight renders — enough to blow a 2 GB heap. Holding creations to one at a
time bounds the worst case to a single creation (~1 s each); waiting acquires just queue on the pool meanwhile.
Nothing else may lock this monitor."
(Object.))
(defn- expiry-timestamp
"Expiry timestamp for a pooled context: now + 10 minutes ± up to 3 minutes of jitter. The jitter keeps pool members
created in the same burst from all expiring — and serially regenerating — in the same window."
^long []
(+ (System/nanoTime)
(.toNanos TimeUnit/MINUTES 7)
(long (rand (.toNanos TimeUnit/MINUTES 6)))))
(defn- generate-pool-entry
"Create a `[context expiry-timestamp]` tuple for the pool. At most one context creation runs at a time (see
[[context-creation-lock]]); acquisition/use is already exclusive per context, so this only bounds pool *growth*."
[]
(locking context-creation-lock
[(create-static-viz-context) (expiry-timestamp)]))
(def ^:private ^Pool static-viz-context-pool
"Pool of Truffle JS contexts. They are not thread-safe, so access is exclusive from acquire to release. Generating a
context is cheap — realm setup plus top-level eval against the engine's code cache, no reparse — and per-context
memory is a realm, not a full parsed copy of the bundle. The pool targets 100% utilization with a maximum of 3
contexts (to bound memory; renders hold a context exclusively, so that is also the render concurrency), but at
least 1 context is always kept in the pool to pick up. Each pooled tuple carries a (jittered, ~10-minute) expiry
timestamp so a context is recycled regardless, bounding per-context leak accumulation. Context creation is
serialized (see [[context-creation-lock]]) so pool growth never runs multiple ~500 MB creations concurrently."
(let [base-controller (Pools/utilizationController 1.0 3 3)]
(Pool. (reify IPool$Generator
(generate [_ _]
(generate-pool-entry))
(destroy [_ _ [^Context ctx _expiry]]
;; Close the context when it's disposed from the pool (expiry/idle shrink/shutdown). Without this, each
;; disposed static-viz context leaks its memory: GraalVM only releases it on `close`, not on GC.
(try
(.close ctx true) ;; force close - can't wait for running code
(catch Exception _))))
;; Wrap the utilization controller with a modification that doesn't allow the pool to go below 1 instance.
(reify IPool$Controller
(shouldIncrement [_ k a b] (.shouldIncrement base-controller k a b))
(adjustment [_ stats]
(let [adj (.adjustment base-controller stats)
n (some-> ^Stats (get stats pool-key) .getNumWorkers)
engines-adj (get adj pool-key)]
(if (and n engines-adj (<= (+ n engines-adj) 0))
;; If the adjustment is going to bring the pool to 0 engines, return empty adjustment instead.
{}
adj))))
65000 ;; Queue size - doesn't matter much.
25 ;; Sampling interval - doesn't matter much.
10000 ;; Recheck every 10 seconds
TimeUnit/MILLISECONDS)))
(defn do-with-static-viz-context
"Impl for [[with-static-viz-context]]."
[f]
(if config/is-dev?
(with-open [ctx (create-static-viz-context)]
(f ctx))
(loop []
(let [[context expiry-ts :as tuple] (.acquire static-viz-context-pool pool-key)]
(if (>= (System/nanoTime) expiry-ts)
(do (.dispose static-viz-context-pool pool-key tuple)
(recur))
(try (f context)
(finally (.release static-viz-context-pool pool-key tuple))))))))
(defmacro with-static-viz-context
"Execute `body` with `binding-name` bound to a pooled static-viz JS context (in dev mode, a fresh context closed
afterwards). The context is borrowed for the dynamic extent of `body`: do not let it — or any context-bound
`Value` — escape, and do not acquire another context inside `body` (nested acquires can deadlock the pool)."
[binding-name & body]
`(do-with-static-viz-context (fn [~binding-name] ~@body)))

18
src/metabase/channel/render/js/protocol.clj Normal file
View File

@@ -0,0 +1,18 @@
;; The protocol implemented by the static-viz renderers.
(ns metabase.channel.render.js.protocol
"The strongly-typed protocol implemented by the static-viz renderers (currently
[[metabase.channel.render.js.graal]]). Callers get the implementation from
[[metabase.channel.render.js.renderer/renderer]] and invoke these methods on it.")
(set! *warn-on-reflection* true)
(defprotocol StaticVizRenderer
"Renders static visualizations. Each method takes a plain map of arguments and returns the corresponding
static-viz bundle function's result parsed from JSON into Clojure data."
(chart [renderer input]
"Render one chart. `input` is a map: the default isomorphic path takes
`{:rawSeries .. :dashcardSettings .. :options ..}`; legacy charts take `{:kind \"funnel\"|\"gauge\" ..}`
plus that kind's fields. Returns a keywordized `{:type .. :content ..}` map.")
(cell-background-colors [renderer input]
"Compute pulse table cell background colors. `input` is `{:rows .. :cols .. :settings .. :cells ..}`.
Returns a vector of color strings (or nil), positionally."))

13
src/metabase/channel/render/js/renderer.clj Normal file
View File

@@ -0,0 +1,13 @@
(ns metabase.channel.render.js.renderer
"Hands callers the static-viz [[metabase.channel.render.js.protocol/StaticVizRenderer]] implementation:
the in-process GraalVM renderer. Kept behind the protocol so an alternative backend can be slotted in
without touching the callers ([[metabase.channel.render.js.svg]], [[metabase.channel.render.js.color]])."
(:require
[metabase.channel.render.js.graal :as graal]))
(set! *warn-on-reflection* true)
(defn renderer
"The static-viz renderer."
[]
(graal/renderer))

68
src/metabase/channel/render/js/svg.clj
View File

@@ -1,18 +1,17 @@
(ns metabase.channel.render.js.svg
"Functions to render charts as svg strings by using graal's js engine. A bundle is built by `bun run build-static-viz`
which has charting library. This namespace has some wrapper functions to invoke those functions. Interop is very
strange, as the jvm datastructures, not just serialized versions are used. This is why we have the `toJSArray` and
`toJSMap` functions to turn Clojure's normal datastructures into js native structures."
"Functions to render charts as svg strings by invoking the static-viz JavaScript bundle (built by `bun run
build-static-viz`) through [[metabase.channel.render.js.protocol]], which runs the JS on a pooled in-process
GraalVM context. This namespace has wrapper functions that build the argument map, call the corresponding
protocol function, and rasterize the resulting SVG to PNG bytes via Batik."
(:require
[clojure.string :as str]
[metabase.appearance.core :as appearance]
[metabase.channel.render.image-buffer :as image-buffer]
[metabase.channel.render.js.engine :as js.engine]
[metabase.channel.render.js.pool :as js.pool]
[metabase.channel.render.js.protocol :as js.protocol]
[metabase.channel.render.js.renderer :as renderer]
[metabase.channel.render.style :as style]
[metabase.lib-be.core :as lib-be]
[metabase.premium-features.core :as premium-features]
[metabase.util.json :as json])
[metabase.premium-features.core :as premium-features])
(:import
(java.io ByteArrayInputStream ByteArrayOutputStream)
(java.nio.charset StandardCharsets)
@@ -168,53 +167,40 @@
frontend/src/metabase/static-viz/components/FunnelChart/types.ts for the actual format options. Returns a byte array
of a png file."
[data settings]
(let [svg-string (js.pool/with-static-viz-context context
(.asString (js.engine/execute-fn-name context "funnel" (json/encode data)
(json/encode settings)
(json/encode (premium-features/token-features)))))]
(svg-string->bytes svg-string)))
(-> (js.protocol/chart (renderer/renderer)
{:kind "funnel"
:data data
:settings settings
:tokenFeatures (premium-features/token-features)})
:content
svg-string->bytes))
(defn ^:dynamic *javascript-visualization*
"Clojure entrypoint to render javascript visualizations. This functions is dynanic only for testing purposes."
[cards-with-data dashcard-viz-settings]
(let [response (js.pool/with-static-viz-context context
(.asString (js.engine/execute-fn-name context "javascript_visualization"
(json/encode cards-with-data)
(json/encode dashcard-viz-settings)
(json/encode (cond-> {:applicationColors (appearance/application-colors)
(-> (js.protocol/chart
(renderer/renderer)
{:rawSeries cards-with-data
:dashcardSettings dashcard-viz-settings
:options (cond-> {:applicationColors (appearance/application-colors)
:startOfWeek (lib-be/start-of-week)
:customFormatting (appearance/custom-formatting)
:tokenFeatures (premium-features/token-features)}
*chart-size*
(assoc :width (:width *chart-size*)
:height (:height *chart-size*)
:fitWithinBounds (boolean (:fit-within? *chart-size*))))))))]
(-> response
json/decode+kw
(update :type (fnil keyword "unknown")))))
:fitWithinBounds (boolean (:fit-within? *chart-size*))))}) (update :type (fnil keyword "unknown"))))
(defn gauge
"Clojure entrypoint to render a gauge chart. Returns a byte array of a png file"
[card data]
(js.pool/with-static-viz-context context
(let [js-res (js.engine/execute-fn-name context "gauge"
(json/encode card)
(json/encode data)
(json/encode (premium-features/token-features)))
svg-string (.asString js-res)]
(svg-string->bytes svg-string))))
(defn progress
"Clojure entrypoint to render a progress bar. Returns a byte array of a png file"
[value goal settings]
(js.pool/with-static-viz-context context
(let [js-res (js.engine/execute-fn-name context "progress"
(json/encode {:value value :goal goal})
(json/encode settings)
(json/encode (appearance/application-colors))
(json/encode (premium-features/token-features)))
svg-string (.asString js-res)]
(svg-string->bytes svg-string))))
(-> (js.protocol/chart (renderer/renderer)
{:kind "gauge"
:card card
:data data
:tokenFeatures (premium-features/token-features)})
:content
svg-string->bytes))
(def ^:private icon-paths
{:dashboard "M32 28a4 4 0 0 1-4 4H4a4.002 4.002 0 0 1-3.874-3H0V4a4 4 0 0 1 4-4h25a3 3 0 0 1 3 3v25zm-4 0V8H4v20h24zM7.273 18.91h10.182v4.363H7.273v-4.364zm0-6.82h17.454v4.365H7.273V12.09zm13.09 6.82h4.364v4.363h-4.363v-4.364z"

27
test/metabase/channel/render/js/engine_test.clj
View File

@@ -1,27 +0,0 @@
(ns metabase.channel.render.js.engine-test
(:require
[clojure.test :refer :all]
[metabase.channel.render.js.engine :as js]
[metabase.test :as mt]))
(set! *warn-on-reflection* true)
(deftest make-context-test
(testing "can make a context that evaluates javascript"
(let [context (js/create-context)]
(js/load-js-string context "function plus (x, y) { return x + y }" "plus test")
(is (= 3 (.asLong (js/execute-fn-name context "plus" 1 2))))))
(testing "can invoke closures return from that javascript"
(let [context (js/create-context)]
(js/load-js-string context "function curry_plus (x) { return function (y) { return x + y}}"
"curried function test")
(let [curried (js/execute-fn-name context "curry_plus" 1)]
(is (= 3 (.asLong (js/execute-fn curried 2))))))))
(deftest thread-safe-execute-fn-name-test
(testing "execute-fn-name is thread safe"
(let [context (js/create-context)]
(js/load-js-string context "function plus (x, y) { return x + y }" "plus test")
(is (= (repeat 10 2)
(mt/repeat-concurrently 10
#(.asLong (js/execute-fn-name context "plus" 1 1))))))))

28
test/metabase/channel/render/js/graal_test.clj Normal file
View File

@@ -0,0 +1,28 @@
(ns metabase.channel.render.js.graal-test
(:require
[clojure.test :refer :all]
[metabase.channel.render.js.graal :as graal]
[metabase.test :as mt]))
(set! *warn-on-reflection* true)
(deftest make-context-test
(testing "can make a context that evaluates javascript"
(let [context (graal/create-context)]
(graal/load-js-string context "function plus (x, y) { return x + y }" "plus test")
(is (= 3 (.asLong (graal/execute-fn-name context "plus" 1 2))))))
(testing "can invoke closures return from that javascript"
(let [context (graal/create-context)]
(graal/load-js-string context "function curry_plus (x) { return function (y) { return x + y}}"
"curried function test")
(let [curried (graal/execute-fn-name context "curry_plus" 1)]
(is (= 3 (.asLong (graal/execute-fn curried 2))))))))
(deftest concurrent-execution-with-locking-test
(testing "concurrent execution on a shared context is safe when callers hold its monitor"
(let [context (graal/create-context)]
(graal/load-js-string context "function plus (x, y) { return x + y }" "plus test")
(is (= (repeat 10 2)
(mt/repeat-concurrently 10
#(locking context
(.asLong (graal/execute-fn-name context "plus" 1 1)))))))))

37
test/metabase/channel/render/js/pool_test.clj
View File

@@ -1,37 +0,0 @@
(ns metabase.channel.render.js.pool-test
(:require
[clojure.test :refer :all]
[metabase.channel.render.js.pool :as js.pool])
(:import
(java.util.concurrent TimeUnit)))
(set! *warn-on-reflection* true)
(deftest generate-pool-entry-serializes-context-creation-test
(testing "no two static-viz context creations run concurrently (#GHY-4077)"
(let [in-flight (atom 0)
max-in-flight (atom 0)
fake-context (Object.)]
(with-redefs [js.pool/create-static-viz-context
(fn []
(let [n (swap! in-flight inc)]
(swap! max-in-flight max n)
(Thread/sleep 50)
(swap! in-flight dec)
fake-context))]
(let [entries (->> (repeatedly 4 #(future (#'js.pool/generate-pool-entry)))
doall
(mapv deref))]
(is (= 1 @max-in-flight))
(testing "every entry is a [context expiry-timestamp] tuple"
(doseq [[ctx expiry-ts] entries]
(is (identical? fake-context ctx))
(is (int? expiry-ts)))))))))
(deftest expiry-timestamp-jitter-test
(testing "expiry is ~10 minutes out with ±3 minutes of jitter"
(let [now (System/nanoTime)
expiries (repeatedly 100 #(#'js.pool/expiry-timestamp))]
(doseq [expiry expiries
:let [delta (- expiry now)]]
(is (<= (.toNanos TimeUnit/MINUTES 7) delta (.toNanos TimeUnit/MINUTES 14)))))))