diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index be897aa82d6..0b2ef9b9bc2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -47,7 +47,7 @@ jobs: steps: - id: set_matrix run: | - MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["dev", "lean"]'; else echo '["dev", "lean", "websocket", "dockerize", "py311", "py312"]'; fi) + MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["superset", "dev", "lean"]'; else echo '["superset", "dev", "lean", "websocket", "dockerize", "py311", "py312"]'; fi) echo "matrix_config=${MATRIX_CONFIG}" >> $GITHUB_OUTPUT echo $GITHUB_OUTPUT @@ -91,7 +91,7 @@ jobs: exit 1 fi } - for preset in dev lean websocket dockerize; do + for preset in superset dev lean websocket dockerize; do assert_effective_py_ver "$preset" "3.11.14-slim-trixie" done assert_effective_py_ver py311 "3.11-slim-bookworm" @@ -226,7 +226,9 @@ jobs: # different tool from a different maintainer with no shared supply # chain, and is already on the ASF Infra GitHub Actions allowlist. - name: Scan built image for vulnerabilities - if: github.event_name == 'push' && github.ref == 'refs/heads/master' && matrix.build_preset == 'lean' + if: >- + github.event_name == 'push' && github.ref == 'refs/heads/master' && + contains(fromJson('["superset", "lean"]'), matrix.build_preset) id: grype-scan # This step's own failure (e.g. a transient issue pulling the Grype # vulnerability DB) must not fail docker-build, matching the @@ -246,22 +248,29 @@ jobs: - name: Upload vulnerability scan results to GitHub Security tab if: >- github.event_name == 'push' && github.ref == 'refs/heads/master' && - matrix.build_preset == 'lean' && steps.grype-scan.outputs.sarif != '' + contains(fromJson('["superset", "lean"]'), matrix.build_preset) && + steps.grype-scan.outputs.sarif != '' continue-on-error: true uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: sarif_file: ${{ steps.grype-scan.outputs.sarif }} + # Both `superset` and `lean` are scanned now; without a distinct + # category per preset they would land under the same code-scanning + # entry and one build_preset's findings would overwrite the + # other's, mirroring the per-language category in + # codeql-analysis.yml. + category: "docker-${{ matrix.build_preset }}" - name: WebSocket server smoke test - if: contains(fromJson('["lean", "dev"]'), matrix.build_preset) + if: contains(fromJson('["superset", "lean", "dev"]'), matrix.build_preset) shell: bash run: | # The realtime WebSocket server is bundled in the official image and # launched via an alternate entrypoint; verify the bundled Node runtime # starts it and it serves /health. (A JWT secret >= 32 bytes is required # or the server refuses to start; no Redis is needed for /health.) - # Both presets are checked because docker-compose-non-dev.yml runs the - # websocket service from the dev target. + # All app-image presets are checked because docker-compose-non-dev.yml + # runs the websocket service from the dev target. docker run -d --name superset-ws \ -e JWT_SECRET="ci-smoke-test-secret-ci-smoke-test-secret" \ -e PORT=8080 -p 8080:8080 \ diff --git a/.github/workflows/scheduled-docker-image-refresh.yml b/.github/workflows/scheduled-docker-image-refresh.yml index 6e8fe938000..bb82dfd6605 100644 --- a/.github/workflows/scheduled-docker-image-refresh.yml +++ b/.github/workflows/scheduled-docker-image-refresh.yml @@ -98,7 +98,7 @@ jobs: # Mirror the same matrix the release publisher uses so every variant # operators consume from Docker Hub gets the refreshed base. matrix: - build_preset: ["dev", "lean", "websocket", "dockerize", "py311", "py312"] + build_preset: ["superset", "dev", "lean", "websocket", "dockerize", "py311", "py312"] fail-fast: false steps: - name: "Checkout release tag: ${{ needs.config.outputs.latest-release }}" diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml index d7a893ddbce..72632e76b12 100644 --- a/.github/workflows/tag-release.yml +++ b/.github/workflows/tag-release.yml @@ -56,7 +56,7 @@ jobs: strategy: matrix: build_preset: - ["dev", "lean", "websocket", "dockerize", "py311", "py312"] + ["superset", "dev", "lean", "websocket", "dockerize", "py311", "py312"] fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" diff --git a/Dockerfile b/Dockerfile index d0c370e2ca1..d49ce307d6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -289,6 +289,43 @@ RUN python -m compileall /app/superset USER superset +###################################################################### +# Batteries-included default image: lean + common drivers, MCP and a +# headless Chromium. This is the image published under the plain tags +# (latest, master, ); the minimal image ships as `lean`. +###################################################################### +FROM lean AS superset +USER root + +# mysqlclient needs the MySQL client dev headers + pkg-config to build. +# python-common already installs the libs for postgres/sasl/ldap, so only +# the MySQL bits are added here — the minimal `lean` stage stays untouched. +RUN /app/docker/apt-install.sh \ + pkg-config \ + default-libmysqlclient-dev + +# Bundle the common metadata/analytics drivers (postgres, mysql) and the +# MCP server dependencies (fastmcp) so the default image is usable without +# layering extra packages. Routed through pip-install.sh because mysqlclient +# is a source dist that needs a C compiler to build; the helper installs +# build-essential just for the build and purges it afterward (the lean base +# ships no compiler). The mysql client lib installed above stays, so the +# compiled extension can link against it at runtime. +RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \ + /app/docker/pip-install.sh --requires-build-essential .[postgres,mysql,fastmcp] + +# Bundle Playwright + a headless Chromium so Alerts & Reports and thumbnail +# generation work out of the box. Installed directly (rather than via the +# INCLUDE_CHROMIUM build arg on python-common) because this stage builds +# FROM lean, which is already past that arg. Chromium ships for both +# linux/amd64 and linux/arm64, so this works on multi-platform builds. +RUN --mount=type=cache,target=${SUPERSET_HOME}/.cache/uv \ + uv pip install playwright \ + && playwright install-deps \ + && playwright install chromium + +USER superset + ###################################################################### # Dev image... ###################################################################### diff --git a/UPDATING.md b/UPDATING.md index 96b9d0a82ac..b2cd0e8da84 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -24,6 +24,29 @@ assists people when migrating to a new version. ## Next +### Default Docker image is now batteries-included; the minimal image moves to `-lean` + +The default `apache/superset` Docker image (the plain tags: `latest`, `master`, +`5.0.0`, per-SHA) is now a batteries-included, production-grade image. It bundles +the common metadata/analytics drivers (`psycopg2-binary` for PostgreSQL, +`mysqlclient` for MySQL), the MCP server dependencies (`fastmcp`), and a headless +Chromium (via Playwright) for Alerts & Reports and thumbnail generation. It still +runs as the non-root `superset` user and is byte-compiled like before. + +The previous minimal image — with **no** database drivers — is still published, +but under `-lean` tags: `latest-lean`, `master-lean`, `5.0.0-lean`, `-lean`. + +**What operators should expect:** + +- **The default image is larger** (several hundred MB more) because it ships a + headless Chromium and extra drivers. If you were relying on the minimal default + and layering your own drivers, switch your base image to the corresponding + `-lean` tag (for example `FROM apache/superset:master` becomes + `FROM apache/superset:master-lean`) to keep the previous footprint. +- **No config change is required** for most deployments; the metadata-database + drivers most installations need are now present out of the box. +- The `-dev` images (`latest-dev`, `master-dev`, …) are unchanged. + ### Report capture readiness is rechecked immediately before screenshots Scheduled report and alert captures require chart readiness to remain stable diff --git a/docker-compose-image-tag.yml b/docker-compose-image-tag.yml index 265499e0cde..1c65dfbd0cc 100644 --- a/docker-compose-image-tag.yml +++ b/docker-compose-image-tag.yml @@ -24,7 +24,7 @@ # For verbose logging during development: # - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs # ----------------------------------------------------------------------- -x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev} +x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest} x-superset-volumes: &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container - ./docker:/app/docker diff --git a/docker-compose-light.yml b/docker-compose-light.yml index 52bb23c5b51..61077f9a87a 100644 --- a/docker-compose-light.yml +++ b/docker-compose-light.yml @@ -69,7 +69,7 @@ x-superset-volumes: &superset-volumes - ./tests:/app/tests x-common-build: &common-build context: . - target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean` + target: ${SUPERSET_BUILD_TARGET:-dev} # dev (default, live-reload); also `superset` (batteries-included) or `lean` (minimal) cache_from: - apache/superset-cache:3.11-slim-trixie args: diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index 5370a42900d..8032c44b231 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -31,7 +31,10 @@ x-superset-volumes: x-common-build: &common-build context: . - target: dev + # Build the batteries-included production image (the default `apache/superset` + # tags), not the `dev` image: this compose runs a prod-like stack from source, + # and `superset` bundles the metadata/analytics drivers it needs. + target: superset cache_from: - apache/superset-cache:3.11-slim-trixie diff --git a/docker-compose.yml b/docker-compose.yml index 0ce3d27d9fa..ac9ddd77ed4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ x-superset-volumes: &superset-volumes - ./local_extensions:/app/local_extensions x-common-build: &common-build context: . - target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean` + target: ${SUPERSET_BUILD_TARGET:-dev} # dev (default, live-reload); also `superset` (batteries-included) or `lean` (minimal) cache_from: - apache/superset-cache:3.11-slim-trixie args: diff --git a/docs/admin_docs/installation/docker-builds.mdx b/docs/admin_docs/installation/docker-builds.mdx index 2f38b27e901..6da9decfb77 100644 --- a/docs/admin_docs/installation/docker-builds.mdx +++ b/docs/admin_docs/installation/docker-builds.mdx @@ -30,13 +30,19 @@ for the build, and/or base image. Here are the build presets that are exposed through the `supersetbot docker` utility: -- `lean`: The default Docker image, including both frontend and backend. Tags - without a build_preset are lean builds (ie: `latest`, `5.0.0`, `4.1.2`, ...). `lean` - builds do not contain database - drivers, meaning you need to install your own. That applies to analytics databases **AND - the metadata database**. You'll likely want to layer either `mysqlclient` or `psycopg2-binary` - depending on the metadata database you choose for your installation, plus the required - drivers to connect to your analytics database(s). +- `superset`: The default Docker image, including both frontend and backend. Tags + without a build_preset suffix are `superset` builds (ie: `latest`, `5.0.0`, `4.1.2`, ...). + It bundles the common metadata/analytics drivers (`psycopg2-binary` for PostgreSQL and + `mysqlclient` for MySQL), the MCP server dependencies, and a headless Chromium (via + Playwright) for Alerts & Reports and thumbnail generation, so it is usable out of the box. + You'll still need to layer any additional drivers required to connect to your specific + analytics database(s). +- `lean`: A minimal image, including both frontend and backend but **no database + drivers** — published under `-lean` tags (ie: `latest-lean`, `5.0.0-lean`, `master-lean`). + That applies to analytics databases **AND the metadata database**, so you'll need to + layer either `mysqlclient` or `psycopg2-binary` depending on the metadata database you + choose, plus the required drivers to connect to your analytics database(s). Use this when + you want the smallest possible image and full control over what gets installed. - `dev`: For development, with a headless browser, dev-related utilities and root access. This includes some commonly used database drivers like `mysqlclient`, `psycopg2-binary` and some other used for development/CI @@ -47,11 +53,15 @@ Here are the build presets that are exposed through the `supersetbot docker` uti ## Key tags examples -- `latest`: The latest official release build +- `latest`: The latest official release build, batteries-included (common drivers, MCP and + a headless browser). +- `latest-lean`: the minimal variant of the latest official release build, with no database + drivers. - `latest-dev`: the `-dev` image of the latest official release build, with a headless browser and root access. -- `master`: The latest build from the `master` branch, implicitly the lean build - preset +- `master`: The latest build from the `master` branch, implicitly the `superset` (default) + build preset +- `master-lean`: Similar to `master` but the minimal variant with no database drivers. - `master-dev`: Similar to `master` but includes a headless browser and root access. - `pr-5252`: The latest commit in PR 5252. - `30948dc401b40982cb7c0dbf6ebbe443b2748c1b-dev`: A build for @@ -65,8 +75,11 @@ GitHub action. ## Building your own production Docker image -Every Superset deployment will require its own set of drivers depending on the data warehouse(s), -etc. so we recommend that users build their own Docker image by extending the `lean` image. +The default `apache/superset` image already bundles the common metadata/analytics drivers +(`psycopg2-binary`, `mysqlclient`), the MCP server dependencies and a headless Chromium, so +for many deployments no custom image is needed. You'll still want to build your own image when +you need additional analytics database drivers, or when you prefer to start from the minimal +`-lean` image and install only what you need. Here's an example Dockerfile that does this. Follow the in-line comments to customize it for your desired Superset version and database drivers. The comments also note that a certain feature flag will @@ -75,9 +88,10 @@ have to be enabled in your config file. You would build the image with `docker build -t mysuperset:latest .` or `docker build -t ourcompanysuperset:5.0.0 .` ```Dockerfile -# change this to apache/superset:5.0.0 or whatever version you want to build from; -# otherwise the default is the latest commit on GitHub master branch -FROM apache/superset:master +# Start from the minimal lean image (no drivers) and add exactly what you need, +# or start from apache/superset:master to build on top of the batteries-included +# default (which already has psycopg2-binary, mysqlclient, MCP and Playwright). +FROM apache/superset:master-lean USER root @@ -183,14 +197,15 @@ To accelerate builds, we follow Docker best practices and use `apache/superset-c ## About database drivers -Our docker images come with little to zero database driver support since -each environment requires different drivers, and maintaining a build with -wide database support would be both challenging (dozens of databases, -python drivers, and os dependencies) and inefficient (longer -build times, larger images, lower layer cache hit rate, ...). +The default `apache/superset` image bundles the drivers most deployments need for their +metadata database — `psycopg2-binary` (PostgreSQL) and `mysqlclient` (MySQL) — but it does +not attempt to cover every analytics database, since maintaining wide database support would +be both challenging (dozens of databases, python drivers, and os dependencies) and +inefficient (longer build times, larger images, lower layer cache hit rate, ...). -For production use cases, we recommend that you derive our `lean` image(s) and -add database support for the database you need. +The `-lean` image(s) deliberately ship with **no** database drivers at all. For deployments +that want the smallest possible footprint, or need drivers beyond the bundled defaults, we +recommend deriving from the `-lean` image and adding exactly the database support you need. ## On supporting different platforms (namely arm64 AND amd64)