Kimirock/metabase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make cross version tests run using Postgres (#79091)

Browse Source 
* add sample db

* delete cross version 57 since we don't support it anymore

* update cross version tests to use postgres

* fix modal selector to work on v1.60

* fix modal selector for v60 and v63 compatibility

* add better logging

* use qa image

* use herestring to handle large output properly

* use data from WA image

* fix flake in nested model assertion

* cleanup the README

* code review fixes
This commit is contained in:
Chris Vandoodewaard
2026-08-19 13:11:33 -04:00
committed by GitHub
parent 3e9460c53e
commit 83adb8f686
22 changed files with 210 additions and 657 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
.github/workflows/cross-version.yml vendored
View File

@@ -98,9 +98,12 @@ jobs:
- name: Show logs on failure
if: failure()
run: |
cd cross-version
docker compose logs metabase || true
docker compose logs postgres || true
for f in cross-version/logs/*.log; do
[ -e "$f" ] || continue
echo "::group::$f"
cat "$f"
echo "::endgroup::"
done
# Notify Slack on nightly failures
notify-on-failure:
@@ -222,6 +225,9 @@ jobs:
- name: Show logs on failure
if: failure()
run: |
cd cross-version
docker compose logs metabase || true
docker compose logs postgres || true
for f in cross-version/logs/*.log; do
[ -e "$f" ] || continue
echo "::group::$f"
cat "$f"
echo "::endgroup::"
done

1
cross-version/.gitignore vendored
View File

@@ -1 +1,2 @@
.xv-*
logs/

14
cross-version/README.md
View File

@@ -21,13 +21,24 @@ For **downgrades**, the script expects the target to refuse startup, then runs c
### `dev.sh` — local development
Starts a single Metabase version in Docker with H2 and opens Cypress interactively. Snapshot/restore works via `/api/testing/*` endpoints, controlled by the `CROSS_VERSION_DEV_MODE` env var.
Starts a single Metabase version in Docker (same Compose stack as `test.sh`) and opens Cypress interactively.
```bash
./dev.sh --version v1.57.6
./dev.sh --version v1.58.3 --port 3001
```
## Test data
The Postgres container runs [`metabase/qa-databases:postgres-sample-15`](https://github.com/metabase/metabase-qa). It serves two databases:
| Database | Contents |
|----------|----------|
| `metabase` | Metabase's application db (created by `init/00-create-app-db.sql`) |
| `sample` | The QA sample data, loaded by the image's own `sample_data.sql.gz` |
Both databases are created on first boot only and the data is persisted as a volume, so `docker compose down -v` is required to reset them.
## E2E test specs
Specs live in `e2e/cross-version/` and are organized by major version:
@@ -61,7 +72,6 @@ Both `test.sh` and `dev.sh` resolve which spec folder to use for a given version
|----------|---------|-------------|
| `METABASE_PORT` | `3000` (`3077` in dev) | Port to expose Metabase |
| `HEALTH_TIMEOUT` | `120` | Seconds to wait for health check |
| `CROSS_VERSION_DEV_MODE` | — | Set automatically by `dev.sh`; enables snapshot/restore helpers in specs |
## Requirements

53
cross-version/dev.sh
View File

@@ -2,8 +2,8 @@
#
# Interactive dev environment for writing cross-version e2e tests.
#
# Starts an older Metabase version in Docker with H2 and opens Cypress.
# Uses H2 with a shared volume so snapshot/restore works via test endpoints.
# Starts the specified Metabase version and opens Cypress.
# Uses Postgres with a shared volume so snapshot/restore works via test endpoints.
#
# Resolves which spec folder to use:
# e2e/cross-version/{major}/ if it exists, otherwise e2e/cross-version/latest/
@@ -63,9 +63,14 @@ cli() {
bun "$SCRIPT_DIR/cli.ts" "$@"
}
# Compose wrapper: pinned to this directory's project so it keeps working
# after we cd into the repo root to open Cypress (and from the exit trap)
compose() {
docker compose -f "$SCRIPT_DIR/docker-compose.yml" "$@"
}
IMAGE=$(cli image "$VERSION")
MAJOR=$(cli major "$VERSION")
H2_DIR="$SCRIPT_DIR/.xv-h2/v${MAJOR}"
LOG_FILE="$SCRIPT_DIR/.xv-metabase.log"
# Resolve spec folder: exact match, then closest older-version folder, then latest
@@ -97,7 +102,6 @@ log "============================================"
log "Version: $VERSION"
log "Image: $IMAGE"
log "Port: $PORT"
log "H2 dir: $H2_DIR"
log "Logs: $LOG_FILE"
log "Specs: $SPECS_DIR"
log "============================================"
@@ -105,37 +109,28 @@ log "============================================"
# Clean up previous session
rm -f "$LOG_FILE"
# Start with a clean app db every time
# H2 files are owned by the Docker container's user, so use Docker to clean up
if [[ -d "$H2_DIR" ]]; then
docker run --rm -v "$(dirname "$H2_DIR"):/data" alpine rm -rf "/data/$(basename "$H2_DIR")"
fi
mkdir -p "$H2_DIR"
LOGS_PID=""
cleanup() {
log "Stopping Metabase..."
docker rm -f xv-dev-metabase 2>/dev/null || true
# Removing the containers ends the log streams, so the follower exits on its
# own once this returns — and its last writes are the shutdown logs
compose down --volumes 2>/dev/null || true
if [[ -n "$LOGS_PID" ]]; then
wait "$LOGS_PID" 2>/dev/null || true
fi
log "Logs from this session: $LOG_FILE"
}
trap cleanup EXIT
# Remove any leftover container from a previous run (e.g., after SIGKILL)
docker rm -f xv-dev-metabase 2>/dev/null || true
log "Starting Metabase ${VERSION}..."
docker run -d \
--name xv-dev-metabase \
-p "${PORT}:3000" \
-v "${H2_DIR}:/metabase.db" \
-e MB_ENABLE_TEST_ENDPOINTS=true \
-e MB_DANGEROUS_UNSAFE_ENABLE_TESTING_H2_CONNECTIONS_DO_NOT_ENABLE=true \
"$IMAGE"
METABASE_IMAGE="$IMAGE" \
METABASE_PORT="$PORT" compose up -d
# Stream backend logs to file for debugging (docker logs xv-dev-metabase)
docker logs -f xv-dev-metabase > "$LOG_FILE" 2>&1 &
# Create snapshots dir inside the container for /api/testing/snapshot and /api/testing/restore
docker exec xv-dev-metabase sh -c "mkdir -p /e2e/snapshots && chmod 777 /e2e/snapshots"
# Follow container logs into a file for debugging (tail -f it in another shell)
compose logs --no-color --follow > "$LOG_FILE" 2>&1 &
LOGS_PID=$!
log "Waiting for Metabase to be ready..."
TIMEOUT=120
@@ -144,7 +139,7 @@ while true; do
ELAPSED=$(( $(date +%s) - START ))
if (( ELAPSED >= TIMEOUT )); then
error "Timed out after ${TIMEOUT}s"
docker logs xv-dev-metabase
tail -n 100 "$LOG_FILE"
exit 1
fi
if curl -sf "http://localhost:${PORT}/api/health" 2>/dev/null | grep -q '"ok"'; then
@@ -154,14 +149,10 @@ while true; do
done
log "Metabase is ready at http://localhost:${PORT}"
log "Saving blank snapshot..."
curl -sf -X POST "http://localhost:${PORT}/api/testing/snapshot/blank"
log ""
log "Opening Cypress..."
cd "$REPO_ROOT"
CROSS_VERSION_DEV_MODE=true \
CYPRESS_BASE_URL="http://localhost:${PORT}" \
bunx cypress open \
--e2e \

19
cross-version/docker-compose.yml
View File

@@ -1,17 +1,28 @@
services:
postgres:
image: postgres:15
image: metabase/qa-databases:postgres-sample-15
environment:
POSTGRES_USER: metabase
POSTGRES_PASSWORD: metabase
POSTGRES_DB: metabase
# The image ships /docker-entrypoint-initdb.d/sample_data.sql.gz, which the
# entrypoint loads into POSTGRES_DB on first boot. Keep this as `sample` so
# the test data lands in its own database instead of the app db.
POSTGRES_DB: sample
healthcheck:
test: ["CMD-SHELL", "pg_isready -U metabase"]
# Only succeeds once the real server starts which means that 00-create-app-db.sql
# and sample_data.sql.gz have finished (a temp server exists during initialization).
test: ["CMD-SHELL", "pg_isready -U metabase -h 127.0.0.1"]
interval: 5s
timeout: 5s
retries: 5
# Grace period where retries aren't counted while everything initializes
start_period: 60s
volumes:
- postgres_data:/var/lib/postgresql/data
# QA image sets PGDATA=/data
- postgres_data:/data
# Runs before sample_data.sql.gz (initdb.d files run in alphabetical order)
# and creates the `metabase` app db alongside the `sample` data db.
- ./init/00-create-app-db.sql:/docker-entrypoint-initdb.d/00-create-app-db.sql:ro
metabase:
image: ${METABASE_IMAGE:-metabase/metabase:latest}

9
cross-version/init/00-create-app-db.sql Normal file
View File

@@ -0,0 +1,9 @@
--
-- The postgres entrypoint creates POSTGRES_DB (`sample`) and loads the QA image's
-- sample data into it. Metabase needs a separate database for its own app db, so
-- create it here — this file runs before the image's `sample_data.sql.gz`.
--
-- This script runs whenever the `postgres_data` volume doesn't exist on `docker compose up`.
--
CREATE DATABASE metabase OWNER metabase;

44
cross-version/test.sh
View File

@@ -20,6 +20,7 @@ TARGET_VERSION=""
METABASE_PORT="${METABASE_PORT:-3000}"
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-120}"
RESULT_FILE="${RESULT_FILE:-/tmp/cross-version-result.json}"
LOG_DIR="${LOG_DIR:-$SCRIPT_DIR/logs}"
# Colors for output
RED='\033[0;31m'
@@ -152,9 +153,25 @@ start_metabase() {
docker compose up -d --quiet-pull
}
# Docker deletes a container's logs when the container is removed, and both
# stop_metabase and cleanup remove containers. Append them to a file first so
# they are still around after the run (CI prints these on failure).
save_logs() {
local service="$1"
local label="$2"
{
echo "===== ${service} — ${label} ====="
docker compose logs --no-color "$service" 2>&1 || true
echo
} >> "$LOG_DIR/docker-${service}.log"
}
# Stop Metabase (but keep postgres)
stop_metabase() {
local label="$1"
log "Stopping Metabase container..."
save_logs metabase "$label"
docker compose stop metabase
docker compose rm -f metabase
}
@@ -240,12 +257,12 @@ migrate_down_step() {
return 1
fi
if echo "$output" | grep -q "ERROR.*liquibase\|RollbackFailedException\|Command failed with exception"; then
if grep -q "ERROR.*liquibase\|RollbackFailedException\|Command failed with exception" <<<"$output"; then
error "migrate down encountered errors (check logs above)"
return 1
fi
if echo "$output" | grep -q "not rolled back"; then
if grep -q "not rolled back" <<<"$output"; then
error "migrate down had changesets that were not rolled back (check logs above)"
return 1
fi
@@ -310,7 +327,7 @@ check_downgrade_refused() {
# Check container logs for downgrade error
local logs=$(docker compose logs metabase 2>&1 || true)
if echo "$logs" | grep -qi "migrate.*down\|downgrade\|database.*newer"; then
if grep -qi "migrate.*down\|downgrade\|database.*newer" <<<"$logs"; then
log "Metabase correctly detected downgrade scenario"
return 0
fi
@@ -325,9 +342,21 @@ check_downgrade_refused() {
done
}
teardown() {
docker compose down -v --remove-orphans 2>/dev/null || true
}
# Ensure each run starts w/ empty log directory
init_logs() {
mkdir -p "$LOG_DIR"
rm -f "$LOG_DIR"/docker-*.log
}
cleanup() {
log "Cleaning up..."
docker compose down -v --remove-orphans 2>/dev/null || true
save_logs metabase "teardown"
save_logs postgres "teardown"
teardown
}
check_image_exists() {
@@ -363,7 +392,8 @@ main() {
log "============================================"
# Ensure clean state
cleanup
teardown
init_logs
trap cleanup EXIT
@@ -389,7 +419,7 @@ main() {
log ""
log "Step 3: Stopping SOURCE version ($SOURCE_VERSION)..."
stop_metabase
stop_metabase "source $SOURCE_VERSION"
log ""
log "Step 4: Starting TARGET version ($TARGET_VERSION)..."
@@ -422,7 +452,7 @@ main() {
fi
# Stop the failed container
stop_metabase
stop_metabase "target $TARGET_VERSION (downgrade refused)"
log ""
log "Step 5: Rolling back database ($SOURCE_VERSION → $TARGET_VERSION)..."

99
e2e/cross-version/57/00-setup.cy.spec.ts
View File

@@ -1,99 +0,0 @@
/**
* Cross-version test: Instance setup
*
* This must be the first test to run. It completes the Metabase setup wizard
* in the source phase, and verifies the admin can authenticate in both phases.
*
* Uses credentials from e2e/support/cypress_data.js so cy.signInAsAdmin() works.
*/
import { USERS } from "e2e/support/cypress_data";
const { email: ADMIN_EMAIL, password: ADMIN_PASSWORD } = USERS.admin;
const { H } = cy;
describe("Cross-version: Instance setup", () => {
it("setup: completes the setup wizard", { tags: ["@source"] }, () => {
H.restoreCrossVersionDev("blank");
cy.visit("/");
cy.request("GET", "/api/session/properties").then(({ body }) => {
const setupToken = body["setup-token"];
cy.log("Setup token must exist on a fresh instance");
expect(setupToken).to.be.a("string");
cy.request("POST", "/api/setup", {
token: setupToken,
user: {
email: ADMIN_EMAIL,
password: ADMIN_PASSWORD,
first_name: "Admin",
last_name: "CrossVersion",
},
prefs: {
site_name: "CrossVersion Test",
site_locale: "en",
allow_tracking: false,
},
}).then((response) => {
expect(response.status).to.eq(200);
});
cy.request("GET", "/api/user/current").then((response) => {
expect(response.status).to.eq(200);
expect(response.body.email).to.eq(ADMIN_EMAIL);
cy.log("Dismiss `it's ok to play around` modal for admin");
cy.request("PUT", `/api/user/${response.body.id}/modal/qbnewb`);
});
H.withSampleDatabase(({ ORDERS, PRODUCTS, REVIEWS }) => {
cy.log("Remap display values to use foreign key as Product Title");
cy.request("POST", `/api/field/${ORDERS.PRODUCT_ID}/dimension`, {
name: "Product ID",
type: "external",
human_readable_field_id: PRODUCTS.TITLE,
});
cy.log("Remap rating to use custom values");
cy.request("POST", `/api/field/${REVIEWS.RATING}/dimension`, {
type: "internal",
name: "Rating",
});
cy.request("POST", `/api/field/${REVIEWS.RATING}/values`, {
values: [
[1, "Awful"],
[2, "Unpleasant"],
[3, "Meh"],
[4, "Enjoyable"],
[5, "Perfecto"],
],
});
});
});
});
it("verify: admin can authenticate", { tags: ["@target"] }, () => {
cy.signIn("admin", { skipCache: true });
cy.visit("/browse/databases");
cy.findByTestId("database-browser").contains("Sample Database").click();
cy.log("Verify REVIEWS.Rating values display custom remapping");
cy.findAllByRole("link").filter(":contains(Reviews)").click();
cy.findAllByRole("gridcell")
.should("contain", "Enjoyable")
.and("contain", "Perfecto");
cy.go("back");
cy.log("Verify ORDERS.ProductId FK is remapped to Product Title");
cy.findAllByRole("link").filter(":contains(Orders)").click();
cy.findAllByRole("gridcell")
.should("contain", "Fantastic Wool Shirt")
.and("contain", "Small Marble Hat");
H.snapshotCrossVersionDev("00-complete");
});
});

77
e2e/cross-version/57/01-pivot.cy.spec.ts
View File

@@ -1,77 +0,0 @@
import { Q1_NAME } from "../constants";
import * as X from "./helpers";
const { H } = cy;
describe("Cross-version questions - pivot", () => {
it("setup: creates a pivot table", { tags: ["@source"] }, () => {
H.restoreCrossVersionDev("00-complete");
cy.signIn("admin", { skipCache: true });
cy.log("-- Create a pivot table --");
// Intentionally start from the root collection to prevent ambiguity
// in the collection picker when we attempt to save the question
cy.visit("/collection/root");
H.newButton("Question").click();
H.modal().contains("People").should("be.visible").click();
cy.log(
"-- Add filter: Narrow down the states that start with K (only two) --",
);
H.getNotebookStep("filter")
.findByText("Add filters to narrow your answer")
.click();
X.selectFromPopover("State");
cy.findByLabelText("Filter operator").click();
X.selectFromPopover("Starts with");
cy.findByLabelText("Filter value").type("K").blur();
cy.button("Add filter").click();
cy.log("-- Add aggregation --");
H.addSummaryField({ metric: "Count of rows" });
cy.log("-- Add breakouts --");
H.addSummaryGroupingField({ field: "State" });
H.addSummaryGroupingField({ field: "Source" });
H.addSummaryGroupingField({ field: "Created At" });
H.visualize();
cy.log("-- Render as pivot table --");
cy.intercept("POST", "/api/dataset/pivot").as("pivot");
H.openVizTypeSidebar();
H.vizTypeSidebar().within(() => {
cy.findAllByRole("option").filter(":contains(Pivot Table)").click();
cy.wait("@pivot");
cy.button("Done").click();
});
H.vizTypeSidebar().should("not.exist");
X.assertRowCount("155");
X.saveQuestion(Q1_NAME);
});
it("verify: pivot table is preserved", { tags: ["@target"] }, () => {
cy.signIn("admin", { skipCache: true });
cy.visit("/collection/root");
cy.log(`-- ${Q1_NAME}: Assert that the pivot table viz is preserved --`);
cy.findAllByTestId("collection-entry-name")
.filter(`:contains(${Q1_NAME})`)
.click();
X.assertRowCount("155");
cy.findAllByTestId("pivot-table-cell")
.should("contain", "State")
.should("contain", "Created At: Month")
.and("contain", "Organic")
.and("contain", "Row totals")
.and("contain", "KS");
H.snapshotCrossVersionDev("01-complete");
});
});

75
e2e/cross-version/57/02-joins.cy.spec.ts
View File

@@ -1,75 +0,0 @@
import { Q2_NAME } from "../constants";
import * as X from "./helpers";
const { H } = cy;
describe("Cross-version questions - joins", () => {
it("setup: creates a joined question", { tags: ["@source"] }, () => {
H.restoreCrossVersionDev("01-complete");
cy.signIn("admin", { skipCache: true });
cy.log("-- Create a joined question --");
cy.visit("/");
H.newButton("Question").click();
X.joinTables("Orders", "Products");
cy.log("-- Filter on the joined table --");
H.getNotebookStep("filter")
.findByText("Add filters to narrow your answer")
.click();
H.popover().within(() => {
cy.findByRole("heading", { name: "Products" }).click();
cy.findByLabelText("Price").click();
});
H.popover().within(() => {
cy.findByLabelText("Filter operator").should("have.text", "Between");
cy.findByPlaceholderText("Min").type("0");
cy.findByPlaceholderText("Min").type("50").blur();
cy.button("Add filter").click();
});
cy.log("-- Add aggregation --");
H.addSummaryField({ metric: "Average of ..." });
X.selectFromPopover("Discount");
cy.log("-- Add breakouts --");
H.addSummaryGroupingField({ table: "Products", field: "Category" });
cy.log("-- Sort the columns --");
cy.findByLabelText("Sort").click();
X.selectFromPopover("Average of Discount");
H.visualize();
X.assertRowCount("4");
X.saveQuestion(Q2_NAME);
});
it("verify: bar chart is preserved", { tags: ["@target"] }, () => {
cy.signIn("admin", { skipCache: true });
cy.visit("/collection/root");
cy.findAllByTestId("collection-entry-name")
.filter(`:contains(${Q2_NAME})`)
.click();
X.assertRowCount("4");
cy.log("-- Assert that there are four bars --");
H.chartPathWithFillColor("#A989C5").should("have.length", 4);
H.echartsContainer()
.find("text")
.should("contain", "Average of Discount")
.and("contain", "Products → Category")
.and("contain", "Doohickey")
.and("contain", "Gizmo")
.and("contain", "$0")
.and("contain", "$5.00");
H.snapshotCrossVersionDev("02-complete");
});
});

90
e2e/cross-version/57/03-nested-model.cy.spec.ts
View File

@@ -1,90 +0,0 @@
import { Q2_NAME, Q3_NAME } from "../constants";
import * as X from "./helpers";
const { H } = cy;
describe("Cross-version questions - nested model", () => {
it(
"setup: creates a model based on another question",
{ tags: ["@source"] },
() => {
H.restoreCrossVersionDev("02-complete");
cy.signIn("admin", { skipCache: true });
cy.log("-- Create a model from a previous question --");
cy.visit("/");
H.newButton("Question").click();
H.modal().within(() => {
cy.findAllByRole("tab")
.should("be.visible")
.filter(":contains(Collections)")
.click();
});
cy.findAllByTestId("picker-item").filter(`:contains(${Q2_NAME})`).click();
cy.log("-- Add aggregation --");
H.addSummaryField({ metric: "Sum of ..." });
X.selectFromPopover("Average of Discount");
H.visualize();
cy.findByTestId("scalar-value").should("have.text", "$20.80");
X.saveQuestion(Q3_NAME);
cy.log("-- Turn question into a model --");
H.openQuestionActions();
X.selectFromPopover("Turn into a model");
H.modal().button("Turn this into a model").click();
cy.location("pathname").should("contain", "model");
cy.log("-- Edit model metadata --");
H.openQuestionActions();
X.selectFromPopover("Edit metadata");
H.waitForLoaderToBeRemoved();
cy.location("pathname").should("include", "columns");
H.renameColumn("Sum of Average of Discount", "Total Discount");
H.rightSidebar().findByDisplayValue("US Dollar").click();
X.selectFromPopover("Euro");
H.rightSidebar().findByDisplayValue("Euro").should("be.visible");
H.saveMetadataChanges();
cy.location("pathname").should("not.include", "columns");
cy.findByTestId("scalar-value").should("have.text", "€20.80");
},
);
it(
"verify: model custom metadata is preserved",
{ tags: ["@target"] },
() => {
cy.signIn("admin", { skipCache: true });
cy.visit("/collection/root");
H.getPinnedSection()
.findByRole("link")
.should("contain", Q3_NAME)
.click();
cy.findByTestId("question-row-count").should(
"have.text",
"Showing 1 row",
);
cy.log("-- Assert that the model metadata is preserved --");
cy.findByTestId("scalar-value").should("have.text", "€20.80");
cy.findByLabelText("Switch to data").click();
cy.findByTestId("scalar-value").should("not.exist");
cy.findByTestId("header-cell").should("contain", "Total Discount (€)");
H.snapshotCrossVersionDev("03-complete");
},
);
});

123
e2e/cross-version/57/04-sql.cy.spec.ts
View File

@@ -1,123 +0,0 @@
import { Q4_SQL_NAME, SQL_QUERY } from "../constants";
import * as X from "./helpers";
const { H } = cy;
describe("Cross-version questions - sql", () => {
it(
"setup: creates a sql question with a snippet and a field filter variable",
{ tags: ["@source"] },
() => {
H.restoreCrossVersionDev("03-complete");
cy.signIn("admin", { setupCache: true });
cy.request("POST", "/api/native-query-snippet", {
content: "REVIEWS.BODY IS NOT NULL",
name: "Body Not Null",
});
cy.visit("/");
H.newButton("SQL query").click();
cy.log("-- Create a SQL question --");
H.NativeEditor.type(SQL_QUERY, {
focus: true,
allowFastSet: true,
}).blur();
cy.log("-- Map the variable to a field filter --");
cy.findByTestId("variable-type-select").click();
X.selectFromPopover("Field Filter");
H.popover().within(() => {
cy.findByRole("heading", { name: "Sample Database" }).should(
"be.visible",
);
cy.findByText("Reviews").click();
});
X.selectFromPopover("Rating");
H.NativeEditor.clickOnRun();
X.assertRowCount("1,112");
cy.log("-- Hide the PK column --");
cy.findByTestId("visualization-root")
.findByTestId("table-header")
.contains("PRODUCT_ID")
.click();
H.popover().icon("eye_crossed_out").click();
cy.log("-- Assert on the preview values in a table --");
cy.findByTestId("visualization-root").within(() => {
cy.findByTestId("table-header")
.should("contain", "REVIEWER")
.and("contain", "RATING")
.and("not.contain", "PRODUCT_ID");
cy.findByTestId("table-body")
.should("contain", "xavier")
.and("contain", "4")
.and("contain", "clement")
.and("contain", "5");
});
cy.log("-- Assert that the filter is mapped to a correct value --");
cy.findByTestId("parameter-widget").should("contain", "Rating");
X.saveQuestion(Q4_SQL_NAME);
},
);
it("verify: sql question can be filtered", { tags: ["@target"] }, () => {
cy.signIn("admin", { skipCache: true });
cy.visit("/collection/root");
cy.findAllByTestId("collection-entry-name")
.filter(`:contains(${Q4_SQL_NAME})`)
.click();
cy.log("-- Assert that the columns are preserved --");
X.assertRowCount("1,112");
cy.findByTestId("visualization-root").within(() => {
cy.findByTestId("table-header")
.should("contain", "REVIEWER")
.and("contain", "RATING")
.and("not.contain", "PRODUCT_ID");
cy.findByTestId("table-body")
.should("contain", "xavier")
.and("contain", "4")
.and("contain", "clement")
.and("contain", "5");
});
cy.log("-- Apply the filter --");
cy.findByTestId("native-query-editor-container").within(() => {
cy.findByText("This question is written in SQL.").should("be.visible");
cy.findByTestId("parameter-widget").should("contain", "Rating").click();
});
H.popover().within(() => {
cy.findByText("Perfecto").click();
cy.findByText("Meh").click();
cy.button("Add filter").click();
});
cy.findByTestId("qb-header").findByLabelText("Get Answer").click();
cy.log("-- Assert that the filter values are correctly applied --");
X.assertRowCount("442");
cy.findByTestId("parameter-widget")
.should("contain", "Rating")
.and("contain", "2 selections");
cy.findByTestId("table-body")
.and("contain", "clement")
.and("contain", "5")
.should("contain", "clement")
.and("contain", "3")
.should("not.contain", "xavier")
.and("not.contain", "4");
});
});

44
e2e/cross-version/57/helpers.ts
View File

@@ -1,44 +0,0 @@
const { H } = cy;
export function saveQuestion(name: string) {
cy.log(`-- Save question: ${name}`);
cy.intercept("POST", "/api/card").as("saveQuestion");
cy.findByTestId("qb-header").button("Save").click();
cy.findByTestId("save-question-modal").within(() => {
cy.findByLabelText("Name").clear().type(name);
cy.button("Save").click();
cy.wait("@saveQuestion");
});
cy.findByTestId("save-question-modal").should("not.exist");
}
export function assertRowCount(count: string) {
cy.findByTestId("question-row-count").should(
"have.text",
`Showing ${count} rows`,
);
}
export function selectFromPopover(item: string) {
H.popover().contains(item).click();
}
export function joinTables(baseTable: string, joinTable: string) {
H.modal().within(() => {
cy.findAllByRole("tab")
.should("be.visible")
.filter(":contains(Tables)")
.click();
cy.findAllByTestId("picker-item").filter(`:contains(${baseTable})`).click();
});
H.join();
H.modal().within(() => {
cy.findAllByRole("tab")
.should("be.visible")
.filter(":contains(Tables)")
.click();
cy.findAllByTestId("picker-item").filter(`:contains(${joinTable})`).click();
});
}

15
e2e/cross-version/constants.ts
View File

@@ -1,10 +1,19 @@
// Postgres DB with the QA sample data (metabase/qa-databases:postgres-sample-15)
export const XV_DATABASE_NAME = "XV Data";
export const Q1_NAME = "cv1-pivot";
export const Q2_NAME = "cv2-joins";
export const Q3_NAME = "cv3-nested";
export const Q4_SQL_NAME = "cv4-sql";
export const SNIPPET_NAME = "Body Not Null";
// XV Data's tables use unquoted lower-case identifiers, which is what Postgres
// folds unquoted identifiers to — so these queries don't need to quote anything.
export const SNIPPET_CONTENT = "reviews.body IS NOT NULL";
export const SQL_QUERY = `
SELECT PRODUCT_ID, REVIEWER, RATING
FROM REVIEWS
WHERE {{snippet: Body Not Null}}
SELECT product_id, reviewer, rating
FROM reviews
WHERE {{snippet: ${SNIPPET_NAME}}}
AND {{rating}}`;

5
e2e/cross-version/cypress.config.js
View File

@@ -2,7 +2,6 @@ const { defineConfig } = require("cypress");
const { defaultConfig } = require("../support/config");
const isDevMode = !!process.env.CROSS_VERSION_DEV_MODE;
const viewportWidth = 1600;
const viewportHeight = 1200;
@@ -12,10 +11,6 @@ module.exports = defineConfig({
specPattern: "e2e/cross-version/latest/**/*.cy.spec.ts",
viewportWidth,
viewportHeight,
expose: {
...defaultConfig.expose,
CROSS_VERSION_DEV_MODE: isDevMode,
},
setupNodeEvents(on, config) {
// Call the default setupNodeEvents first
defaultConfig.setupNodeEvents(on, config);

70
e2e/cross-version/latest/00-setup.cy.spec.ts
View File

@@ -9,13 +9,14 @@
import { USERS } from "e2e/support/cypress_data";
import { XV_DATABASE_NAME } from "../constants";
const { email: ADMIN_EMAIL, password: ADMIN_PASSWORD } = USERS.admin;
const { H } = cy;
describe("Cross-version: Instance setup", () => {
it("setup: completes the setup wizard", { tags: ["@source"] }, () => {
H.restoreCrossVersionDev("blank");
cy.visit("/");
cy.request("GET", "/api/session/properties").then(({ body }) => {
@@ -49,28 +50,53 @@ describe("Cross-version: Instance setup", () => {
cy.request("PUT", `/api/user/${response.body.id}/modal/qbnewb`);
});
H.withSampleDatabase(({ ORDERS, PRODUCTS, REVIEWS }) => {
cy.log("Remap display values to use foreign key as Product Title");
cy.request("POST", `/api/field/${ORDERS.PRODUCT_ID}/dimension`, {
name: "Product ID",
type: "external",
human_readable_field_id: PRODUCTS.TITLE,
cy.request("POST", "/api/database", {
name: XV_DATABASE_NAME,
engine: "postgres",
details: {
host: "postgres",
port: 5432,
dbname: "sample",
user: "metabase",
password: "metabase",
},
}).then((response) => {
expect(response.status).to.eq(200);
const dbId = response.body.id;
expect(dbId).to.be.a("number");
cy.log("Wait for the initial sync so field ids are available");
// Table names are matched verbatim and the QA sample data uses
// lower-case identifiers. `withDatabase` upper-cases them again.
H.resyncDatabase({
dbId,
tables: ["people", "orders", "products", "reviews"],
});
cy.log("Remap rating to use custom values");
cy.request("POST", `/api/field/${REVIEWS.RATING}/dimension`, {
type: "internal",
name: "Rating",
});
H.withDatabase(dbId, ({ ORDERS, PRODUCTS, REVIEWS }) => {
cy.log("Remap display values to use foreign key as Product Title");
cy.request("POST", `/api/field/${ORDERS.PRODUCT_ID}/dimension`, {
name: "Product ID",
type: "external",
human_readable_field_id: PRODUCTS.TITLE,
});
cy.request("POST", `/api/field/${REVIEWS.RATING}/values`, {
values: [
[1, "Awful"],
[2, "Unpleasant"],
[3, "Meh"],
[4, "Enjoyable"],
[5, "Perfecto"],
],
cy.log("Remap rating to use custom values");
cy.request("POST", `/api/field/${REVIEWS.RATING}/dimension`, {
type: "internal",
name: "Rating",
});
cy.request("POST", `/api/field/${REVIEWS.RATING}/values`, {
values: [
[1, "Awful"],
[2, "Unpleasant"],
[3, "Meh"],
[4, "Enjoyable"],
[5, "Perfecto"],
],
});
});
});
});
@@ -80,7 +106,7 @@ describe("Cross-version: Instance setup", () => {
cy.signIn("admin", { skipCache: true });
cy.visit("/browse/databases");
cy.findByTestId("database-browser").contains("Sample Database").click();
cy.findByTestId("database-browser").contains(XV_DATABASE_NAME).click();
cy.log("Verify REVIEWS.Rating values display custom remapping");
cy.findAllByRole("link").filter(":contains(Reviews)").click();
@@ -94,7 +120,5 @@ describe("Cross-version: Instance setup", () => {
cy.findAllByRole("gridcell")
.should("contain", "Fantastic Wool Shirt")
.and("contain", "Small Marble Hat");
H.snapshotCrossVersionDev("00-complete");
});
});

7
e2e/cross-version/latest/01-pivot.cy.spec.ts
View File

@@ -1,4 +1,4 @@
import { Q1_NAME } from "../constants";
import { Q1_NAME, XV_DATABASE_NAME } from "../constants";
import * as X from "./helpers";
@@ -6,7 +6,6 @@ const { H } = cy;
describe("Cross-version questions - pivot", () => {
it("setup: creates a pivot table", { tags: ["@source"] }, () => {
H.restoreCrossVersionDev("00-complete");
cy.signIn("admin", { skipCache: true });
cy.log("-- Create a pivot table --");
@@ -14,7 +13,7 @@ describe("Cross-version questions - pivot", () => {
// in the collection picker when we attempt to save the question
cy.visit("/collection/root");
H.newButton("Question").click();
X.selectFromPopover("Sample Database");
X.selectFromPopover(XV_DATABASE_NAME);
X.selectFromPopover("People");
cy.log(
@@ -72,7 +71,5 @@ describe("Cross-version questions - pivot", () => {
.and("contain", "Organic")
.and("contain", "Row totals")
.and("contain", "KS");
H.snapshotCrossVersionDev("01-complete");
});
});

3
e2e/cross-version/latest/02-joins.cy.spec.ts
View File

@@ -6,7 +6,6 @@ const { H } = cy;
describe("Cross-version questions - joins", () => {
it("setup: creates a joined question", { tags: ["@source"] }, () => {
H.restoreCrossVersionDev("01-complete");
cy.signIn("admin", { skipCache: true });
cy.log("-- Create a joined question --");
@@ -69,7 +68,5 @@ describe("Cross-version questions - joins", () => {
.and("contain", "Gizmo")
.and("contain", "$0")
.and("contain", "$5.00");
H.snapshotCrossVersionDev("02-complete");
});
});

11
e2e/cross-version/latest/03-nested-model.cy.spec.ts
View File

@@ -9,7 +9,6 @@ describe("Cross-version questions - nested model", () => {
"setup: creates a model based on another question",
{ tags: ["@source"] },
() => {
H.restoreCrossVersionDev("02-complete");
cy.signIn("admin", { skipCache: true });
cy.log("-- Create a model from a previous question --");
@@ -32,7 +31,8 @@ describe("Cross-version questions - nested model", () => {
cy.log("-- Turn question into a model --");
H.openQuestionActions();
X.selectFromPopover("Turn into a model");
H.modal().button("Turn this into a model").click();
// Doesn't use modal() helper from e2e-ui-elements-helpers.js because the modal change significantly between v60 -> v63.
cy.findByRole("dialog").button("Turn this into a model").click();
cy.location("pathname").should("contain", "model");
cy.log("-- Edit model metadata --");
@@ -48,8 +48,11 @@ describe("Cross-version questions - nested model", () => {
H.rightSidebar().findByDisplayValue("Euro").should("be.visible");
H.saveMetadataChanges();
cy.location("pathname").should("not.include", "columns");
// Reload to ensure a scalar is displayed, not a table.
cy.intercept("GET", "/api/card/*/query_metadata").as("modelQuery");
cy.reload();
cy.wait("@modelQuery");
cy.findByTestId("scalar-value").should("have.text", "€20.80");
},
);
@@ -78,8 +81,6 @@ describe("Cross-version questions - nested model", () => {
cy.findByTestId("scalar-value").should("not.exist");
cy.findByTestId("header-cell").should("contain", "Total Discount (€)");
H.snapshotCrossVersionDev("03-complete");
},
);
});

43
e2e/cross-version/latest/04-sql.cy.spec.ts
View File

@@ -1,4 +1,10 @@
import { Q4_SQL_NAME, SQL_QUERY } from "../constants";
import {
Q4_SQL_NAME,
SNIPPET_CONTENT,
SNIPPET_NAME,
SQL_QUERY,
XV_DATABASE_NAME,
} from "../constants";
import * as X from "./helpers";
@@ -9,17 +15,29 @@ describe("Cross-version questions - sql", () => {
"setup: creates a sql question with a snippet and a field filter variable",
{ tags: ["@source"] },
() => {
H.restoreCrossVersionDev("03-complete");
cy.signIn("admin", { setupCache: true });
cy.request("POST", "/api/native-query-snippet", {
content: "REVIEWS.BODY IS NOT NULL",
name: "Body Not Null",
content: SNIPPET_CONTENT,
name: SNIPPET_NAME,
});
cy.visit("/");
H.newButton("SQL query").click();
cy.log("-- Pick the database to query --");
// With more than one database and none previously used, the picker
// opens on its own, so there is no trigger to click here
cy.findByTestId("native-query-top-bar").should(
"contain",
"Select a database",
);
X.selectFromPopover(XV_DATABASE_NAME);
cy.findByTestId("selected-database").should(
"have.text",
XV_DATABASE_NAME,
);
cy.log("-- Create a SQL question --");
H.NativeEditor.type(SQL_QUERY, {
focus: true,
@@ -30,7 +48,7 @@ describe("Cross-version questions - sql", () => {
cy.findByTestId("variable-type-select").click();
X.selectFromPopover("Field Filter");
H.popover().within(() => {
cy.findByRole("heading", { name: "Sample Database" }).should(
cy.findByRole("heading", { name: XV_DATABASE_NAME }).should(
"be.visible",
);
cy.findByText("Reviews").click();
@@ -41,18 +59,19 @@ describe("Cross-version questions - sql", () => {
X.assertRowCount("1,112");
cy.log("-- Hide the PK column --");
// Native results keep the raw column names, and the QA sample data's are lower-case
cy.findByTestId("visualization-root")
.findByTestId("table-header")
.contains("PRODUCT_ID")
.contains("product_id")
.click();
H.popover().icon("eye_crossed_out").click();
cy.log("-- Assert on the preview values in a table --");
cy.findByTestId("visualization-root").within(() => {
cy.findByTestId("table-header")
.should("contain", "REVIEWER")
.and("contain", "RATING")
.and("not.contain", "PRODUCT_ID");
.should("contain", "reviewer")
.and("contain", "rating")
.and("not.contain", "product_id");
cy.findByTestId("table-body")
.should("contain", "xavier")
@@ -81,9 +100,9 @@ describe("Cross-version questions - sql", () => {
X.assertRowCount("1,112");
cy.findByTestId("visualization-root").within(() => {
cy.findByTestId("table-header")
.should("contain", "REVIEWER")
.and("contain", "RATING")
.and("not.contain", "PRODUCT_ID");
.should("contain", "reviewer")
.and("contain", "rating")
.and("not.contain", "product_id");
cy.findByTestId("table-body")
.should("contain", "xavier")

6
e2e/cross-version/latest/helpers.ts
View File

@@ -1,3 +1,5 @@
import { XV_DATABASE_NAME } from "../constants";
const { H } = cy;
export function saveQuestion(name: string) {
@@ -35,11 +37,11 @@ export function visitRootCollectionAndWait() {
}
export function joinTables(baseTable: string, joinTable: string) {
selectFromPopover("Sample Database");
selectFromPopover(XV_DATABASE_NAME);
selectFromPopover(baseTable);
H.join();
selectFromPopover("Sample Database");
selectFromPopover(XV_DATABASE_NAME);
selectFromPopover(joinTable);
}

41
e2e/support/helpers/e2e-setup-helpers.js
View File

@@ -4,26 +4,6 @@ export function snapshot(name) {
cy.request("POST", `/api/testing/snapshot/${name}`);
}
/**
* Take a snapshot during cross-version test development.
*
* This is a no-op in CI / production runs. It only takes effect when
* the `CROSS_VERSION_DEV_MODE` env var is exposed through Cypress,
* allowing developers to incrementally snapshot state while iterating
* on cross-version test scenarios locally.
*
* @param {string} name - snapshot identifier
*/
export function snapshotCrossVersionDev(name) {
if (!Cypress.expose?.("CROSS_VERSION_DEV_MODE")) {
cy.log(
`skipping cross-version snapshot "${name}" — not running in dev mode (production runs against Postgres)`,
);
return;
}
cy.request("POST", `/api/testing/snapshot/${name}`);
}
/**
*
* @param { |
@@ -56,24 +36,3 @@ export function restore(name = "default") {
return cy.request("POST", `/api/testing/restore/${name}`);
}
/**
* Restore a snapshot during cross-version test development.
*
* This is a no-op in CI / production runs. It only takes effect when
* the `CROSS_VERSION_DEV_MODE` env var is exposed through Cypress,
* allowing developers to restore a previously-saved snapshot while
* iterating on cross-version test scenarios locally.
*
* @param {string} [name="blank"] - snapshot identifier to restore
*/
export function restoreCrossVersionDev(name = "blank") {
if (!Cypress.expose?.("CROSS_VERSION_DEV_MODE")) {
cy.log(
`skipping cross-version restore "${name}" — not running in dev mode (production runs against Postgres)`,
);
return;
}
return cy.request("POST", `/api/testing/restore/${name}`);
}