ff8e7993d3
* Internal load-namespaces tool to check for compilation errors * Ignore load-namespaces in module checks * lint load-namespaces * Reject positional args * Test the new utility * Guard against nonexistent root. Make the code more testable.
368 lines
15 KiB
YAML
368 lines
15 KiB
YAML
name: Backend
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
skip:
|
|
type: boolean
|
|
default: false
|
|
force-run:
|
|
description: Run the full suite regardless of the diff (master / release branches).
|
|
type: boolean
|
|
default: false
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-backend
|
|
cancel-in-progress: true
|
|
|
|
# Exposed to the test jobs so they can report failures to ci-conductor
|
|
# (secrets: inherit is already set on the backend call in run-tests.yml).
|
|
# See DEV-2224.
|
|
env:
|
|
CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}
|
|
CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
|
|
|
|
jobs:
|
|
be-linter-clj-kondo:
|
|
if: ${{ !inputs.skip }}
|
|
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
|
|
timeout-minutes: 10
|
|
name: Clj-Kondo
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Prepare back-end environment
|
|
uses: ./.github/actions/prepare-backend
|
|
- name: Run clj-kondo
|
|
run: ./bin/mage kondo
|
|
|
|
be-linter-eastwood:
|
|
if: ${{ !inputs.skip }}
|
|
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
|
|
timeout-minutes: 20
|
|
name: Eastwood
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Prepare back-end environment
|
|
uses: ./.github/actions/prepare-backend
|
|
- run: clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test:eastwood
|
|
name: Run Eastwood linter
|
|
|
|
be-linter-eastwood-test:
|
|
# Master / release safety net: lint every backend test namespace.
|
|
if: ${{ !inputs.skip && inputs.force-run }}
|
|
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
|
|
timeout-minutes: 45
|
|
name: Eastwood (tests only)
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Prepare back-end environment
|
|
uses: ./.github/actions/prepare-backend
|
|
- run: clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test:eastwood/test
|
|
name: Run Eastwood linter on our backend tests
|
|
|
|
be-linter-eastwood-test-changed:
|
|
# Lint only the test namespaces touched by the diff.
|
|
if: ${{ !inputs.skip && !inputs.force-run && github.event_name == 'pull_request' }}
|
|
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
|
|
timeout-minutes: 15
|
|
name: Eastwood (changed tests only)
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
# Enough history for `git merge-base` on a typical PR.
|
|
fetch-depth: 200
|
|
- name: Prepare back-end environment
|
|
uses: ./.github/actions/prepare-backend
|
|
- name: Compute changed test namespaces
|
|
id: changed
|
|
env:
|
|
BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
|
|
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
# Diff from the merge-base so changes that landed on the base branch after the PR was
|
|
# last synced don't widen our scope. Falls back to BASE_SHA if older than fetched depth.
|
|
git fetch --no-tags --depth=200 origin "$BASE_SHA"
|
|
DIFF_BASE=$(git merge-base "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "$BASE_SHA")
|
|
# Test roots we lint: core, EE, and each driver module's test tree.
|
|
# :eastwood/test derives matching :source-paths from the classpath, so any
|
|
# namespace we name here is reachable. Include renames (R) so a test that
|
|
# was just renamed still gets linted under its new name.
|
|
FILES=$(git diff --name-only --diff-filter=AMR "$DIFF_BASE" "$HEAD_SHA" \
|
|
-- 'test/**/*.clj' 'test/**/*.cljc' \
|
|
'enterprise/backend/test/**/*.clj' 'enterprise/backend/test/**/*.cljc' \
|
|
'modules/drivers/**/test/**/*.clj' 'modules/drivers/**/test/**/*.cljc')
|
|
|
|
if [ -z "$FILES" ]; then
|
|
echo "No backend test files changed; skipping lint."
|
|
echo "namespaces=" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
|
|
# Defence-in-depth: paths flow into a subsequent shell command via $GITHUB_OUTPUT.
|
|
if grep -Eqv '^[A-Za-z0-9_./-]+$' <<<"$FILES"; then
|
|
echo "Refusing to lint: unexpected characters in changed file paths:" >&2
|
|
echo "$FILES" >&2
|
|
exit 1
|
|
fi
|
|
|
|
NS=$(sed -E 's#^(test/|enterprise/backend/test/|modules/drivers/[^/]+/test/)##; s#\.cljc?$##; s#/#.#g; s#_#-#g' <<<"$FILES" \
|
|
| sort -u \
|
|
| paste -sd' ' -)
|
|
echo "Changed test namespaces: $NS"
|
|
echo "namespaces=$NS" >> "$GITHUB_OUTPUT"
|
|
- name: Run Eastwood on changed test namespaces
|
|
if: steps.changed.outputs.namespaces != ''
|
|
env:
|
|
NAMESPACES: ${{ steps.changed.outputs.namespaces }}
|
|
run: clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test:eastwood/test :namespaces "[$NAMESPACES]"
|
|
|
|
be-tests:
|
|
if: ${{ !inputs.skip }}
|
|
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
|
|
timeout-minutes: 40
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
java-version: [25]
|
|
job:
|
|
- name: Enterprise Tests
|
|
edition: ee
|
|
test-group-name: ee
|
|
test-args: >-
|
|
:only '"enterprise/backend/test"'
|
|
- name: EE App DB Tests (Part 1)
|
|
edition: ee
|
|
test-group-name: ee-app-db-1
|
|
test-args: >-
|
|
:only '["test" ".clj-kondo/test"]'
|
|
:partition/total 2
|
|
:partition/index 0
|
|
- name: EE App DB Tests (Part 2)
|
|
edition: ee
|
|
test-group-name: ee-app-db-2
|
|
test-args: >-
|
|
:only '["test" ".clj-kondo/test"]'
|
|
:partition/total 2
|
|
:partition/index 1
|
|
- name: OSS App DB Tests (Part 1)
|
|
edition: oss
|
|
test-group-name: oss-app-db-1
|
|
test-args: >-
|
|
:only '["test" ".clj-kondo/test"]'
|
|
:partition/total 2
|
|
:partition/index 0
|
|
- name: OSS App DB Tests (Part 2)
|
|
edition: oss
|
|
test-group-name: oss-app-db-2
|
|
test-args: >-
|
|
:only '["test" ".clj-kondo/test"]'
|
|
:partition/total 2
|
|
:partition/index 1
|
|
name: "Java ${{ matrix.java-version }} ${{ matrix.job.name }}"
|
|
env:
|
|
# Old migrations tests only run on a force-run ref; everything else excludes
|
|
# the `mb/old-migrations-test` tag. `__ADDITIONAL_EXCLUDED_TAG__` is not used
|
|
# anywhere outside of splicing it in to the command below.
|
|
__ADDITIONAL_EXCLUDED_TAG__: ${{ !inputs.force-run && ':mb/old-migrations-test' || '' }}
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Prepare front-end environment
|
|
uses: ./.github/actions/prepare-frontend
|
|
- name: Prepare back-end environment
|
|
uses: ./.github/actions/prepare-backend
|
|
with:
|
|
java-version: "${{ matrix.java-version }}"
|
|
- name: Build static-viz frontend
|
|
run: bun run build-static-viz
|
|
env:
|
|
MB_EDITION: ${{ matrix.job.edition }}
|
|
|
|
- name: Resolve current job id
|
|
# Resolve JOB_ID before the long test step so the ci-conductor reporter
|
|
# below can link failures to this exact job. Best-effort.
|
|
uses: ./.github/actions/resolve-job-id
|
|
|
|
# Granular rerun: on a re-run of a failed job, restore the list of tests
|
|
# that failed on the previous attempt (uploaded below) so the test step
|
|
# can target only those instead of the whole partition. Best-effort: a
|
|
# missing/empty artifact leaves no file and the run falls back to the full
|
|
# partition. First attempts skip this entirely.
|
|
#
|
|
# Scoped to the IMMEDIATELY preceding attempt: an attempt that passed, or
|
|
# that failed with nothing attributable to a var, uploads no artifact, and
|
|
# a narrow set must not outlive it. Reading `attempt - 1` means "only what
|
|
# the last attempt actually recorded" — anything else falls back to the
|
|
# full partition. Workflow expressions have no arithmetic, so the shell
|
|
# computes it.
|
|
- name: Resolve previous attempt
|
|
if: ${{ github.run_attempt != '1' }}
|
|
run: echo "RERUN_PREV_ATTEMPT=$((GITHUB_RUN_ATTEMPT - 1))" >> "$GITHUB_ENV"
|
|
|
|
# `pattern` (not `name`) because the artifact is legitimately absent whenever the
|
|
# previous attempt recorded nothing: `name` treats that as an error, `pattern`
|
|
# downloads nothing and moves on. The value is a literal — no glob metacharacters —
|
|
# so it matches exactly one artifact, which lands at `target/failed-tests`. A pattern
|
|
# that ever matched two would nest them under their artifact names instead, leaving
|
|
# the file absent and falling back to the full partition.
|
|
- name: Restore previously-failed tests
|
|
if: ${{ github.run_attempt != '1' }}
|
|
continue-on-error: true
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
pattern: failed-tests-be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}-attempt-${{ env.RERUN_PREV_ATTEMPT }}
|
|
path: target
|
|
|
|
- name: "Test Java ${{ matrix.java-version }} ${{ matrix.job.name }}"
|
|
id: run-java-tests
|
|
# On a rerun with a non-empty failed-tests file, target only those vars
|
|
# via hawk `:only`, dropping the tag/partition args — the listed vars are
|
|
# exactly this job's failures, already filtered and partitioned on the
|
|
# first attempt. Otherwise run exactly as before. Quote the file so the
|
|
# whole `[ns/var ...]` vector reaches clojure -X as a single argument.
|
|
run: |
|
|
BASE="clojure -X:dev:ci:test:${{ matrix.job.edition }}:${{ matrix.job.edition }}-dev"
|
|
if [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; then
|
|
echo "Granular rerun of previously-failed tests: $(cat target/failed-tests)"
|
|
$BASE :only "$(cat target/failed-tests)"
|
|
else
|
|
$BASE :exclude-tags '[:mb/driver-tests ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]' ${{ matrix.job.test-args }}
|
|
fi
|
|
# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.
|
|
continue-on-error: true
|
|
|
|
- name: Report failed tests to ci-conductor
|
|
if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}
|
|
# This step should NEVER fail the job!
|
|
continue-on-error: true
|
|
uses: ./.github/actions/report-failures-to-ci-conductor
|
|
with:
|
|
test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}
|
|
adapter: backend
|
|
|
|
# Granular rerun: record which tests failed so a re-run of this job targets
|
|
# only those (restored above). Reuses the backend JUnit adapter from the
|
|
# ci-conductor sources the report step just prepared. Best-effort: writes
|
|
# nothing unless every failure is attributable to a var, so an
|
|
# unattributable failure falls back to a full partition rerun.
|
|
- name: Collect failed tests for rerun
|
|
if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}
|
|
continue-on-error: true
|
|
# Clear the restored set first, so a collector that dies before deciding
|
|
# can't leave the previous attempt's file to be uploaded as this one's.
|
|
run: |
|
|
rm -f target/failed-tests
|
|
bun release/ci-conductor/src/collect-failed-backend.ts
|
|
|
|
- name: Upload failed tests for rerun
|
|
if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: failed-tests-be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}-attempt-${{ github.run_attempt }}
|
|
path: target/failed-tests
|
|
if-no-files-found: ignore
|
|
|
|
- name: Quarantine gate
|
|
if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}
|
|
continue-on-error: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
|
|
timeout-minutes: 3
|
|
uses: ./.github/actions/quarantine-gate
|
|
with:
|
|
adapter: backend
|
|
test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}
|
|
enforce-quarantine: ${{ vars.QUARANTINE_ENGINE == 'ci-conductor' }}
|
|
|
|
- name: Publish Test Report (JUnit)
|
|
uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
|
|
if: failure()
|
|
with:
|
|
path: "target/junit/**/*_test.xml"
|
|
name: JUnit Test Report be-tests-java-${{ matrix.java-version }}-${{ matrix.job.edition }}
|
|
reporter: java-junit
|
|
list-suites: failed
|
|
list-tests: failed
|
|
fail-on-error: false
|
|
|
|
- name: Upload Logs on Failure
|
|
if: always() && !success()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: test-logs-${{ matrix.java-version }}-${{ matrix.job.edition }}-${{ github.job }}-${{ matrix.job.test-group-name }}
|
|
path: logs/test-log.json
|
|
retention-days: 1
|
|
|
|
# checks that all the namespaces we actually ship can be compiled, without any dependencies that we don't ship (such
|
|
# as `:dev` dependencies). See #27009 for more context.
|
|
be-check:
|
|
if: ${{ !inputs.skip }}
|
|
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
|
|
timeout-minutes: 10
|
|
strategy:
|
|
matrix:
|
|
java-version: [25]
|
|
name: "Check Java ${{ matrix.java-version }}"
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Prepare backend
|
|
uses: ./.github/actions/prepare-backend
|
|
# TODO -- we should probably also check WITHOUT ee -- do a check with only OSS namespaces on the classpath
|
|
- name: Check namespaces
|
|
run: clojure -M:ee:drivers:load-namespaces
|
|
|
|
be-cljfmt:
|
|
if: ${{ !inputs.skip }}
|
|
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
|
|
timeout-minutes: 10
|
|
name: Cljfmt
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Prepare back-end environment
|
|
uses: ./.github/actions/prepare-backend
|
|
- name: Run cljfmt
|
|
run: ./bin/mage cljfmt-all --force-check
|
|
|
|
be-tests-result:
|
|
needs:
|
|
- be-linter-eastwood
|
|
- be-linter-eastwood-test
|
|
- be-linter-eastwood-test-changed
|
|
- be-linter-clj-kondo
|
|
- be-cljfmt
|
|
- be-check
|
|
- be-tests
|
|
if: always() && !cancelled()
|
|
runs-on: ${{ vars.SLIM_RUNNER_KEY }}
|
|
timeout-minutes: 5
|
|
env:
|
|
needs: ${{ toJson(needs) }}
|
|
steps:
|
|
- name: Check backend job status
|
|
uses: actions/github-script@v9
|
|
env:
|
|
needs: ${{ toJson(needs) }}
|
|
with:
|
|
script: | # js
|
|
const needs = JSON.parse(process.env.needs);
|
|
const jobs = Object.entries(needs).map(
|
|
([jobName, jobValues]) => ({
|
|
name: jobName,
|
|
result: jobValues.result
|
|
}));
|
|
|
|
// are all jobs skipped or successful?
|
|
if (jobs.every(job => (job.result === 'skipped' || job.result === 'success'))) {
|
|
console.log('All jobs are skipped or successful');
|
|
process.exit(0);
|
|
}
|
|
|
|
// otherwise, something failed
|
|
console.log('Some backend jobs failed');
|
|
jobs.forEach((job) => {
|
|
if (job.result !== 'success') {
|
|
console.log(`${job.name} - ${job.result}`);
|
|
}
|
|
});
|
|
process.exit(1);
|