gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-22 04:16:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: harden test selection and nightly coverage (#6341)

Browse Source
This commit is contained in:
Zhengchao An
2026-08-21 23:04:08 +08:00
committed by GitHub
parent bce5922aef
commit bc07cfd115
31 changed files with 1146 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/s3tests/README.md
+3 -4
View File
@@ -46,10 +46,9 @@ lists when upstream changes.
the PR.
- **Weekly + manual**: `.github/workflows/e2e-s3tests.yml` runs the full
upstream suite (`TEST_SCOPE=all`) against a Docker deployment (single node
or a 4-node distributed cluster behind HAProxy). It fails only on
regressions in the implemented whitelist and publishes a classification
report (`compat-report.md`, also shown in the job summary) listing promotion
candidates and unclassified tests.
or a 4-node distributed cluster behind HAProxy). The canonical gate policy
and compatibility-report behavior are documented in
[`scripts/s3-tests/README.md`](../../scripts/s3-tests/README.md).
## Running Tests Locally
.github/workflows/ci-docs-only.yml
+3
View File
@@ -125,6 +125,9 @@ jobs:
- name: Check no embedded secret material
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
.github/workflows/ci.yml
+18 -5
View File
@@ -160,6 +160,9 @@ jobs:
- name: Check no embedded secret material
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
@@ -686,9 +689,9 @@ jobs:
- name: Make binary executable
run: chmod +x ./target/debug/rustfs
# Build the e2e test graph once. The archive is reused by the security
# count-floor check and the smoke run below, avoiding a second compile of
# the same e2e_test target on cold runners (backlog#1645).
# Build the e2e test graph once. The archive is reused by the smoke
# selection guard, security exact-count check, and run below, avoiding a
# second compile of the same e2e_test target on cold runners (backlog#1645).
- name: Archive e2e smoke test binaries
env:
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
@@ -696,6 +699,7 @@ jobs:
run: |
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
@@ -760,7 +764,7 @@ jobs:
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
# every PR, so it is gated to main pushes, the merge queue, and manual
# dispatch. protocols / the 6 cluster suites / replication / #[ignore] are
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
if: >-
github.event_name == 'workflow_dispatch' ||
@@ -820,6 +824,13 @@ jobs:
- name: Make binary executable
run: chmod +x ./target/debug/rustfs
- name: Verify e2e full membership
env:
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
run: |
cargo nextest list --profile e2e-full -p e2e_test --message-format json > "${NEXTEST_LISTING}"
python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"
# Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
# default-filter in .config/nextest.toml is the single wiring mechanism —
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
@@ -832,7 +843,9 @@ jobs:
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: e2e-full-junit-${{ github.run_number }}
path: target/nextest/e2e-full/junit.xml
path: |
target/nextest/e2e-full/junit.xml
${{ runner.temp }}/rustfs-e2e-full-list.json
retention-days: 7
e2e-tests-rio-v2:
.github/workflows/e2e-replication-nightly.yml
+122 -11
View File
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Nightly full replication e2e lane (backlog#1147 repl-1, deps: ci-4).
# Consolidated nightly e2e lane for replication, cluster faults, and protocols.
#
# The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the
# FAST replication tests. This scheduled lane runs the remaining heavier
@@ -28,15 +28,12 @@
# add ad-hoc cargo-test steps here; change the filterset instead. The
# authoritative membership and count come from
# `cargo nextest list -p e2e_test --profile e2e-repl-nightly`; the PR/nightly
# count invariant is maintained next to the filtersets in .config/nextest.toml
# (deliberately not duplicated here).
# selection digest is committed under .config/.
#
# Explicit division of labor: the nightly subset runs ONLY here, never double-run
# in ci-5's future e2e-full merge gate. TODO(ci-7): once the ci domain's
# consolidated scheduled e2e workflow exists, fold this interim repl-owned lane
# into it rather than growing a second scheduled entrypoint.
# Explicit division of labor: these subsets run only here and never double-run
# in the e2e-full merge gate.
name: e2e-replication-nightly
name: e2e-nightly
on:
workflow_dispatch:
@@ -50,6 +47,10 @@ on:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
jobs:
repl-nightly:
name: Replication e2e (nightly)
@@ -97,9 +98,20 @@ jobs:
# demand otherwise, but a single explicit build avoids several parallel
# nextest test processes racing to build it at once.
- name: Build rustfs binary
run: cargo build -p rustfs --bins
run: |
cargo build -p rustfs --bins
: > target/debug/rustfs.features
- name: Verify replication e2e membership
env:
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
run: |
cargo nextest list --profile e2e-repl-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
python3 ./scripts/check_test_wiring.py --check-profile e2e-repl-nightly "${NEXTEST_LISTING}"
- name: Run replication e2e nightly suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
- name: Upload nextest junit report
@@ -107,13 +119,112 @@ jobs:
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: e2e-replication-nightly-junit-${{ github.run_number }}
path: target/nextest/e2e-repl-nightly/junit.xml
path: |
target/nextest/e2e-repl-nightly/junit.xml
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
${{ runner.temp }}/rustfs-e2e-repl-nightly-logs/
retention-days: 7
if-no-files-found: ignore
cluster-nightly:
name: Cluster fault e2e (nightly)
runs-on: sm-standard-4
timeout-minutes: 90
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-e2e-nightly
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Build rustfs binary
run: |
cargo build -p rustfs --bins --features e2e-test-hooks
: > target/debug/rustfs.features
- name: Verify cluster fault e2e membership
env:
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
run: |
cargo nextest list --profile e2e-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
python3 ./scripts/check_test_wiring.py --check-profile e2e-nightly "${NEXTEST_LISTING}"
- name: Run cluster fault e2e nightly suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
run: cargo nextest run --profile e2e-nightly -p e2e_test
- name: Upload cluster fault diagnostics
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: e2e-cluster-nightly-${{ github.run_number }}
path: |
target/nextest/e2e-nightly/junit.xml
${{ runner.temp }}/rustfs-e2e-nightly-list.json
${{ runner.temp }}/rustfs-e2e-nightly-logs/
retention-days: 7
if-no-files-found: warn
protocols-nightly:
name: Protocol e2e (nightly)
runs-on: sm-standard-4
timeout-minutes: 90
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-e2e-protocols
cache-save-if: 'false'
install-build-packaging-tools: 'false'
# The suite owns fixed protocol ports and serializes its internal cases.
- name: Verify protocol e2e membership
env:
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-protocols-list.json
run: |
cargo nextest list --profile e2e-protocols -p e2e_test --message-format json > "${NEXTEST_LISTING}"
python3 ./scripts/check_test_wiring.py --check-profile e2e-protocols "${NEXTEST_LISTING}"
- name: Run protocol e2e nightly suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
run: >-
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
- name: Upload protocol diagnostics
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: e2e-protocol-nightly-${{ github.run_number }}
path: |
target/nextest/e2e-protocols/junit.xml
${{ runner.temp }}/rustfs-e2e-protocols-list.json
${{ runner.temp }}/rustfs-protocol-e2e-logs/
retention-days: 7
if-no-files-found: warn
alert-on-failure:
name: Alert on scheduled failure
needs: [repl-nightly]
needs: [repl-nightly, cluster-nightly, protocols-nightly]
# Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
# manual workflow_dispatch runs stay quiet so a debugging run never files a
# spurious alert.
.github/workflows/e2e-s3tests.yml
+28 -35
View File
@@ -18,10 +18,9 @@
# runs only the implemented_tests.txt whitelist. This workflow complements it:
#
# - Scheduled weekly full sweep (TEST_SCOPE=all): runs the ENTIRE upstream
# suite and reports promotion candidates (tests that newly pass) and
# unclassified tests. The job fails only on regressions in the implemented
# whitelist or on infrastructure errors — expected failures from
# not-yet-implemented features do not turn the run red.
# suite and reports promotion candidates. Regressions, unclassified tests,
# incomplete execution, and infrastructure errors fail the job; classified
# failures for not-yet-implemented features remain informational.
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
#
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
@@ -45,13 +44,6 @@
# The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
# via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.
# DISABLED. This workflow is switched off in the repository's Actions settings
# (state: disabled_manually) and does not run on any trigger, including its cron
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
# reading this file, which has already misled at least one audit — hence this
# banner. Re-enabling is a UI action; anyone doing so should first check that the
# workflow still matches the current CI layout. See rustfs/backlog#1603.
#
name: e2e-s3tests
on:
@@ -81,6 +73,19 @@ on:
description: "Stop after N failures. '0' to run everything."
required: false
default: "0"
shard-count:
description: "Exact-node-ID shard count for a targeted manual run"
required: false
default: "1"
type: choice
options:
- "1"
- "2"
- "4"
shard-index:
description: "Zero-based shard index for a targeted manual run"
required: false
default: "0"
markexpr:
description: "Optional pytest -m expression"
required: false
@@ -111,6 +116,8 @@ env:
XDIST: ${{ github.event.inputs.xdist || '4' }}
MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }}
TEST_TIMEOUT: "300"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
@@ -127,19 +134,22 @@ defaults:
jobs:
s3tests:
name: s3tests (${{ matrix.test-mode }}, shard ${{ matrix.shard-index }})
# GitHub-hosted: reliably provides Docker + docker compose + python3/pip.
# See the header note (ci-1) for why the self-hosted sm-standard-4 label
# was abandoned. TODO(ci-8): scheduled-failure alerting (auto-open issue)
# is added by the ci-8 composite action; do not implement it here.
# was abandoned. Scheduled failures are handled by alert-on-failure below.
runs-on: ubuntu-latest
timeout-minutes: 180
strategy:
fail-fast: false
max-parallel: 2
matrix:
# Scheduled sweeps cover both topologies; manual runs use the input.
test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
shard-index: ${{ github.event_name == 'schedule' && fromJSON('[0, 1, 2, 3]') || fromJSON(format('[{0}]', github.event.inputs.shard-index || '0')) }}
env:
TEST_MODE: ${{ matrix.test-mode }}
S3_SHARD_INDEX: ${{ matrix.shard-index }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
@@ -181,6 +191,7 @@ jobs:
- name: Start single RustFS
if: env.TEST_MODE == 'single'
run: |
SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
docker rm -f rustfs-single >/dev/null 2>&1 || true
# The four disks share one physical device on the runner (a single
@@ -193,6 +204,7 @@ jobs:
-e RUSTFS_ADDRESS=0.0.0.0:9000 \
-e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
-e RUSTFS_SSE_S3_MASTER_KEY="${SSE_KEY}" \
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
-v /tmp/rustfs-single:/data \
@@ -201,6 +213,7 @@ jobs:
- name: Start 4-node distributed cluster
if: env.TEST_MODE == 'multi'
run: |
SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
# A real distributed deployment: every node lists all endpoints in
# RUSTFS_VOLUMES so data is erasure-coded ACROSS nodes. Do not use
# node-local volume paths here — that would create four independent
@@ -213,6 +226,7 @@ jobs:
RUSTFS_ADDRESS: "0.0.0.0:9000"
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
RUSTFS_SSE_S3_MASTER_KEY: "${SSE_KEY}"
RUSTFS_VOLUMES: "http://rustfs{1...4}:9000/data/rustfs{0...3}"
# Each node's four disks share one physical device inside its
# container, so bypass the local physical-disk-independence guard
@@ -294,7 +308,6 @@ jobs:
- name: Run ceph s3-tests
run: |
set +e
DEPLOY_MODE=existing \
TEST_MODE="${TEST_MODE}" \
TEST_SCOPE="${TEST_SCOPE}" \
@@ -302,26 +315,6 @@ jobs:
MAXFAIL="${MAXFAIL}" \
MARKEXPR="${MARKEXPR}" \
./scripts/s3-tests/run.sh
RC=$?
set -e
if [ "${TEST_SCOPE}" = "implemented" ]; then
# Whitelist run: every failure is a regression.
exit "${RC}"
fi
# Full sweep: failures outside the implemented whitelist are
# inventory (promotion candidates / unimplemented features), not a
# gate. Fail only on whitelist regressions or infrastructure errors.
JUNIT="artifacts/s3tests-${TEST_MODE}/junit.xml"
if [ ! -f "${JUNIT}" ]; then
echo "No junit.xml produced — infrastructure failure (exit ${RC})" >&2
exit "${RC}"
fi
python3 scripts/s3-tests/report_compat.py \
--junit "${JUNIT}" \
--lists-dir scripts/s3-tests \
--fail-on-regression
- name: Publish compatibility report
if: always()
@@ -346,7 +339,7 @@ jobs:
if: always() && env.ACT != 'true'
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: s3tests-${{ env.TEST_MODE }}
name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
path: artifacts/**
alert-on-failure:
.github/workflows/fuzz.yml
+11 -24
View File
@@ -12,27 +12,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# DISABLED. This workflow is switched off in the repository's Actions settings
# (state: disabled_manually) and does not run on any trigger, including its cron
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
# reading this file, which has already misled at least one audit — hence this
# banner. Re-enabling is a UI action; anyone doing so should first check that the
# workflow still matches the current CI layout. See rustfs/backlog#1603.
#
name: Fuzz
on:
pull_request:
types: [ opened, synchronize, reopened, closed ]
# PR trigger is intentionally narrow: only changes to the fuzz harness
# itself gate a PR. Broad crate paths (ecstore/filemeta/utils/policy/…)
# are covered by the nightly `schedule` run below, which fuzzes against
# whatever landed on main. Widening these paths previously queued a
# ~45min fuzz-build on nearly every PR and is why this workflow was
# disabled; do not re-add crate paths here.
# Run when the harness or any directly fuzzed production crate changes.
paths:
- "fuzz/**"
- "scripts/fuzz/**"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/policy/**"
- "crates/security-governance/**"
- "crates/utils/**"
- "Cargo.toml"
- "Cargo.lock"
- ".github/workflows/fuzz.yml"
schedule:
- cron: "0 2 * * *"
@@ -81,7 +76,7 @@ jobs:
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch'
runs-on: sm-standard-4
timeout-minutes: 45
timeout-minutes: 60
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
@@ -121,12 +116,7 @@ jobs:
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: fuzz-prebuilt-binaries-${{ github.run_number }}
path: |
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/archive_extract
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/bucket_validation
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/local_metadata
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/path_containment
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/policy_ingress
path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/
if-no-files-found: error
retention-days: 1
compression-level: 0
@@ -192,10 +182,7 @@ jobs:
nightly-fuzz-corpus:
name: "Nightly / ${{ matrix.target }}"
needs: fuzz-build
# TODO(ci-8): when the schedule-failure-issue composite action lands,
# add a step here (or a dependent job) that opens/updates a GitHub issue
# on nightly failure. ci-8 is the single alerting mechanism for all
# scheduled workflows; do not self-roll alerting in this workflow.
# Scheduled failures are handled by alert-on-failure below.
if: >
github.event_name == 'schedule' ||
(github.event_name == 'workflow_dispatch' &&