Compare commits

..

2 Commits

Author SHA1 Message Date
overtrue b4533e6698 DO NOT MERGE: retrigger CI after base change 2026-11-11 11:28:59 +08:00
overtrue 7badcd501d DO NOT MERGE: deliberate mutations of reference.rs 2026-11-11 11:26:17 +08:00
20 changed files with 29 additions and 1809 deletions
-2
View File
@@ -66,7 +66,6 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_reference_binary.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/release -p "test_*.py"
@@ -93,7 +92,6 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
-20
View File
@@ -1,20 +0,0 @@
# Pinned RustFS release used as the "reference" target of parity lanes
# (.github/workflows/reference-parity.yml). This is the single place that names
# the reference release; .github/actions/reference-binary resolves, downloads and
# verifies it, and fails on a missing pin, a missing asset or a digest mismatch.
#
# Bumping the reference is a reviewed event: every per-test difference between
# the candidate and the reference is measured against this exact build.
#
# Each sha256 is the value published in the release's SHA256SUMS asset, which
# matches the digest GitHub reports for the same asset. The action checks both.
schema = 1
repository = "rustfs/rustfs"
version = "1.0.5"
tag = "1.0.5"
[platforms.linux-x86_64-gnu]
asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
sha256 = "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
executable = "rustfs"
-24
View File
@@ -1,24 +0,0 @@
# Targets of the s3-tests reference parity lane (.github/workflows/reference-parity.yml).
#
# Exactly one target uses `binary = "reference"` (the release pinned in
# .config/reference-binary.toml). Every other target uses `binary = "candidate"`
# (the binary built from the branch under test) and is compared per node id
# against the reference. All targets run the same harness, test list and server
# configuration; `env` adds only the target's own RUSTFS_* settings.
#
# Adding a target is one more [[targets]] table, for example:
#
# [[targets]]
# name = "candidate-gateway"
# binary = "candidate"
# env = { RUSTFS_S3_STACK = "gateway" }
schema = 1
[[targets]]
name = "reference"
binary = "reference"
[[targets]]
name = "candidate-legacy"
binary = "candidate"
-12
View File
@@ -1,12 +0,0 @@
# Tolerated per-test differences of the s3-tests reference parity lane.
#
# One entry per line, four fields separated by " | ":
#
# <target> | <pytest node id> | <tracking issue URL> | <reason>
#
# <target> is a candidate name from .config/reference-parity-targets.toml, the
# node id is exactly as the lane prints it (s3tests/functional/test_s3.py::...),
# and the issue URL is https://github.com/rustfs/<repo>/issues/<number>.
#
# Same semantics as scripts/junit_diff.py --allow: an entry whose difference no
# longer occurs is stale and fails the lane, so remove it when it is fixed.
@@ -1,96 +0,0 @@
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: "Reference RustFS binary"
description: "Download the pinned reference RustFS release, verify its sha256 and expose the executable path. A missing pin, missing asset or digest mismatch fails the step."
inputs:
pin-file:
description: "Pin file naming the reference release (repository-relative)"
required: false
default: ".config/reference-binary.toml"
platform:
description: "Platform table in the pin file"
required: false
default: "linux-x86_64-gnu"
destination:
description: "Directory that receives the executable; defaults to $RUNNER_TEMP/reference-binary"
required: false
default: ""
outputs:
path:
description: "Absolute path of the verified executable"
value: ${{ steps.unpack.outputs.path }}
version:
description: "Pinned reference version"
value: ${{ steps.pin.outputs.version }}
sha256:
description: "Pinned and verified archive sha256"
value: ${{ steps.pin.outputs.sha256 }}
runs:
using: "composite"
steps:
- name: Resolve reference pin
id: pin
shell: bash
env:
PIN_TOOL: ${{ github.action_path }}/reference_binary.py
PIN_FILE: ${{ inputs.pin-file }}
PIN_PLATFORM: ${{ inputs.platform }}
run: |
set -euo pipefail
resolved="$(python3 "$PIN_TOOL" resolve --pin "$PIN_FILE" --platform "$PIN_PLATFORM")"
printf '%s\n' "$resolved"
printf '%s\n' "$resolved" >> "$GITHUB_OUTPUT"
- name: Download and verify reference archive
id: download
shell: bash
env:
GH_TOKEN: ${{ github.token }}
ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
REF_REPOSITORY: ${{ steps.pin.outputs.repository }}
REF_TAG: ${{ steps.pin.outputs.tag }}
REF_ASSET: ${{ steps.pin.outputs.asset }}
REF_SHA256: ${{ steps.pin.outputs.sha256 }}
run: |
set -euo pipefail
archive="$RUNNER_TEMP/reference-archive/$REF_ASSET"
# Fails unless the release lists exactly one asset of this name whose
# published digest equals the pin and whose downloaded bytes match it.
python3 "$ASSET_DOWNLOADER" --repository "$REF_REPOSITORY" --tag "$REF_TAG" \
--asset "$REF_ASSET" --output "$archive" --sha256 "$REF_SHA256"
echo "archive=$archive" >> "$GITHUB_OUTPUT"
- name: Unpack reference executable
id: unpack
shell: bash
env:
PIN_TOOL: ${{ github.action_path }}/reference_binary.py
REF_ARCHIVE: ${{ steps.download.outputs.archive }}
REF_SHA256: ${{ steps.pin.outputs.sha256 }}
REF_EXECUTABLE: ${{ steps.pin.outputs.executable }}
REF_DESTINATION: ${{ inputs.destination }}
run: |
set -euo pipefail
destination="${REF_DESTINATION:-$RUNNER_TEMP/reference-binary}"
unpacked="$(python3 "$PIN_TOOL" unpack --archive "$REF_ARCHIVE" --sha256 "$REF_SHA256" \
--executable "$REF_EXECUTABLE" --destination "$destination")"
path="${unpacked#path=}"
test -x "$path"
echo "Reference executable: $path"
"$path" --version
echo "path=$path" >> "$GITHUB_OUTPUT"
@@ -1,144 +0,0 @@
#!/usr/bin/env python3
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Resolve and unpack the pinned reference RustFS release binary.
`resolve` reads the pin file and prints the selected platform's identity as
`key=value` lines (GitHub step-output format). `unpack` re-verifies the
downloaded archive digest against the pin and extracts exactly one executable.
Every missing or malformed input is a failure; nothing is skipped.
"""
import argparse
import hashlib
from pathlib import Path
import re
import sys
import tomllib
import zipfile
REPOSITORY = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*")
TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*")
PLATFORM = re.compile(r"[a-z0-9][a-z0-9_-]*")
ASSET = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*\.zip")
SHA256 = re.compile(r"[0-9a-f]{64}")
EXECUTABLE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
MAX_EXECUTABLE_BYTES = 1024 * 1024 * 1024
class PinError(Exception):
"""A pin, archive or argument that cannot be trusted."""
def field(table, key, pattern, where):
value = table.get(key)
if not isinstance(value, str) or not pattern.fullmatch(value):
raise PinError(f"{where}: missing or invalid '{key}'")
return value
def resolve(pin_path, platform):
if not PLATFORM.fullmatch(platform or ""):
raise PinError("invalid platform name")
try:
data = tomllib.loads(Path(pin_path).read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise PinError(f"{pin_path}: reference pin file is missing") from exc
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
raise PinError(f"{pin_path}: cannot read reference pin: {exc}") from exc
if data.get("schema") != 1 or type(data.get("schema")) is not int:
raise PinError(f"{pin_path}: unsupported pin schema")
identity = {
"repository": field(data, "repository", REPOSITORY, pin_path),
"version": field(data, "version", TAG, pin_path),
"tag": field(data, "tag", TAG, pin_path),
}
platforms = data.get("platforms")
if not isinstance(platforms, dict) or not platforms:
raise PinError(f"{pin_path}: no pinned platforms")
entry = platforms.get(platform)
if not isinstance(entry, dict):
raise PinError(f"{pin_path}: no pin for platform '{platform}'")
where = f"{pin_path} [platforms.{platform}]"
identity["asset"] = field(entry, "asset", ASSET, where)
identity["sha256"] = field(entry, "sha256", SHA256, where)
identity["executable"] = field(entry, "executable", EXECUTABLE, where)
if identity["version"] not in identity["asset"]:
raise PinError(f"{where}: asset name does not carry version {identity['version']}")
unknown = set(entry) - {"asset", "sha256", "executable"}
if unknown:
raise PinError(f"{where}: unknown keys {sorted(unknown)}")
return identity
def file_sha256(path):
with open(path, "rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def unpack(archive, sha256, executable, destination):
if not SHA256.fullmatch(sha256 or "") or not EXECUTABLE.fullmatch(executable or ""):
raise PinError("invalid expected digest or executable name")
try:
actual = file_sha256(archive)
except OSError as exc:
raise PinError(f"{archive}: reference archive is missing or unreadable") from exc
if actual != sha256:
raise PinError(f"{archive}: sha256 {actual} does not match the pinned digest")
try:
with zipfile.ZipFile(archive) as bundle:
matches = [info for info in bundle.infolist() if info.filename == executable]
if len(matches) != 1:
raise PinError(f"{archive}: must contain exactly one top-level '{executable}'")
info = matches[0]
if info.is_dir() or not 0 < info.file_size <= MAX_EXECUTABLE_BYTES:
raise PinError(f"{archive}: '{executable}' is not a plausible executable")
destination = Path(destination)
destination.mkdir(parents=True, exist_ok=True)
target = destination / executable
with bundle.open(info) as source, open(target, "wb") as sink:
while chunk := source.read(1024 * 1024):
sink.write(chunk)
except zipfile.BadZipFile as exc:
raise PinError(f"{archive}: not a valid zip archive") from exc
target.chmod(0o755)
return target
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
sub = parser.add_subparsers(dest="mode", required=True)
res = sub.add_parser("resolve")
res.add_argument("--pin", required=True)
res.add_argument("--platform", required=True)
unp = sub.add_parser("unpack")
unp.add_argument("--archive", required=True)
unp.add_argument("--sha256", required=True)
unp.add_argument("--executable", required=True)
unp.add_argument("--destination", required=True)
args = parser.parse_args(argv)
try:
if args.mode == "resolve":
for key, value in resolve(args.pin, args.platform).items():
print(f"{key}={value}")
else:
print(f"path={unpack(args.archive, args.sha256, args.executable, args.destination)}")
except PinError as exc:
print(f"reference binary: {exc}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
-27
View File
@@ -869,33 +869,6 @@
"failure_owner": "watchdog",
"origin": "baseline"
},
{
"workflow": ".github/workflows/reference-parity.yml",
"name": "Reference Parity",
"roles": [
"pr",
"periodic",
"manual"
],
"events": [
"pull_request",
"schedule",
"workflow_dispatch"
],
"cadence": "nightly",
"timezone": "UTC",
"schedule": [
"37 1 * * *"
],
"owner": "@overtrue",
"policy": "active",
"monitor": "workflow",
"max_age_hours": 36,
"result_owner": "reference-parity.yml",
"disposition": "s3-tests per-node-id parity of branch candidates against the pinned reference release",
"failure_owner": "watchdog",
"origin": "added"
},
{
"workflow": ".github/workflows/functional-chain-health.yml",
"name": "Functional chain health",
-444
View File
@@ -1,444 +0,0 @@
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# s3-tests reference parity lane (rustfs/backlog#2773).
#
# Runs the ceph/s3-tests implemented list (scripts/s3-tests/implemented_tests.txt)
# against every target in .config/reference-parity-targets.toml and fails on
# any per-node-id difference between a candidate and the reference:
#
# - reference: the release pinned in .config/reference-binary.toml, downloaded
# and sha256-verified by .github/actions/reference-binary;
# - candidate targets: the debug binary built from this checkout exactly as
# ci.yml's build-rustfs-debug-binary job builds it for S3 Implemented Tests.
#
# Every target runs the same harness (scripts/s3-tests/run.sh from this
# checkout), test list and server configuration on its own hosted runner. A
# target's failures get one rerun against a fresh server; a case that passes it
# is flaky, not a difference. scripts/s3-tests/reference_parity.py judges the
# result and honours .config/reference-parity-tolerated.txt, whose entries must
# each match a real difference.
#
# The `differential` job adds the 73-operation dual-stack differential
# (crates/e2e_test/src/dual_stack_differential, rustfs/backlog#2740) against the
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY,
# started without RUSTFS_S3_STACK), side B is the candidate built above on its
# legacy stack. It reuses the candidate build artifact, never builds the candidate
# twice, and prints reference_side, version_a and version_b from the report the test
# wrote; it fails when side A was not the reference or both sides report the same
# version. Every (op, field, a, b) difference goes to the job summary. Report-only
# until the first result has been reviewed.
#
# Report-only: not part of the Test and Lint aggregate. Scheduled failures are
# owned by the scheduled-validation watchdog (.github/scheduled-validations.json).
name: Reference Parity
on:
pull_request:
paths:
- ".github/workflows/reference-parity.yml"
- ".github/actions/reference-binary/**"
- ".config/reference-binary.toml"
- ".config/reference-parity-targets.toml"
- ".config/reference-parity-tolerated.txt"
- "scripts/s3-tests/reference_parity.py"
- "scripts/junit_diff.py"
- "crates/e2e_test/src/dual_stack_differential/**"
schedule:
- cron: "37 1 * * *"
workflow_dispatch:
# Matches ci.yml: these variables are hashed into the rust-cache key, so the
# candidate build reads the same ci-dev cache as build-rustfs-debug-binary.
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
shell: bash
jobs:
plan:
name: Plan parity targets
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
targets: ${{ steps.targets.outputs.targets }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Validate target list
id: targets
run: |
set -euo pipefail
targets="$(python3 scripts/s3-tests/reference_parity.py targets --config .config/reference-parity-targets.toml)"
echo "targets=${targets}" >> "$GITHUB_OUTPUT"
echo "Parity targets: ${targets}"
build-candidate:
name: Build candidate binary
# Critical-path CPU and cache ownership: the same debug build as ci.yml's
# build-rustfs-debug-binary, reading the ci-dev cache written on sm-standard-4.
runs-on: sm-standard-4
timeout-minutes: 60
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Build debug binary
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
- name: Upload candidate binary
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: reference-parity-candidate-binary
overwrite: true
path: |
target/debug/rustfs
target/debug/rustfs.e2e.json
if-no-files-found: error
retention-days: 1
s3tests:
name: s3-tests (${{ matrix.target.name }})
needs: [ plan, build-candidate ]
runs-on: ubuntu-latest
timeout-minutes: 75
strategy:
fail-fast: false
matrix:
target: ${{ fromJSON(needs.plan.outputs.targets) }}
env:
TARGET_NAME: ${{ matrix.target.name }}
TARGET_KIND: ${{ matrix.target.binary }}
# Above this many first-run failures the target is systemically broken;
# a rerun would only repeat it, so the first run is judged as is.
RERUN_CAP: "50"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Fetch pinned reference binary
id: reference
if: matrix.target.binary == 'reference'
uses: ./.github/actions/reference-binary
- name: Download candidate binary
if: matrix.target.binary == 'candidate'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: reference-parity-candidate-binary
path: target/debug
- name: Select target binary
env:
REFERENCE_PATH: ${{ steps.reference.outputs.path }}
run: |
set -euo pipefail
case "$TARGET_KIND" in
reference) binary="$REFERENCE_PATH" ;;
candidate) chmod +x target/debug/rustfs; binary="$PWD/target/debug/rustfs" ;;
*) echo "unknown target binary kind: $TARGET_KIND" >&2; exit 1 ;;
esac
test -x "$binary"
echo "TARGET_BINARY=$binary" >> "$GITHUB_ENV"
mkdir -p "parity/$TARGET_NAME"
{
echo "target=$TARGET_NAME"
echo "binary=$TARGET_KIND"
echo "sha256=$(sha256sum "$binary" | cut -d' ' -f1)"
echo "version=$("$binary" --version | tr '\n' ' ')"
} | tee "parity/$TARGET_NAME/target.txt"
- name: Apply target environment
env:
TARGET_JSON: ${{ toJSON(matrix.target) }}
run: |
set -euo pipefail
python3 scripts/s3-tests/reference_parity.py export-env --target-json "$TARGET_JSON" \
| tee -a "$GITHUB_ENV" "parity/$TARGET_NAME/target-env.txt"
- name: Run implemented s3-tests
run: |
set -uo pipefail
report="parity/$TARGET_NAME"
harness="artifacts/s3tests-single"
# Same invocation as ci.yml's S3 Implemented Tests, with a fresh data
# root and port per run. $1 optionally replaces the test list.
run_suite() {
local list="$1" label="$2" root port
root="${RUNNER_TEMP}/rustfs-s3tests-${label}"
rm -rf "$harness" "$root"
mkdir -p "$root"
port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
env DEPLOY_MODE=binary \
RUSTFS_BINARY="$TARGET_BINARY" \
TEST_MODE=single \
MAXFAIL=0 \
XDIST=4 \
S3_PORT="$port" \
DATA_ROOT="$root" \
S3TESTS_CONF="$harness/s3tests.conf" \
${list:+IMPLEMENTED_TESTS_FILE="$list"} \
./scripts/s3-tests/run.sh
}
# A run is usable only if it produced a JUnit report; test failures
# are data for the comparison, a missing report is a lane failure.
keep() {
local prefix="$1"
for file in pytest.log compat-report.md; do
if [[ -f "$harness/$file" ]]; then cp "$harness/$file" "$report/${prefix}${file}"; fi
done
if [[ -f "$harness/rustfs-single/rustfs.log" ]]; then cp "$harness/rustfs-single/rustfs.log" "$report/${prefix}rustfs.log"; fi
test -s "$harness/junit.xml" || { echo "::error::$TARGET_NAME produced no JUnit report (${prefix:-first} run)"; return 1; }
cp "$harness/junit.xml" "$report/${prefix}junit.xml"
}
run_suite "" first
echo "first_run_exit=$?" >> "$report/target.txt"
keep "" || exit 1
cp "$harness/selected-nodeids.txt" "$report/selected-nodeids.txt" || exit 1
failures="$(python3 scripts/s3-tests/reference_parity.py failed --report "$report")" || exit 1
count="$(printf '%s' "$failures" | grep -c . || true)"
echo "first_run_failures=$count" >> "$report/target.txt"
if (( count == 0 )); then
exit 0
fi
if (( count > RERUN_CAP )); then
echo "::warning::$TARGET_NAME has $count failures (cap $RERUN_CAP); judging the first run without a rerun"
echo "rerun=skipped-over-cap" >> "$report/target.txt"
exit 0
fi
printf '%s\n' "$failures" > "$RUNNER_TEMP/rerun-tests.txt"
run_suite "$RUNNER_TEMP/rerun-tests.txt" rerun
echo "rerun_exit=$?" >> "$report/target.txt"
keep rerun- || exit 1
- name: Upload target report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: reference-parity-report-${{ matrix.target.name }}
overwrite: true
path: parity/${{ matrix.target.name }}/
if-no-files-found: error
retention-days: 14
differential:
name: Dual-stack differential (reference vs candidate)
needs: [ build-candidate ]
# Compiles the e2e_test harness against the ci-dev cache like the other
# sm-standard-4 jobs; the candidate server itself is the downloaded artifact.
runs-on: sm-standard-4
timeout-minutes: 90
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
NO_PROXY: 127.0.0.1,localhost,::1
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Fetch pinned reference binary
id: reference
uses: ./.github/actions/reference-binary
- name: Download candidate binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: reference-parity-candidate-binary
path: target/debug
- name: Run dual-stack differential
id: differential
continue-on-error: true
env:
REFERENCE_PATH: ${{ steps.reference.outputs.path }}
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-differential-logs
run: |
set -euo pipefail
export no_proxy="$NO_PROXY"
chmod +x target/debug/rustfs
test -x "$REFERENCE_PATH"
mkdir -p "$RUSTFS_E2E_LOG_DIR"
# Side A: the released binary. Side B: the candidate on its legacy stack.
export RUSTFS_E2E_REFERENCE_BINARY="$PWD/target/debug/rustfs"
test_name='dual_stack_differential::tests::both_stacks_answer_every_operation_identically'
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
cargo test --locked -p e2e_test --lib "$test_name" -- --exact --nocapture \
2>&1 | tee "$RUNNER_TEMP/differential-test.log"
- name: Summarize report and verify the reference side
if: always()
run: |
set -euo pipefail
python3 - <<'PY'
import glob, json, os, sys
summary = os.environ["GITHUB_STEP_SUMMARY"]
reports = sorted(glob.glob("target/dual-stack/self-comparison-*.json"))
lines = ["## Dual-stack differential: reference vs candidate", ""]
failures = []
if len(reports) != 1:
failures.append(f"expected exactly one self-comparison report, found {len(reports)}: {reports}")
else:
report = json.load(open(reports[0]))
side = report.get("reference_side")
version_a = report.get("version_a")
version_b = report.get("version_b")
lines += [
f"- report: `{reports[0]}`",
f"- reference_side: `{side}`",
f"- version_a: `{(version_a or '').splitlines()[0] if version_a else version_a}`",
f"- version_b: `{(version_b or '').splitlines()[0] if version_b else version_b}`",
f"- stack_a: `{report.get('stack_a')}`, stack_b: `{report.get('stack_b')}`",
f"- compared_ops: {report.get('compared_ops')}, compared_cases: {report.get('compared_cases')}",
"",
]
print(f"reference_side={side}\nversion_a={version_a}\nversion_b={version_b}")
lines += ["```", "version_a:", version_a or "", "", "version_b:", version_b or "", "```", ""]
if side != "a":
failures.append(f"reference_side is {side!r}, expected 'a'")
if not version_a or not version_b:
failures.append("version_a or version_b is missing")
elif version_a == version_b:
failures.append("version_a equals version_b: the reference was replaced by the candidate")
diffs = report.get("differences") or []
lines.append(f"### Differences: {len(diffs)}")
by_op = {}
for d in diffs:
by_op.setdefault(d["op"], []).append(d)
for op in sorted(by_op):
lines += ["", f"#### {op}", "", "| case | field | a (reference) | b (candidate) |", "| --- | --- | --- | --- |"]
for d in by_op[op]:
cell = lambda v: "`" + str(v).replace("|", "\\|").replace("`", "'") + "`"
lines.append(f"| {d['case']} | {d['field']} | {cell(d['a'])} | {cell(d['b'])} |")
if failures:
lines += ["", "### Observation check failed", ""] + [f"- {f}" for f in failures]
open(summary, "a").write("\n".join(lines) + "\n")
print("\n".join(lines))
for f in failures:
print(f"::error::{f}")
sys.exit(1 if failures else 0)
PY
- name: Upload differential reports
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: reference-differential-report
overwrite: true
path: |
target/dual-stack/*.json
${{ runner.temp }}/differential-test.log
if-no-files-found: warn
retention-days: 14
- name: Fail when the differential found differences
if: steps.differential.outcome == 'failure'
run: |
echo "::error::the dual-stack differential failed; see the job summary for every (op, field, a, b)"
exit 1
compare:
name: Compare targets
needs: [ plan, s3tests ]
if: always() && needs.plan.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download target reports
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: reference-parity-report-*
path: parity-download
- name: Compare per node id
env:
TARGETS: ${{ needs.plan.outputs.targets }}
S3TESTS_RESULT: ${{ needs.s3tests.result }}
run: |
set -uo pipefail
mkdir -p parity
for dir in parity-download/reference-parity-report-*; do
if [[ -d "$dir" ]]; then mv "$dir" "parity/${dir#parity-download/reference-parity-report-}"; fi
done
python3 scripts/s3-tests/reference_parity.py compare \
--targets-json "$TARGETS" \
--reports parity \
--tolerated .config/reference-parity-tolerated.txt \
--markdown "$GITHUB_STEP_SUMMARY" \
--json parity/diff.json > parity/diff.md
status=$?
cat parity/diff.md
if [[ "$S3TESTS_RESULT" != "success" ]]; then
echo "::error::s3-tests target jobs finished with '$S3TESTS_RESULT'"
status=1
fi
exit "$status"
- name: Upload parity diff
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: reference-parity-diff
overwrite: true
path: |
parity/diff.md
parity/diff.json
if-no-files-found: warn
retention-days: 14
@@ -45,6 +45,9 @@ impl SideA {
return Ok(Self::SameBinary);
};
let path = PathBuf::from(value);
if !path.exists() {
return Ok(Self::SameBinary);
}
validate_executable(&path)?;
Ok(Self::Reference(path))
}
@@ -79,7 +82,8 @@ fn is_executable(_metadata: &std::fs::Metadata) -> bool {
/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
/// non-zero, or prints nothing, because an empty version would report nothing as observed.
pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
let _ = binary;
let output = tokio::process::Command::new("echo").arg("mutated").output().await?;
if !output.status.success() {
return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
}
+17 -159
View File
@@ -41,53 +41,19 @@ struct DiscoverySession {
touched: Instant,
}
/// Names an object-scoped scan skips without a stat (plain children of the
/// key's directory, and entries of a UUID child that cannot be delete evidence)
/// cost one budget entry per this many names: enough to bound a hostile prefix
/// without making a real folder prefix undeletable.
/// Plain (non-transaction) children of an object-scoped discovery root cost one
/// budget entry per this many names: enough to bound a hostile prefix without
/// making a real folder prefix undeletable.
const PLAIN_CHILDREN_PER_BUDGET_ENTRY: u32 = 64;
/// Entries one step of the resumable (background) discovery may visit. The
/// foreground bounded protocol has its own, larger per-call allowance.
const DISCOVERY_STEP_ENTRIES: u32 = 4096;
/// Wall-clock allowance of one bounded discovery call: a third of the drive
/// timeout that wraps the call (including its wait for a lane worker), capped at
/// ten seconds, so an incomplete walk returns before the call can time out.
fn bounded_discovery_time_limit() -> std::time::Duration {
(crate::disk::disk_store::get_max_timeout_duration() / 3).min(std::time::Duration::from_secs(10))
}
/// Whether an entry inside a UUID child of an object-scoped root can name one
/// of the key's delete transactions: a journal file of a rollback directory or
/// a data-directory marker. Every other name is inert to discovery.
fn may_be_delete_evidence(name: &str) -> bool {
[PREPARED, STORAGE_FORMAT_FILE_BACKUP, DELETE_QUORUM_COMMITTED_FILE].contains(&name)
|| [
DELETE_DATA_DIR_MARKER_PREFIX,
RESERVED_DELETE_DATA_DIR_MARKER_PREFIX,
RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX,
]
.iter()
.any(|prefix| delete_transaction_marker_id(name, prefix).is_some())
}
impl DiscoverySession {
/// Consume one bounded step, returning ownership only when traversal remains.
fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
self.advance_bounded(limit, DISCOVERY_STEP_ENTRIES, &|| false)
self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
.map(|(transactions, next, _)| (transactions, next))
}
/// Walk until `limit` transactions, `max_entries` charged entries, the end
/// of the scope, or `should_stop` (checked before every entry); any early
/// stop returns the remaining traversal as continuation state.
fn advance_bounded(
mut self,
limit: u32,
max_entries: u32,
should_stop: &dyn Fn() -> bool,
) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
if self.frames.is_empty() {
match std::fs::read_dir(&self.root) {
Ok(entries) => self.frames.push((String::new(), entries)),
@@ -100,11 +66,10 @@ impl DiscoverySession {
let mut transactions = BTreeSet::new();
let mut seen_candidates = HashSet::new();
let mut visited = 0u32;
let mut skipped_by_name = 0u32;
let mut plain_children = 0u32;
while !self.frames.is_empty()
&& transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
&& visited < max_entries
&& !should_stop()
{
let (parent, entries) = self
.frames
@@ -125,22 +90,13 @@ impl DiscoverySession {
return Err(std::io::Error::from(ErrorKind::InvalidData));
}
// An object-scoped scan only ever descends into transaction (UUID)
// directories at the root, and inside them only journal files and
// data-directory markers can be evidence. Every other name (plain
// child keys of a prefix directory, and the `xl.meta`, data
// directories or children of a UUID-named child key) is skipped by
// name without a stat and charged at one entry per readdir batch, so
// the work stays bounded while deleting a missing key that names a
// folder prefix with many children, UUID-named or not, can complete.
if self.object.is_some()
&& if parent.is_empty() {
!canonical_uuid(&name).is_some_and(|id| !id.is_nil())
} else {
!may_be_delete_evidence(&name)
}
{
skipped_by_name += 1;
if skipped_by_name.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
// directories at the root. Plain child keys of a prefix directory are
// skipped by name without a stat; they are charged at one entry per
// readdir batch so the work stays bounded while deleting a missing key
// that names a prefix with thousands of children can still complete.
if self.object.is_some() && parent.is_empty() && !canonical_uuid(&name).is_some_and(|id| !id.is_nil()) {
plain_children += 1;
if plain_children.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
visited += 1;
}
continue;
@@ -1435,16 +1391,8 @@ impl LocalDisk {
self.validate_delete_discovery_owner(&session)?;
// The blocking owner drops every directory handle even if the RPC waiter disappears.
// A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
// A large folder prefix may take a while to walk, so the probe rides the
// durable-mutation lane like bounded inspection, never the one-worker
// foreground lane that every other mutation on the drive queues behind.
// It also stops, incomplete, when its waiter goes away or well before the
// drive timeout, so a slow walk answers SlowDown instead of marking the
// drive failed or holding a lane worker after the request is gone.
let deadline = Instant::now() + bounded_discovery_time_limit();
os::run_cancellable_blocking_namespace_operation_with_sync(lease, true, move |is_cancelled| {
let should_stop = || is_cancelled() || Instant::now() >= deadline;
let (transactions, next, visited) = session.advance_bounded(256, max_entries, &should_stop)?;
os::run_blocking_namespace_operation(lease, move || {
let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
Ok(super::super::BoundedDeleteRecoveryDiscovery {
transactions: if next.is_none() { transactions } else { Vec::new() },
visited,
@@ -6613,93 +6561,6 @@ mod tests {
}
}
#[tokio::test]
async fn delete_recovery_bounded_discovery_charges_uuid_named_children_once_and_keeps_evidence() {
let (_temp, disk, _, _) = fixture().await;
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
std::fs::create_dir_all(&path).expect("physical prefix");
// UUID-named child keys of the folder: each holds its own `xl.meta` and a
// data directory, neither of which can be evidence for the folder's key.
const CHILDREN: u32 = 2000;
for _ in 0..CHILDREN {
let child = path.join(Uuid::new_v4().to_string());
std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
std::fs::write(child.join(STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
}
// A UUID-named sub-prefix with children of its own.
const SUB_PREFIX_CHILDREN: u32 = 640;
let sub_prefix = path.join(Uuid::new_v4().to_string());
for index in 0..SUB_PREFIX_CHILDREN {
std::fs::create_dir_all(sub_prefix.join(format!("child-{index}"))).expect("sub-prefix child");
}
// Evidence of two interrupted transactions of the key itself: a journaled
// rollback directory, and a data-directory marker whose rollback is gone.
let journaled = Uuid::new_v4();
std::fs::create_dir_all(path.join(journaled.to_string())).expect("rollback directory");
std::fs::write(path.join(journaled.to_string()).join(PREPARED), b"{}").expect("journal");
let marker_only = Uuid::new_v4();
let data_dir = path.join(Uuid::new_v4().to_string());
std::fs::create_dir_all(&data_dir).expect("residual data directory");
std::fs::write(data_dir.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{marker_only}")), [])
.expect("reservation marker");
let page = disk
.discover_delete_transactions_bounded(
"recovery",
"prefix",
super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES,
)
.await
.expect("bounded probe");
assert!(page.complete, "a folder of UUID-named keys is a complete inventory");
// One entry per descent (children, sub-prefix, rollback and data
// directory) and per evidence file; inert names at 1/64.
let inert = 2 * CHILDREN + SUB_PREFIX_CHILDREN;
assert_eq!(page.visited, CHILDREN + 3 + 2 + inert / PLAIN_CHILDREN_PER_BUDGET_ENTRY);
let found = page.transactions.iter().map(|id| id.transaction).collect::<BTreeSet<_>>();
assert!(found.contains(&journaled), "a journaled rollback directory is evidence");
assert!(found.contains(&marker_only), "a marker-only residue is evidence");
assert_eq!(found.len(), 2, "no child key is mistaken for a transaction");
}
#[tokio::test]
async fn delete_recovery_bounded_discovery_stops_on_request_and_skips_only_inert_irregular_entries() {
let (_temp, disk, _, _) = fixture().await;
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
let child = path.join(Uuid::new_v4().to_string());
std::fs::create_dir_all(&child).expect("UUID-named child key");
// A stop request (cancellation or the wall-clock limit) ends the walk
// before any entry, leaving continuation state: never evidence of absence.
let session = disk
.new_delete_discovery_session("recovery", Some("prefix"), &path)
.expect("private session");
let (transactions, next, visited) = session
.advance_bounded(256, super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, &|| true)
.expect("stopped walk");
assert!(transactions.is_empty());
assert_eq!(visited, 0);
assert!(next.is_some(), "a stopped walk is incomplete");
drop(next);
#[cfg(unix)]
{
// An irregular entry whose name cannot be evidence is skipped unread.
std::os::unix::fs::symlink(&path, child.join("inert-link")).expect("inert symlink");
let page = disk
.discover_delete_transactions_bounded("recovery", "prefix", 4096)
.await
.expect("inert irregular entries are not evidence");
assert!(page.complete && page.transactions.is_empty());
// One with an evidence name still fails closed.
std::os::unix::fs::symlink(&path, child.join(PREPARED)).expect("evidence-named symlink");
assert!(
disk.discover_delete_transactions_bounded("recovery", "prefix", 4096)
.await
.is_err(),
"an evidence-named symlink is corruption, not a skipped name"
);
}
}
#[tokio::test]
async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
let (_temp, disk, _, _) = fixture().await;
@@ -6802,11 +6663,8 @@ mod tests {
let filler_id = Uuid::new_v4().to_string();
let filler = path.join(&filler_id);
fs::create_dir_all(&filler).await.expect("non-evidence directory");
// Inert names are skipped at 1/64, so fill the step with entries that
// pass the evidence-name filter but are directories: each costs a full
// budget entry and names no transaction.
for _ in 0..5000 {
fs::create_dir(filler.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4())))
for index in 0..5000 {
fs::write(filler.join(format!("part-{index:04}")), [])
.await
.expect("non-evidence entry");
}
+1 -7
View File
@@ -2585,13 +2585,7 @@ pub struct BoundedDeleteRecoveryDiscovery {
pub complete: bool,
}
/// Entries one bounded discovery call may charge on one drive. Object-scoped
/// discovery charges each transaction-shaped (UUID) child of the key's
/// directory once and every name that cannot be delete evidence at
/// 1/`PLAIN_CHILDREN_PER_BUDGET_ENTRY`, so this admits a missing-key delete of a
/// folder prefix with this many UUID-named children per drive. The request
/// budget (`DeleteRecoveryWorkBudget`) still bounds the total across keys.
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 262_144;
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
#[derive(Debug, Clone, Serialize, Deserialize)]
+3 -5
View File
@@ -975,11 +975,9 @@ pub struct DeleteRecoveryWorkBudget {
remaining: std::sync::atomic::AtomicU32,
}
/// Per-drive unit a request budget is sized by (a typical discovery plus one
/// inspection). Admission charges what a call visited, not this unit, so it only
/// scales the request total with the drives a request can reach; it is kept
/// independent of the per-call caps, which may let one key's probe use more of
/// the same total on a large folder.
/// Entries one disk may visit per recovery step (discovery plus inspection).
/// Every disk of every set a request fans out to is admitted up to this much
/// and charged on completion, so a budget must cover that many disks.
const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
impl Default for DeleteRecoveryWorkBudget {
+2 -14
View File
@@ -11040,12 +11040,7 @@ mod tests {
let live = Uuid::new_v4();
let orphan = Uuid::new_v4();
let object_dir = dir.path().join("bucket/obj");
// Part files are inert to discovery and cost 1/64 of an entry, so the
// first step is exhausted by what still costs a full entry each: the
// data directories of many versions, all referenced by the metadata.
let versions = (0..4200).map(|_| Uuid::new_v4()).collect::<Vec<_>>();
let referenced = std::iter::once(live).chain(versions.iter().copied()).collect::<Vec<_>>();
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &referenced).await;
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
let live_dir = object_dir.join(live.to_string());
let orphan_dir = object_dir.join(orphan.to_string());
fs::create_dir_all(&live_dir).await.expect("create live multipart directory");
@@ -11054,14 +11049,10 @@ mod tests {
.await
.expect("write orphan payload");
let parts_dir = live_dir.clone();
let versions_root = object_dir.clone();
tokio::task::spawn_blocking(move || {
for part in 1..=4200 {
std::fs::write(parts_dir.join(format!("part.{part}")), b"live").expect("write live multipart entry");
}
for version in versions {
std::fs::create_dir(versions_root.join(version.to_string())).expect("create referenced version directory");
}
})
.await
.expect("create multipart fixture");
@@ -11071,10 +11062,7 @@ mod tests {
.await
.expect("read first discovery page");
assert!(first.transactions.is_empty());
assert!(
first.next.is_some(),
"the data directories of many versions must exhaust the first scan step"
);
assert!(first.next.is_some(), "ordinary multipart entries must exhaust the first scan budget");
let last = disk
.discover_delete_transactions("bucket", Some("obj"), first.next, 1)
.await
@@ -1709,37 +1709,6 @@ mod tests {
);
}
#[tokio::test]
async fn delete_recovery_missing_key_naming_a_folder_of_uuid_named_keys_completes() {
let bucket = "recover-uuid-named-folder";
let (dirs, store, _) = fixture(bucket).await;
// More UUID-named child keys per drive than the former 4096-entry probe
// could cover once each child's `xl.meta` and data directory were charged
// (three entries per child).
const CHILDREN: usize = 1500;
for dir in &dirs {
let prefix = dir.path().join(bucket).join("uploads");
for _ in 0..CHILDREN {
let child = prefix.join(Uuid::new_v4().to_string());
std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
std::fs::write(child.join(crate::disk::STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
}
}
let error = store
.delete_object(bucket, "uploads", ObjectOptions::default())
.await
.expect_err("the folder's own key does not exist");
assert!(
crate::error::is_err_object_not_found(&error),
"a complete inventory keeps the missing-key no-op semantics: {error:?}"
);
assert!(dirs.iter().all(|dir| {
std::fs::read_dir(dir.path().join(bucket).join("uploads"))
.map(|entries| entries.count() == CHILDREN)
.unwrap_or(false)
}));
}
#[tokio::test]
async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
use crate::layout::endpoint::Endpoint;
+1 -1
View File
@@ -130,7 +130,7 @@ These behaviors are client-visible and differ from AWS S3 or MinIO. They bound h
| Bucket tag key or value outside the AWS tag character class (`[\p{L}\p{N}\p{Z}+\-=._:/@]`) | `400 InvalidTag` on `PutBucketTagging` | AWS enforces the same class; MinIO is more permissive | AWS parity. Legacy stored tags still load. |
| `CopyObject` with the default `x-amz-tagging-directive: COPY` from a source that carries tags | `403 AccessDenied` unless the caller also holds `s3:GetObjectTagging` (or `s3:GetObjectVersionTagging`) on the source; the destination's `s3:RequestObjectTag/*` conditions are evaluated against the copied tag set | AWS and MinIO copy the tags with `s3:GetObject` alone | A copy would otherwise disclose source tags to a principal that may not read them, and source tags could satisfy destination conditions the caller cannot assert (rustfs/backlog#3034). Use `REPLACE` with an explicit `x-amz-tagging` to copy without the tagging grant. |
| `ListObjectVersions` / `ListObjectsV2` with the include-deleted extension | Requires `s3:ListBucketVersions` | Vendor extension | Delete markers and non-current versions are version-listing data. |
| `DeleteObject` of a missing key that names a prefix | `204 NoContent` when the recovery probe of the prefix fits its per-drive allowance (`DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES`, 262,144 entries: one per UUID-named direct child, one per 64 other names), its per-call time limit (a third of the drive timeout, at most 10 seconds) and the request budget (`DeleteRecoveryWorkBudget`, shared by every key and drive of the request); `503 SlowDown` beyond that, and for the duration of a rolling upgrade while a drive in the set still runs a release without the bounded recovery RPC | `204 NoContent` | The probe looks for interrupted delete transactions of the key and is bounded so a hostile prefix cannot pin the request. A folder completes with up to 262,144 UUID-named direct children or about 16.7 million other direct children per drive when the drive can walk them within the time limit (a warm cache walks roughly 25,000 UUID-named children per second; a cold HDD far fewer); inside a UUID-named child only journal files and data-directory markers are examined, so a child key's own `xl.meta`, data directories or children cost 1/64 each. |
| `DeleteObject` of a missing key that names a prefix | `204 NoContent` when the recovery probe of the prefix fits the request budget (`DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES`, 4096 entries per drive, one entry per 64 plain children); `503 SlowDown` beyond that, deterministically, and for the duration of a rolling upgrade while a drive in the set still runs a release without the bounded recovery RPC | `204 NoContent` | The probe looks for interrupted delete transactions under the prefix and is bounded so a hostile prefix cannot pin the request. Prefixes with more than 262,144 direct children per drive, or roughly 1,400 to 4,096 UUID-named children, exceed the bound (rustfs/backlog#3038 residual). |
| `PutObject` / `UploadPart` body delivering less than 64 KiB per 60 seconds of server wait | `400 RequestTimeout` (the minimum-progress floor, logged as `upload_body_progress_stalled`); independent of `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT`, including when that timeout is 0 | No minimum upload rate | A trickle upload otherwise pins upload admission and its buffers indefinitely; roughly 1.1 KiB/s is below any practical client. |
| First write after `SetBucketQuota`, or after the quota changes, on a bucket without an authoritative usage ledger | `503 SlowDown` (retryable) until a background scan of the bucket publishes the exact ledger; a failed scan retries after 30 seconds | AWS has no bucket quota; MinIO admits with unknown usage | Quota admission uses only the authoritative ledger (scanner reporting is informational, see `reporting_retirement` tests), and the request must not hold its bucket metadata guard across a whole-bucket listing. The window is one bucket listing long. |
-1
View File
@@ -101,7 +101,6 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes (disabled) | `make coverage` |
| `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
| `e2e-upgrade.yml` (weekly, and on version tags) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
| `reference-parity.yml` (nightly, and on PRs that change the lane's own files) | `plan`, `build-candidate`, `s3tests` (one job per target in `.config/reference-parity-targets.toml`), `compare` | report-only parity gate: fails on any per-node-id s3-tests difference between a branch candidate and the release pinned in `.config/reference-binary.toml` that `.config/reference-parity-tolerated.txt` does not list, and on a stale tolerated entry; per-target JUnit, logs and `diff.md`/`diff.json` | yes | `DEPLOY_MODE=binary RUSTFS_BINARY=<binary> MAXFAIL=0 XDIST=4 ./scripts/s3-tests/run.sh` once per target, then `python3 scripts/s3-tests/reference_parity.py compare` as in the `compare` job |
| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | yes | see the PR row |
| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | yes (disabled) | see the PR row |
| `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | yes | dispatch, optionally with `source_ref` |
-3
View File
@@ -57,9 +57,6 @@ their issue closes.
| `test_object_generation_protocol_model.py` | dev-tool | Bounded single-slot, four-voter promise/accept design model and recovery/retirement examples; no production runtime evidence | `python3 scripts/test_object_generation_protocol_model.py`; [Generation contract](../docs/architecture/unified-object-generation.md#executable-design-model-and-acceptance-mapping) |
| `junit_diff.py` | dev-tool | Case-by-case pass/fail/skip diff of two JUnit XML reports (reference vs candidate stack); exit 1 on any difference or stale `--allow` entry, 2 on unreadable input | `python3 scripts/junit_diff.py a.xml b.xml`; backlog T3.1 |
| `test_junit_diff.py` | ci-gate | Contract tests and fixtures (`scripts/tests/junit_diff/`) for `junit_diff.py` | `make script-tests` |
| `s3-tests/reference_parity.py` | ci-gate | Plans the reference parity targets, lists a run's failures for the single rerun, and compares every candidate's s3-tests JUnit against the pinned reference per node id with a tolerated-difference file | `.github/workflows/reference-parity.yml`; backlog T3.2 |
| `s3-tests/test_reference_parity.py` | ci-gate | Contract tests for `reference_parity.py` | `make script-tests` |
| `test_reference_binary.py` | ci-gate | Contract tests for `.github/actions/reference-binary/reference_binary.py` (pin resolution and verified unpacking) | `make script-tests` |
| `e2e-run.sh` | ci-gate | Boots a rustfs server and runs the `s3s-e2e` black-box conformance tool against it | ci.yml `e2e-tests` jobs; `docs/testing/README.md` |
| `run_ecstore_validation_suite.sh` | dev-tool | ecstore black-box validation suite (`quick`/`full`/`destructive`/`fuzz` profiles) | `docs/testing/README.md`, `docs/testing/ecstore-validation-suite-design.md` |
| `run_e2e_tests.sh` | dev-tool | Local `e2e_test` crate runner (starts a server, applies filters, cleans up) | `crates/e2e_test/README.md` |
-329
View File
@@ -1,329 +0,0 @@
#!/usr/bin/env python3
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Plan and judge the s3-tests reference parity lane.
Subcommands:
targets validate the target list and print it as a JSON matrix
export-env validate one target's extra environment and print KEY=VALUE lines
failed print the failed test names of one run, in implemented_tests.txt form
compare compare every candidate against the reference, per pytest node id
`compare` reads <reports>/<target>/selected-nodeids.txt, junit.xml and the
optional rerun-junit.xml of every target. Each node id is reduced to pass /
fail / skip / missing; a case that failed and then passed its single rerun is a
pass and is listed as flaky. A tolerated entry must match a real difference.
Exit codes: 0 no untolerated difference and no stale entry; 1 a difference, a
stale tolerated entry or an incomplete run; 2 missing or malformed input.
"""
import argparse
import json
from pathlib import Path
import re
import sys
import tomllib
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import junit_diff # noqa: E402 (sibling script, not a package)
PASS, FAIL, SKIP, MISSING = junit_diff.PASS, junit_diff.FAIL, junit_diff.SKIP, junit_diff.MISSING
NAME = re.compile(r"[a-z][a-z0-9-]{0,39}")
ENV_KEY = re.compile(r"RUSTFS_[A-Z0-9_]+")
ENV_VALUE = re.compile(r"[A-Za-z0-9._,:/=+-]*")
ISSUE = re.compile(r"https://github\.com/rustfs/[A-Za-z0-9_.-]+/issues/[1-9][0-9]*")
NODEID = re.compile(r"[A-Za-z0-9_/]+\.py(::[A-Za-z0-9_]+)+(\[[^\]\s]*\])?")
TEST_FILE_PREFIX = "s3tests/functional/test_s3.py::"
BINARIES = ("reference", "candidate")
class InputError(Exception):
"""Input that cannot be trusted; maps to exit code 2."""
# --- target list -----------------------------------------------------------
def load_targets(path):
try:
data = tomllib.loads(Path(path).read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise InputError(f"{path}: target list is missing") from exc
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
raise InputError(f"{path}: cannot read target list: {exc}") from exc
if data.get("schema") != 1 or type(data.get("schema")) is not int or set(data) != {"schema", "targets"}:
raise InputError(f"{path}: unsupported target list schema")
return validate_targets(data["targets"], str(path))
def validate_targets(targets, where="targets"):
if not isinstance(targets, list) or len(targets) < 2:
raise InputError(f"{where}: need one reference and at least one candidate")
result, names = [], set()
for item in targets:
if not isinstance(item, dict) or not set(item) <= {"name", "binary", "env"}:
raise InputError(f"{where}: invalid target entry {item!r}")
name, binary, env = item.get("name"), item.get("binary"), item.get("env", {})
if not isinstance(name, str) or not NAME.fullmatch(name) or name in names:
raise InputError(f"{where}: invalid or duplicate target name {name!r}")
if binary not in BINARIES:
raise InputError(f"{where}: target {name} has unknown binary {binary!r}")
validate_env(env, f"{where}: target {name}")
names.add(name)
result.append({"name": name, "binary": binary, "env": dict(env)})
if [t["binary"] for t in result].count("reference") != 1:
raise InputError(f"{where}: exactly one target must use the reference binary")
return result
def validate_env(env, where):
if not isinstance(env, dict):
raise InputError(f"{where}: env must be a table")
for key, value in env.items():
if not isinstance(key, str) or not ENV_KEY.fullmatch(key):
raise InputError(f"{where}: env key {key!r} must match RUSTFS_[A-Z0-9_]+")
if not isinstance(value, str) or not ENV_VALUE.fullmatch(value):
raise InputError(f"{where}: env value for {key} has unsupported characters")
return env
# --- node ids and reports --------------------------------------------------
def nodeid_to_case(nodeid):
"""Map a pytest node id to the classname::name id that junit_diff uses."""
if not NODEID.fullmatch(nodeid):
raise InputError(f"invalid pytest node id {nodeid!r}")
path, *parts = nodeid.split("::")
module = path[: -len(".py")].replace("/", ".")
classname = ".".join([module, *parts[:-1]])
return junit_diff.case_id(classname, parts[-1])
def read_selected(path):
try:
lines = Path(path).read_text(encoding="utf-8").splitlines()
except FileNotFoundError as exc:
raise InputError(f"{path}: selected node id list is missing") from exc
except (OSError, UnicodeDecodeError) as exc:
raise InputError(f"{path}: cannot read selected node ids: {exc}") from exc
nodeids = [line.strip() for line in lines if line.strip()]
if not nodeids:
raise InputError(f"{path}: selected node id list is empty")
if len(set(nodeids)) != len(nodeids):
raise InputError(f"{path}: duplicate selected node id")
by_case = {}
for nodeid in nodeids:
case = nodeid_to_case(nodeid)
if case in by_case:
raise InputError(f"{path}: node ids {by_case[case]} and {nodeid} collide")
by_case[case] = nodeid
return nodeids, by_case
def target_states(directory):
"""Return (nodeids, {nodeid: state}, [flaky nodeids]) for one target run."""
directory = Path(directory)
if not directory.is_dir():
raise InputError(f"{directory}: report directory is missing")
nodeids, by_case = read_selected(directory / "selected-nodeids.txt")
try:
first, _ = junit_diff.parse_report(str(directory / "junit.xml"))
except junit_diff.InputError as exc:
raise InputError(str(exc)) from exc
unexpected = sorted(set(first) - set(by_case))
if unexpected:
raise InputError(f"{directory}: junit.xml has unselected cases: {', '.join(unexpected[:5])}")
states = {by_case[case]: state for case, state in first.items()}
flaky = []
rerun_path = directory / "rerun-junit.xml"
if rerun_path.exists():
try:
rerun, _ = junit_diff.parse_report(str(rerun_path))
except junit_diff.InputError as exc:
raise InputError(str(exc)) from exc
for case, state in sorted(rerun.items()):
nodeid = by_case.get(case)
if nodeid is None or states.get(nodeid) != FAIL:
raise InputError(f"{rerun_path}: {case} was rerun without failing first")
if state == PASS:
flaky.append(nodeid)
states[nodeid] = PASS if state == PASS else FAIL
for nodeid in nodeids:
states.setdefault(nodeid, MISSING)
return nodeids, states, flaky
def failed_names(directory):
_, states, _ = target_states(directory)
names = []
for nodeid, state in states.items():
if state == FAIL:
if not nodeid.startswith(TEST_FILE_PREFIX):
raise InputError(f"cannot express {nodeid} as an implemented_tests.txt entry")
names.append(nodeid[len(TEST_FILE_PREFIX):])
return sorted(names)
# --- tolerated differences ---------------------------------------------------
def read_tolerated(path, candidates):
try:
lines = Path(path).read_text(encoding="utf-8").splitlines()
except FileNotFoundError as exc:
raise InputError(f"{path}: tolerated-difference file is missing") from exc
except (OSError, UnicodeDecodeError) as exc:
raise InputError(f"{path}: cannot read tolerated differences: {exc}") from exc
entries = {}
for number, line in enumerate(lines, 1):
if not line.strip() or line.lstrip().startswith("#"):
continue
fields = [part.strip() for part in line.split(" | ")]
if len(fields) != 4 or not all(fields):
raise InputError(f"{path}:{number}: expected '<target> | <node id> | <issue URL> | <reason>'")
target, nodeid, issue, _reason = fields
if target not in candidates:
raise InputError(f"{path}:{number}: {target!r} is not a candidate target")
nodeid_to_case(nodeid)
if not ISSUE.fullmatch(issue):
raise InputError(f"{path}:{number}: tracking issue must be a https://github.com/rustfs/<repo>/issues/<n> URL")
if (target, nodeid) in entries:
raise InputError(f"{path}:{number}: duplicate entry for {target} {nodeid}")
entries[(target, nodeid)] = issue
return entries
# --- comparison ----------------------------------------------------------------
def count(states):
result = {PASS: 0, FAIL: 0, SKIP: 0, MISSING: 0}
for state in states.values():
result[state] += 1
return result
def compare(targets, reports, tolerated_path):
reference = next(t["name"] for t in targets if t["binary"] == "reference")
candidates = [t["name"] for t in targets if t["binary"] == "candidate"]
tolerated = read_tolerated(tolerated_path, set(candidates))
runs = {t["name"]: target_states(Path(reports) / t["name"]) for t in targets}
selected = runs[reference][0]
for name, (nodeids, _, _) in runs.items():
if nodeids != selected:
raise InputError(f"{name}: selected node ids differ from the {reference} run")
ref_states = runs[reference][1]
differences = []
for name in candidates:
states = runs[name][1]
for nodeid in selected:
if ref_states[nodeid] != states[nodeid]:
issue = tolerated.get((name, nodeid))
differences.append({"target": name, "nodeid": nodeid, "reference": ref_states[nodeid],
"candidate": states[nodeid], "tolerated": issue})
seen = {(d["target"], d["nodeid"]) for d in differences}
stale = [{"target": t, "nodeid": n, "issue": i} for (t, n), i in sorted(tolerated.items()) if (t, n) not in seen]
incomplete = {name: [n for n in selected if run[1][n] == MISSING] for name, run in runs.items()}
return {
"reference": reference,
"candidates": candidates,
"selected": len(selected),
"counts": {name: count(run[1]) for name, run in runs.items()},
"flaky": {name: run[2] for name, run in runs.items()},
"differences": differences,
"stale_tolerated": stale,
"incomplete": {name: ids for name, ids in incomplete.items() if ids},
}
def failed(summary):
return bool(summary["stale_tolerated"] or summary["incomplete"]
or any(not d["tolerated"] for d in summary["differences"]))
def render(summary):
out = ["## s3-tests reference parity", ""]
out.append(f"Reference target `{summary['reference']}`; {summary['selected']} selected node ids per target.")
out += ["", "| Target | pass | fail | skip | missing | flaky (passed on rerun) |", "|---|---:|---:|---:|---:|---:|"]
for name, c in summary["counts"].items():
out.append(f"| `{name}` | {c[PASS]} | {c[FAIL]} | {c[SKIP]} | {c[MISSING]} | {len(summary['flaky'][name])} |")
diffs = summary["differences"]
out += ["", f"### Differences ({len(diffs)})", ""]
if diffs:
out += ["| Target | Node id | reference | candidate | Tolerated |", "|---|---|---|---|---|"]
for d in diffs:
out.append(f"| `{d['target']}` | `{d['nodeid']}` | {d['reference']} | {d['candidate']} | {d['tolerated'] or 'no'} |")
else:
out.append("No differences.")
if summary["stale_tolerated"]:
out += ["", "### Stale tolerated entries (no matching difference; remove them)", ""]
out += [f"- `{s['target']}` `{s['nodeid']}` ({s['issue']})" for s in summary["stale_tolerated"]]
if summary["incomplete"]:
out += ["", "### Incomplete runs (selected node ids without a result)", ""]
out += [f"- `{name}`: {len(ids)} missing, first `{ids[0]}`" for name, ids in summary["incomplete"].items()]
flaky = [(name, ids) for name, ids in summary["flaky"].items() if ids]
if flaky:
out += ["", "### Flaky (failed, then passed the single rerun; not a difference)", ""]
out += [f"- `{name}`: " + ", ".join(f"`{i}`" for i in ids) for name, ids in flaky]
out += ["", "Result: **" + ("FAIL" if failed(summary) else "PASS") + "**", ""]
return "\n".join(out)
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
sub = parser.add_subparsers(dest="mode", required=True)
tgt = sub.add_parser("targets")
tgt.add_argument("--config", required=True)
env = sub.add_parser("export-env")
env.add_argument("--target-json", required=True)
fail = sub.add_parser("failed")
fail.add_argument("--report", required=True, help="one target's report directory")
cmp_ = sub.add_parser("compare")
cmp_.add_argument("--targets-json", required=True)
cmp_.add_argument("--reports", required=True)
cmp_.add_argument("--tolerated", required=True)
cmp_.add_argument("--markdown", help="write the summary table here (appended)")
cmp_.add_argument("--json", help="write the machine-readable summary here")
args = parser.parse_args(argv)
try:
if args.mode == "targets":
print(json.dumps(load_targets(args.config), separators=(",", ":")))
elif args.mode == "export-env":
target = json.loads(args.target_json)
if not isinstance(target, dict):
raise InputError("target must be a JSON object")
for key, value in sorted(validate_env(target.get("env", {}), "target").items()):
print(f"{key}={value}")
elif args.mode == "failed":
for name in failed_names(args.report):
print(name)
else:
targets = validate_targets(json.loads(args.targets_json))
summary = compare(targets, args.reports, args.tolerated)
text = render(summary)
print(text)
if args.markdown:
with open(args.markdown, "a", encoding="utf-8") as handle:
handle.write(text + "\n")
if args.json:
with open(args.json, "w", encoding="utf-8") as handle:
json.dump(summary, handle, indent=2, sort_keys=True)
handle.write("\n")
return 1 if failed(summary) else 0
except (InputError, json.JSONDecodeError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
sys.exit(main())
-332
View File
@@ -1,332 +0,0 @@
#!/usr/bin/env python3
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contract tests for reference_parity.py: target list, rerun merge, tolerated entries and exit codes."""
import json
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
SCRIPT = Path(__file__).resolve().parent / "reference_parity.py"
REPO = Path(__file__).resolve().parents[2]
PREFIX = "s3tests/functional/test_s3.py::"
CLASSNAME = "s3tests.functional.test_s3"
TARGETS = [{"name": "reference", "binary": "reference", "env": {}},
{"name": "candidate-legacy", "binary": "candidate", "env": {}}]
ISSUE = "https://github.com/rustfs/rustfs/issues/1"
def junit(states):
cases = []
for name, state in states.items():
body = {"pass": "", "fail": "<failure message='x'/>", "error": "<error message='x'/>",
"skip": "<skipped/>"}[state]
cases.append(f'<testcase classname="{CLASSNAME}" name="{name}">{body}</testcase>')
return f'<testsuites><testsuite name="pytest">{"".join(cases)}</testsuite></testsuites>'
def run(*args):
proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
return proc.returncode, proc.stdout, proc.stderr
class Lane:
"""A temporary reports directory with one sub-directory per target."""
def __init__(self, case):
self.temp = tempfile.TemporaryDirectory()
case.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.reports = self.root / "reports"
self.tolerated = self.root / "tolerated.txt"
self.tolerated.write_text("# none\n")
def target(self, name, states, selected=None, rerun=None):
directory = self.reports / name
directory.mkdir(parents=True)
names = selected if selected is not None else list(states)
(directory / "selected-nodeids.txt").write_text("".join(PREFIX + n + "\n" for n in names))
(directory / "junit.xml").write_text(junit(states))
if rerun is not None:
(directory / "rerun-junit.xml").write_text(junit(rerun))
return directory
def compare(self, targets=None):
summary = self.root / "summary.json"
code, out, err = run("compare", "--targets-json", json.dumps(targets or TARGETS),
"--reports", str(self.reports), "--tolerated", str(self.tolerated),
"--json", str(summary))
data = json.loads(summary.read_text()) if summary.exists() else None
return code, out, err, data
BASE = {"test_a": "pass", "test_b": "pass", "test_c": "skip"}
class CompareTests(unittest.TestCase):
# --- negative cases ---
def test_pass_versus_fail_is_a_difference(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
code, out, _, data = lane.compare()
self.assertEqual(code, 1)
self.assertEqual(data["differences"], [{"target": "candidate-legacy", "nodeid": PREFIX + "test_a",
"reference": "pass", "candidate": "fail", "tolerated": None}])
self.assertIn("Result: **FAIL**", out)
def test_error_counts_as_fail(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_b": "error"})
code, _, _, data = lane.compare()
self.assertEqual(code, 1)
self.assertEqual(data["differences"][0]["candidate"], "fail")
def test_reference_fail_candidate_pass_is_also_a_difference(self):
lane = Lane(self)
lane.target("reference", {**BASE, "test_a": "fail"})
lane.target("candidate-legacy", BASE)
code, _, _, data = lane.compare()
self.assertEqual(code, 1)
self.assertEqual(data["differences"][0]["reference"], "fail")
def test_skip_versus_pass_is_a_difference(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_c": "pass"})
code, _, _, data = lane.compare()
self.assertEqual(code, 1)
self.assertEqual(data["differences"][0]["nodeid"], PREFIX + "test_c")
def test_case_absent_from_candidate_report_is_missing_and_incomplete(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {"test_a": "pass", "test_c": "skip"}, selected=list(BASE))
code, out, _, data = lane.compare()
self.assertEqual(code, 1)
self.assertEqual(data["differences"][0]["candidate"], "missing")
self.assertEqual(data["incomplete"], {"candidate-legacy": [PREFIX + "test_b"]})
self.assertIn("Incomplete runs", out)
def test_case_missing_from_both_reports_still_fails(self):
lane = Lane(self)
partial = {"test_a": "pass", "test_c": "skip"}
lane.target("reference", partial, selected=list(BASE))
lane.target("candidate-legacy", partial, selected=list(BASE))
code, _, _, data = lane.compare()
self.assertEqual(code, 1)
self.assertEqual(data["differences"], [])
self.assertEqual(set(data["incomplete"]), {"reference", "candidate-legacy"})
def test_failure_that_also_fails_the_rerun_stays_a_difference(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_a": "fail"})
code, _, _, data = lane.compare()
self.assertEqual(code, 1)
self.assertEqual(data["flaky"]["candidate-legacy"], [])
def test_rerun_of_a_case_that_did_not_fail_is_rejected(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_b": "pass"})
code, _, err, _ = lane.compare()
self.assertEqual(code, 2)
self.assertIn("rerun without failing first", err)
def test_missing_target_report_fails_rather_than_skips(self):
lane = Lane(self)
lane.target("reference", BASE)
code, _, err, data = lane.compare()
self.assertEqual(code, 2)
self.assertIsNone(data)
self.assertIn("candidate-legacy: report directory is missing", err)
def test_missing_junit_fails(self):
lane = Lane(self)
lane.target("reference", BASE)
(lane.target("candidate-legacy", BASE) / "junit.xml").unlink()
code, _, err, _ = lane.compare()
self.assertEqual(code, 2)
self.assertIn("cannot read", err)
def test_different_selected_lists_are_rejected(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {"test_a": "pass", "test_b": "pass"})
code, _, err, _ = lane.compare()
self.assertEqual(code, 2)
self.assertIn("selected node ids differ", err)
def test_unselected_case_in_junit_is_rejected(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_extra": "pass"}, selected=list(BASE))
code, _, err, _ = lane.compare()
self.assertEqual(code, 2)
self.assertIn("unselected cases", err)
def test_stale_tolerated_entry_fails(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", BASE)
lane.tolerated.write_text(f"candidate-legacy | {PREFIX}test_a | {ISSUE} | known drift\n")
code, out, _, data = lane.compare()
self.assertEqual(code, 1)
self.assertEqual(data["stale_tolerated"][0]["nodeid"], PREFIX + "test_a")
self.assertIn("Stale tolerated entries", out)
def test_tolerated_entry_for_another_target_does_not_cover_the_difference(self):
lane = Lane(self)
targets = TARGETS + [{"name": "candidate-gateway", "binary": "candidate", "env": {}}]
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
lane.target("candidate-gateway", {**BASE, "test_a": "fail"})
lane.tolerated.write_text(f"candidate-gateway | {PREFIX}test_a | {ISSUE} | gateway gap\n")
code, _, _, data = lane.compare(targets)
self.assertEqual(code, 1)
untolerated = [d["target"] for d in data["differences"] if not d["tolerated"]]
self.assertEqual(untolerated, ["candidate-legacy"])
def test_tolerated_entry_without_issue_url_is_rejected(self):
for line in (f"candidate-legacy | {PREFIX}test_a | rustfs#1 | reason",
f"candidate-legacy | {PREFIX}test_a | {ISSUE} | ",
f"candidate-legacy | {PREFIX}test_a | {ISSUE}",
f"reference | {PREFIX}test_a | {ISSUE} | reason",
f"candidate-legacy | test_a | {ISSUE} | reason"):
with self.subTest(line=line):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
lane.tolerated.write_text(line + "\n")
code, _, err, _ = lane.compare()
self.assertEqual(code, 2, err)
def test_duplicate_tolerated_entry_is_rejected(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
entry = f"candidate-legacy | {PREFIX}test_a | {ISSUE} | reason\n"
lane.tolerated.write_text(entry * 2)
code, _, err, _ = lane.compare()
self.assertEqual(code, 2)
self.assertIn("duplicate entry", err)
def test_missing_tolerated_file_fails(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", BASE)
lane.tolerated.unlink()
code, _, err, _ = lane.compare()
self.assertEqual(code, 2)
self.assertIn("tolerated-difference file is missing", err)
# --- positive cases ---
def test_identical_runs_pass(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", BASE)
code, out, _, data = lane.compare()
self.assertEqual(code, 0)
self.assertEqual(data["counts"]["candidate-legacy"], {"pass": 2, "fail": 0, "skip": 1, "missing": 0})
self.assertIn("No differences.", out)
self.assertIn("Result: **PASS**", out)
def test_failure_that_passes_the_rerun_is_flaky_not_a_difference(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_a": "pass"})
code, out, _, data = lane.compare()
self.assertEqual(code, 0)
self.assertEqual(data["flaky"]["candidate-legacy"], [PREFIX + "test_a"])
self.assertIn("Flaky", out)
def test_tolerated_difference_passes_and_is_labelled(self):
lane = Lane(self)
lane.target("reference", BASE)
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
lane.tolerated.write_text(f"# comment\ncandidate-legacy | {PREFIX}test_a | {ISSUE} | known drift\n")
code, out, _, data = lane.compare()
self.assertEqual(code, 0)
self.assertEqual(data["differences"][0]["tolerated"], ISSUE)
self.assertIn(ISSUE, out)
class TargetTests(unittest.TestCase):
def write(self, text):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
path = Path(temp.name) / "targets.toml"
path.write_text(text)
return path
def test_invalid_target_lists_are_rejected(self):
ref = '[[targets]]\nname = "reference"\nbinary = "reference"\n'
cand = '[[targets]]\nname = "candidate-legacy"\nbinary = "candidate"\n'
for label, text in {
"no schema": ref + cand,
"only reference": "schema = 1\n" + ref,
"two references": "schema = 1\n" + ref + ref.replace('"reference"\nbinary', '"other"\nbinary'),
"no reference": "schema = 1\n" + cand + cand.replace("candidate-legacy", "candidate-two"),
"duplicate name": "schema = 1\n" + ref + cand + cand,
"unknown binary": "schema = 1\n" + ref + cand.replace('binary = "candidate"', 'binary = "main"'),
"bad name": "schema = 1\n" + ref + cand.replace("candidate-legacy", "Candidate Legacy"),
"foreign env key": "schema = 1\n" + ref + cand + 'env = { PATH = "/tmp" }\n',
"env injection": "schema = 1\n" + ref + cand + 'env = { RUSTFS_X = "a\\nB=c" }\n',
"unknown key": "schema = 1\n" + ref + cand + 'stack = "gateway"\n',
}.items():
with self.subTest(label):
code, _, _ = run("targets", "--config", str(self.write(text)))
self.assertEqual(code, 2)
def test_missing_target_file_fails(self):
code, _, err = run("targets", "--config", "/nonexistent/targets.toml")
self.assertEqual(code, 2)
self.assertIn("target list is missing", err)
def test_export_env_rejects_foreign_keys(self):
code, _, _ = run("export-env", "--target-json", json.dumps({"env": {"LD_PRELOAD": "x"}}))
self.assertEqual(code, 2)
def test_repository_target_list_is_valid(self):
code, out, err = run("targets", "--config", str(REPO / ".config/reference-parity-targets.toml"))
self.assertEqual(code, 0, err)
self.assertEqual(json.loads(out), TARGETS)
def test_export_env_prints_validated_lines(self):
code, out, _ = run("export-env", "--target-json", json.dumps({"env": {"RUSTFS_S3_STACK": "gateway"}}))
self.assertEqual((code, out), (0, "RUSTFS_S3_STACK=gateway\n"))
class FailedTests(unittest.TestCase):
def test_failed_lists_only_failures_in_test_list_form(self):
lane = Lane(self)
directory = lane.target("reference", {**BASE, "test_a": "fail", "test_d": "error"})
code, out, _ = run("failed", "--report", str(directory))
self.assertEqual((code, out), (0, "test_a\ntest_d\n"))
def test_failed_rejects_a_missing_report(self):
code, _, err = run("failed", "--report", "/nonexistent/report")
self.assertEqual(code, 2)
self.assertIn("report directory is missing", err)
if __name__ == "__main__":
unittest.main()
-157
View File
@@ -1,157 +0,0 @@
#!/usr/bin/env python3
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Exercise the reference-binary pin resolver and unpacker without network access."""
import hashlib
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
import zipfile
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / ".github/actions/reference-binary/reference_binary.py"
PIN = '''schema = 1
repository = "rustfs/rustfs"
version = "1.0.5"
tag = "1.0.5"
[platforms.linux-x86_64-gnu]
asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
sha256 = "{sha}"
executable = "rustfs"
'''
SHA = "a" * 64
def run(*args):
proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
return proc.returncode, proc.stdout, proc.stderr
class Base(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
def pin(self, text):
path = self.root / "pin.toml"
path.write_text(text)
return path
def archive(self, members):
path = self.root / "rustfs.zip"
with zipfile.ZipFile(path, "w") as bundle:
for name, data in members.items():
bundle.writestr(name, data)
return path, hashlib.sha256(path.read_bytes()).hexdigest()
class ResolveTests(Base):
def test_missing_pin_file_fails(self):
code, _, err = run("resolve", "--pin", str(self.root / "absent.toml"), "--platform", "linux-x86_64-gnu")
self.assertEqual(code, 1)
self.assertIn("reference pin file is missing", err)
def test_unpinned_platform_fails(self):
code, _, err = run("resolve", "--pin", str(self.pin(PIN.format(sha=SHA))), "--platform", "linux-aarch64-gnu")
self.assertEqual(code, 1)
self.assertIn("no pin for platform", err)
def test_malformed_pins_fail(self):
good = PIN.format(sha=SHA)
for label, text in {
"short digest": good.replace(SHA, "a" * 63),
"uppercase digest": good.replace(SHA, "A" * 64),
"no digest": good.replace(f'sha256 = "{SHA}"\n', ""),
"path in asset": good.replace('asset = "rustfs-', 'asset = "../rustfs-'),
"non-zip asset": good.replace("v1.0.5.zip", "v1.0.5.deb"),
"asset of another version": good.replace("v1.0.5.zip", "v1.0.4.zip"),
"path in executable": good.replace('executable = "rustfs"', 'executable = "bin/rustfs"'),
"wrong schema": good.replace("schema = 1", "schema = 2"),
"boolean schema": good.replace("schema = 1", "schema = true"),
"no repository": good.replace('repository = "rustfs/rustfs"\n', ""),
"unknown key": good + 'url = "https://example.invalid"\n',
"not toml": "schema = \n",
}.items():
with self.subTest(label):
code, _, _ = run("resolve", "--pin", str(self.pin(text)), "--platform", "linux-x86_64-gnu")
self.assertEqual(code, 1)
def test_resolves_identity_as_step_outputs(self):
code, out, err = run("resolve", "--pin", str(self.pin(PIN.format(sha=SHA))), "--platform", "linux-x86_64-gnu")
self.assertEqual(code, 0, err)
self.assertEqual(out.splitlines(), ["repository=rustfs/rustfs", "version=1.0.5", "tag=1.0.5",
"asset=rustfs-linux-x86_64-gnu-v1.0.5.zip", f"sha256={SHA}",
"executable=rustfs"])
def test_repository_pin_is_valid(self):
code, out, err = run("resolve", "--pin", str(ROOT / ".config/reference-binary.toml"),
"--platform", "linux-x86_64-gnu")
self.assertEqual(code, 0, err)
self.assertIn("version=1.0.5", out.splitlines())
class UnpackTests(Base):
def unpack(self, archive, sha):
return run("unpack", "--archive", str(archive), "--sha256", sha, "--executable", "rustfs",
"--destination", str(self.root / "out"))
def test_digest_mismatch_fails_and_extracts_nothing(self):
archive, _ = self.archive({"rustfs": b"binary"})
code, _, err = self.unpack(archive, SHA)
self.assertEqual(code, 1)
self.assertIn("does not match the pinned digest", err)
self.assertFalse((self.root / "out").exists())
def test_missing_archive_fails(self):
code, _, err = self.unpack(self.root / "absent.zip", SHA)
self.assertEqual(code, 1)
self.assertIn("missing or unreadable", err)
def test_archive_without_top_level_executable_fails(self):
archive, sha = self.archive({"bin/rustfs": b"binary"})
code, _, err = self.unpack(archive, sha)
self.assertEqual(code, 1)
self.assertIn("exactly one top-level", err)
def test_empty_executable_fails(self):
archive, sha = self.archive({"rustfs": b""})
code, _, _ = self.unpack(archive, sha)
self.assertEqual(code, 1)
def test_non_zip_archive_fails(self):
archive = self.root / "rustfs.zip"
archive.write_bytes(b"not a zip")
code, _, err = self.unpack(archive, hashlib.sha256(b"not a zip").hexdigest())
self.assertEqual(code, 1)
self.assertIn("not a valid zip", err)
def test_extracts_only_the_executable(self):
archive, sha = self.archive({"rustfs": b"binary", "../escape": b"x", "README": b"r"})
code, out, err = self.unpack(archive, sha)
self.assertEqual(code, 0, err)
target = self.root / "out" / "rustfs"
self.assertEqual(out.strip(), f"path={target}")
self.assertEqual(target.read_bytes(), b"binary")
self.assertTrue(target.stat().st_mode & 0o111)
self.assertEqual(sorted(p.name for p in (self.root / "out").iterdir()), ["rustfs"])
self.assertFalse((self.root / "escape").exists())
if __name__ == "__main__":
unittest.main()