ci(s3-tests): add reference parity lane against pinned 1.0.5 (#9288 )

2026-11-11 04:39:00 +00:00 · 2026-11-10 22:50:17 -05:00
13 changed files with 1455 additions and 0 deletions
@@ -66,6 +66,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_reference_binary.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
 	$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/release -p "test_*.py"
@@ -92,6 +93,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
 	./scripts/validate_object_data_cache_cold_stampede.sh --self-test
@@ -0,0 +1,20 @@
+# Pinned RustFS release used as the "reference" target of parity lanes
+# (.github/workflows/reference-parity.yml). This is the single place that names
+# the reference release; .github/actions/reference-binary resolves, downloads and
+# verifies it, and fails on a missing pin, a missing asset or a digest mismatch.
+#
+# Bumping the reference is a reviewed event: every per-test difference between
+# the candidate and the reference is measured against this exact build.
+#
+# Each sha256 is the value published in the release's SHA256SUMS asset, which
+# matches the digest GitHub reports for the same asset. The action checks both.
+
+schema = 1
+repository = "rustfs/rustfs"
+version = "1.0.5"
+tag = "1.0.5"
+
+[platforms.linux-x86_64-gnu]
+asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
+sha256 = "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
+executable = "rustfs"
@@ -0,0 +1,24 @@
+# Targets of the s3-tests reference parity lane (.github/workflows/reference-parity.yml).
+#
+# Exactly one target uses `binary = "reference"` (the release pinned in
+# .config/reference-binary.toml). Every other target uses `binary = "candidate"`
+# (the binary built from the branch under test) and is compared per node id
+# against the reference. All targets run the same harness, test list and server
+# configuration; `env` adds only the target's own RUSTFS_* settings.
+#
+# Adding a target is one more [[targets]] table, for example:
+#
+#   [[targets]]
+#   name = "candidate-gateway"
+#   binary = "candidate"
+#   env = { RUSTFS_S3_STACK = "gateway" }
+
+schema = 1
+
+[[targets]]
+name = "reference"
+binary = "reference"
+
+[[targets]]
+name = "candidate-legacy"
+binary = "candidate"
@@ -0,0 +1,12 @@
+# Tolerated per-test differences of the s3-tests reference parity lane.
+#
+# One entry per line, four fields separated by " | ":
+#
+#   <target> | <pytest node id> | <tracking issue URL> | <reason>
+#
+# <target> is a candidate name from .config/reference-parity-targets.toml, the
+# node id is exactly as the lane prints it (s3tests/functional/test_s3.py::...),
+# and the issue URL is https://github.com/rustfs/<repo>/issues/<number>.
+#
+# Same semantics as scripts/junit_diff.py --allow: an entry whose difference no
+# longer occurs is stale and fails the lane, so remove it when it is fixed.
@@ -0,0 +1,96 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "Reference RustFS binary"
+description: "Download the pinned reference RustFS release, verify its sha256 and expose the executable path. A missing pin, missing asset or digest mismatch fails the step."
+
+inputs:
+  pin-file:
+    description: "Pin file naming the reference release (repository-relative)"
+    required: false
+    default: ".config/reference-binary.toml"
+  platform:
+    description: "Platform table in the pin file"
+    required: false
+    default: "linux-x86_64-gnu"
+  destination:
+    description: "Directory that receives the executable; defaults to $RUNNER_TEMP/reference-binary"
+    required: false
+    default: ""
+
+outputs:
+  path:
+    description: "Absolute path of the verified executable"
+    value: ${{ steps.unpack.outputs.path }}
+  version:
+    description: "Pinned reference version"
+    value: ${{ steps.pin.outputs.version }}
+  sha256:
+    description: "Pinned and verified archive sha256"
+    value: ${{ steps.pin.outputs.sha256 }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Resolve reference pin
+      id: pin
+      shell: bash
+      env:
+        PIN_TOOL: ${{ github.action_path }}/reference_binary.py
+        PIN_FILE: ${{ inputs.pin-file }}
+        PIN_PLATFORM: ${{ inputs.platform }}
+      run: |
+        set -euo pipefail
+        resolved="$(python3 "$PIN_TOOL" resolve --pin "$PIN_FILE" --platform "$PIN_PLATFORM")"
+        printf '%s\n' "$resolved"
+        printf '%s\n' "$resolved" >> "$GITHUB_OUTPUT"
+
+    - name: Download and verify reference archive
+      id: download
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+        ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
+        REF_REPOSITORY: ${{ steps.pin.outputs.repository }}
+        REF_TAG: ${{ steps.pin.outputs.tag }}
+        REF_ASSET: ${{ steps.pin.outputs.asset }}
+        REF_SHA256: ${{ steps.pin.outputs.sha256 }}
+      run: |
+        set -euo pipefail
+        archive="$RUNNER_TEMP/reference-archive/$REF_ASSET"
+        # Fails unless the release lists exactly one asset of this name whose
+        # published digest equals the pin and whose downloaded bytes match it.
+        python3 "$ASSET_DOWNLOADER" --repository "$REF_REPOSITORY" --tag "$REF_TAG" \
+          --asset "$REF_ASSET" --output "$archive" --sha256 "$REF_SHA256"
+        echo "archive=$archive" >> "$GITHUB_OUTPUT"
+
+    - name: Unpack reference executable
+      id: unpack
+      shell: bash
+      env:
+        PIN_TOOL: ${{ github.action_path }}/reference_binary.py
+        REF_ARCHIVE: ${{ steps.download.outputs.archive }}
+        REF_SHA256: ${{ steps.pin.outputs.sha256 }}
+        REF_EXECUTABLE: ${{ steps.pin.outputs.executable }}
+        REF_DESTINATION: ${{ inputs.destination }}
+      run: |
+        set -euo pipefail
+        destination="${REF_DESTINATION:-$RUNNER_TEMP/reference-binary}"
+        unpacked="$(python3 "$PIN_TOOL" unpack --archive "$REF_ARCHIVE" --sha256 "$REF_SHA256" \
+          --executable "$REF_EXECUTABLE" --destination "$destination")"
+        path="${unpacked#path=}"
+        test -x "$path"
+        echo "Reference executable: $path"
+        "$path" --version
+        echo "path=$path" >> "$GITHUB_OUTPUT"
@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Resolve and unpack the pinned reference RustFS release binary.
+
+`resolve` reads the pin file and prints the selected platform's identity as
+`key=value` lines (GitHub step-output format). `unpack` re-verifies the
+downloaded archive digest against the pin and extracts exactly one executable.
+Every missing or malformed input is a failure; nothing is skipped.
+"""
+
+import argparse
+import hashlib
+from pathlib import Path
+import re
+import sys
+import tomllib
+import zipfile
+
+REPOSITORY = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*")
+TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*")
+PLATFORM = re.compile(r"[a-z0-9][a-z0-9_-]*")
+ASSET = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*\.zip")
+SHA256 = re.compile(r"[0-9a-f]{64}")
+EXECUTABLE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
+MAX_EXECUTABLE_BYTES = 1024 * 1024 * 1024
+
+
+class PinError(Exception):
+    """A pin, archive or argument that cannot be trusted."""
+
+
+def field(table, key, pattern, where):
+    value = table.get(key)
+    if not isinstance(value, str) or not pattern.fullmatch(value):
+        raise PinError(f"{where}: missing or invalid '{key}'")
+    return value
+
+
+def resolve(pin_path, platform):
+    if not PLATFORM.fullmatch(platform or ""):
+        raise PinError("invalid platform name")
+    try:
+        data = tomllib.loads(Path(pin_path).read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise PinError(f"{pin_path}: reference pin file is missing") from exc
+    except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
+        raise PinError(f"{pin_path}: cannot read reference pin: {exc}") from exc
+    if data.get("schema") != 1 or type(data.get("schema")) is not int:
+        raise PinError(f"{pin_path}: unsupported pin schema")
+    identity = {
+        "repository": field(data, "repository", REPOSITORY, pin_path),
+        "version": field(data, "version", TAG, pin_path),
+        "tag": field(data, "tag", TAG, pin_path),
+    }
+    platforms = data.get("platforms")
+    if not isinstance(platforms, dict) or not platforms:
+        raise PinError(f"{pin_path}: no pinned platforms")
+    entry = platforms.get(platform)
+    if not isinstance(entry, dict):
+        raise PinError(f"{pin_path}: no pin for platform '{platform}'")
+    where = f"{pin_path} [platforms.{platform}]"
+    identity["asset"] = field(entry, "asset", ASSET, where)
+    identity["sha256"] = field(entry, "sha256", SHA256, where)
+    identity["executable"] = field(entry, "executable", EXECUTABLE, where)
+    if identity["version"] not in identity["asset"]:
+        raise PinError(f"{where}: asset name does not carry version {identity['version']}")
+    unknown = set(entry) - {"asset", "sha256", "executable"}
+    if unknown:
+        raise PinError(f"{where}: unknown keys {sorted(unknown)}")
+    return identity
+
+
+def file_sha256(path):
+    with open(path, "rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def unpack(archive, sha256, executable, destination):
+    if not SHA256.fullmatch(sha256 or "") or not EXECUTABLE.fullmatch(executable or ""):
+        raise PinError("invalid expected digest or executable name")
+    try:
+        actual = file_sha256(archive)
+    except OSError as exc:
+        raise PinError(f"{archive}: reference archive is missing or unreadable") from exc
+    if actual != sha256:
+        raise PinError(f"{archive}: sha256 {actual} does not match the pinned digest")
+    try:
+        with zipfile.ZipFile(archive) as bundle:
+            matches = [info for info in bundle.infolist() if info.filename == executable]
+            if len(matches) != 1:
+                raise PinError(f"{archive}: must contain exactly one top-level '{executable}'")
+            info = matches[0]
+            if info.is_dir() or not 0 < info.file_size <= MAX_EXECUTABLE_BYTES:
+                raise PinError(f"{archive}: '{executable}' is not a plausible executable")
+            destination = Path(destination)
+            destination.mkdir(parents=True, exist_ok=True)
+            target = destination / executable
+            with bundle.open(info) as source, open(target, "wb") as sink:
+                while chunk := source.read(1024 * 1024):
+                    sink.write(chunk)
+    except zipfile.BadZipFile as exc:
+        raise PinError(f"{archive}: not a valid zip archive") from exc
+    target.chmod(0o755)
+    return target
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    sub = parser.add_subparsers(dest="mode", required=True)
+    res = sub.add_parser("resolve")
+    res.add_argument("--pin", required=True)
+    res.add_argument("--platform", required=True)
+    unp = sub.add_parser("unpack")
+    unp.add_argument("--archive", required=True)
+    unp.add_argument("--sha256", required=True)
+    unp.add_argument("--executable", required=True)
+    unp.add_argument("--destination", required=True)
+    args = parser.parse_args(argv)
+    try:
+        if args.mode == "resolve":
+            for key, value in resolve(args.pin, args.platform).items():
+                print(f"{key}={value}")
+        else:
+            print(f"path={unpack(args.archive, args.sha256, args.executable, args.destination)}")
+    except PinError as exc:
+        print(f"reference binary: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -869,6 +869,33 @@
      "failure_owner": "watchdog",
      "origin": "baseline"
    },
+    {
+      "workflow": ".github/workflows/reference-parity.yml",
+      "name": "Reference Parity",
+      "roles": [
+        "pr",
+        "periodic",
+        "manual"
+      ],
+      "events": [
+        "pull_request",
+        "schedule",
+        "workflow_dispatch"
+      ],
+      "cadence": "nightly",
+      "timezone": "UTC",
+      "schedule": [
+        "37 1 * * *"
+      ],
+      "owner": "@overtrue",
+      "policy": "active",
+      "monitor": "workflow",
+      "max_age_hours": 36,
+      "result_owner": "reference-parity.yml",
+      "disposition": "s3-tests per-node-id parity of branch candidates against the pinned reference release",
+      "failure_owner": "watchdog",
+      "origin": "added"
+    },
    {
      "workflow": ".github/workflows/functional-chain-health.yml",
      "name": "Functional chain health",
@@ -0,0 +1,308 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# s3-tests reference parity lane (rustfs/backlog#2773).
+#
+# Runs the ceph/s3-tests implemented list (scripts/s3-tests/implemented_tests.txt)
+# against every target in .config/reference-parity-targets.toml and fails on
+# any per-node-id difference between a candidate and the reference:
+#
+#   - reference: the release pinned in .config/reference-binary.toml, downloaded
+#     and sha256-verified by .github/actions/reference-binary;
+#   - candidate targets: the debug binary built from this checkout exactly as
+#     ci.yml's build-rustfs-debug-binary job builds it for S3 Implemented Tests.
+#
+# Every target runs the same harness (scripts/s3-tests/run.sh from this
+# checkout), test list and server configuration on its own hosted runner. A
+# target's failures get one rerun against a fresh server; a case that passes it
+# is flaky, not a difference. scripts/s3-tests/reference_parity.py judges the
+# result and honours .config/reference-parity-tolerated.txt, whose entries must
+# each match a real difference.
+#
+# Report-only: not part of the Test and Lint aggregate. Scheduled failures are
+# owned by the scheduled-validation watchdog (.github/scheduled-validations.json).
+
+name: Reference Parity
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/reference-parity.yml"
+      - ".github/actions/reference-binary/**"
+      - ".config/reference-binary.toml"
+      - ".config/reference-parity-targets.toml"
+      - ".config/reference-parity-tolerated.txt"
+      - "scripts/s3-tests/reference_parity.py"
+      - "scripts/junit_diff.py"
+  schedule:
+    - cron: "37 1 * * *"
+  workflow_dispatch:
+
+# Matches ci.yml: these variables are hashed into the rust-cache key, so the
+# candidate build reads the same ci-dev cache as build-rustfs-debug-binary.
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  plan:
+    name: Plan parity targets
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      targets: ${{ steps.targets.outputs.targets }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Validate target list
+        id: targets
+        run: |
+          set -euo pipefail
+          targets="$(python3 scripts/s3-tests/reference_parity.py targets --config .config/reference-parity-targets.toml)"
+          echo "targets=${targets}" >> "$GITHUB_OUTPUT"
+          echo "Parity targets: ${targets}"
+
+  build-candidate:
+    name: Build candidate binary
+    # Critical-path CPU and cache ownership: the same debug build as ci.yml's
+    # build-rustfs-debug-binary, reading the ci-dev cache written on sm-standard-4.
+    runs-on: sm-standard-4
+    timeout-minutes: 60
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Build debug binary
+        run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
+
+      - name: Upload candidate binary
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: reference-parity-candidate-binary
+          overwrite: true
+          path: |
+            target/debug/rustfs
+            target/debug/rustfs.e2e.json
+          if-no-files-found: error
+          retention-days: 1
+
+  s3tests:
+    name: s3-tests (${{ matrix.target.name }})
+    needs: [ plan, build-candidate ]
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ${{ fromJSON(needs.plan.outputs.targets) }}
+    env:
+      TARGET_NAME: ${{ matrix.target.name }}
+      TARGET_KIND: ${{ matrix.target.binary }}
+      # Above this many first-run failures the target is systemically broken;
+      # a rerun would only repeat it, so the first run is judged as is.
+      RERUN_CAP: "50"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Fetch pinned reference binary
+        id: reference
+        if: matrix.target.binary == 'reference'
+        uses: ./.github/actions/reference-binary
+
+      - name: Download candidate binary
+        if: matrix.target.binary == 'candidate'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: reference-parity-candidate-binary
+          path: target/debug
+
+      - name: Select target binary
+        env:
+          REFERENCE_PATH: ${{ steps.reference.outputs.path }}
+        run: |
+          set -euo pipefail
+          case "$TARGET_KIND" in
+            reference) binary="$REFERENCE_PATH" ;;
+            candidate) chmod +x target/debug/rustfs; binary="$PWD/target/debug/rustfs" ;;
+            *) echo "unknown target binary kind: $TARGET_KIND" >&2; exit 1 ;;
+          esac
+          test -x "$binary"
+          echo "TARGET_BINARY=$binary" >> "$GITHUB_ENV"
+          mkdir -p "parity/$TARGET_NAME"
+          {
+            echo "target=$TARGET_NAME"
+            echo "binary=$TARGET_KIND"
+            echo "sha256=$(sha256sum "$binary" | cut -d' ' -f1)"
+            echo "version=$("$binary" --version | tr '\n' ' ')"
+          } | tee "parity/$TARGET_NAME/target.txt"
+
+      - name: Apply target environment
+        env:
+          TARGET_JSON: ${{ toJSON(matrix.target) }}
+        run: |
+          set -euo pipefail
+          python3 scripts/s3-tests/reference_parity.py export-env --target-json "$TARGET_JSON" \
+            | tee -a "$GITHUB_ENV" "parity/$TARGET_NAME/target-env.txt"
+
+      - name: Run implemented s3-tests
+        run: |
+          set -uo pipefail
+          report="parity/$TARGET_NAME"
+          harness="artifacts/s3tests-single"
+
+          # Same invocation as ci.yml's S3 Implemented Tests, with a fresh data
+          # root and port per run. $1 optionally replaces the test list.
+          run_suite() {
+            local list="$1" label="$2" root port
+            root="${RUNNER_TEMP}/rustfs-s3tests-${label}"
+            rm -rf "$harness" "$root"
+            mkdir -p "$root"
+            port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
+            env DEPLOY_MODE=binary \
+              RUSTFS_BINARY="$TARGET_BINARY" \
+              TEST_MODE=single \
+              MAXFAIL=0 \
+              XDIST=4 \
+              S3_PORT="$port" \
+              DATA_ROOT="$root" \
+              S3TESTS_CONF="$harness/s3tests.conf" \
+              ${list:+IMPLEMENTED_TESTS_FILE="$list"} \
+              ./scripts/s3-tests/run.sh
+          }
+
+          # A run is usable only if it produced a JUnit report; test failures
+          # are data for the comparison, a missing report is a lane failure.
+          keep() {
+            local prefix="$1"
+            for file in pytest.log compat-report.md; do
+              if [[ -f "$harness/$file" ]]; then cp "$harness/$file" "$report/${prefix}${file}"; fi
+            done
+            if [[ -f "$harness/rustfs-single/rustfs.log" ]]; then cp "$harness/rustfs-single/rustfs.log" "$report/${prefix}rustfs.log"; fi
+            test -s "$harness/junit.xml" || { echo "::error::$TARGET_NAME produced no JUnit report (${prefix:-first} run)"; return 1; }
+            cp "$harness/junit.xml" "$report/${prefix}junit.xml"
+          }
+
+          run_suite "" first
+          echo "first_run_exit=$?" >> "$report/target.txt"
+          keep "" || exit 1
+          cp "$harness/selected-nodeids.txt" "$report/selected-nodeids.txt" || exit 1
+
+          failures="$(python3 scripts/s3-tests/reference_parity.py failed --report "$report")" || exit 1
+          count="$(printf '%s' "$failures" | grep -c . || true)"
+          echo "first_run_failures=$count" >> "$report/target.txt"
+          if (( count == 0 )); then
+            exit 0
+          fi
+          if (( count > RERUN_CAP )); then
+            echo "::warning::$TARGET_NAME has $count failures (cap $RERUN_CAP); judging the first run without a rerun"
+            echo "rerun=skipped-over-cap" >> "$report/target.txt"
+            exit 0
+          fi
+          printf '%s\n' "$failures" > "$RUNNER_TEMP/rerun-tests.txt"
+          run_suite "$RUNNER_TEMP/rerun-tests.txt" rerun
+          echo "rerun_exit=$?" >> "$report/target.txt"
+          keep rerun- || exit 1
+
+      - name: Upload target report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: reference-parity-report-${{ matrix.target.name }}
+          overwrite: true
+          path: parity/${{ matrix.target.name }}/
+          if-no-files-found: error
+          retention-days: 14
+
+  compare:
+    name: Compare targets
+    needs: [ plan, s3tests ]
+    if: always() && needs.plan.result == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download target reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: reference-parity-report-*
+          path: parity-download
+
+      - name: Compare per node id
+        env:
+          TARGETS: ${{ needs.plan.outputs.targets }}
+          S3TESTS_RESULT: ${{ needs.s3tests.result }}
+        run: |
+          set -uo pipefail
+          mkdir -p parity
+          for dir in parity-download/reference-parity-report-*; do
+            if [[ -d "$dir" ]]; then mv "$dir" "parity/${dir#parity-download/reference-parity-report-}"; fi
+          done
+          python3 scripts/s3-tests/reference_parity.py compare \
+            --targets-json "$TARGETS" \
+            --reports parity \
+            --tolerated .config/reference-parity-tolerated.txt \
+            --markdown "$GITHUB_STEP_SUMMARY" \
+            --json parity/diff.json > parity/diff.md
+          status=$?
+          cat parity/diff.md
+          if [[ "$S3TESTS_RESULT" != "success" ]]; then
+            echo "::error::s3-tests target jobs finished with '$S3TESTS_RESULT'"
+            status=1
+          fi
+          exit "$status"
+
+      - name: Upload parity diff
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: reference-parity-diff
+          overwrite: true
+          path: |
+            parity/diff.md
+            parity/diff.json
+          if-no-files-found: warn
+          retention-days: 14
@@ -101,6 +101,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
 | `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes (disabled) | `make coverage` |
 | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
 | `e2e-upgrade.yml` (weekly, and on version tags) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
+| `reference-parity.yml` (nightly, and on PRs that change the lane's own files) | `plan`, `build-candidate`, `s3tests` (one job per target in `.config/reference-parity-targets.toml`), `compare` | report-only parity gate: fails on any per-node-id s3-tests difference between a branch candidate and the release pinned in `.config/reference-binary.toml` that `.config/reference-parity-tolerated.txt` does not list, and on a stale tolerated entry; per-target JUnit, logs and `diff.md`/`diff.json` | yes | `DEPLOY_MODE=binary RUSTFS_BINARY=<binary> MAXFAIL=0 XDIST=4 ./scripts/s3-tests/run.sh` once per target, then `python3 scripts/s3-tests/reference_parity.py compare` as in the `compare` job |
 | `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | yes | see the PR row |
 | `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | yes (disabled) | see the PR row |
 | `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | yes | dispatch, optionally with `source_ref` |
@@ -57,6 +57,9 @@ their issue closes.
 | `test_object_generation_protocol_model.py` | dev-tool | Bounded single-slot, four-voter promise/accept design model and recovery/retirement examples; no production runtime evidence | `python3 scripts/test_object_generation_protocol_model.py`; [Generation contract](../docs/architecture/unified-object-generation.md#executable-design-model-and-acceptance-mapping) |
 | `junit_diff.py` | dev-tool | Case-by-case pass/fail/skip diff of two JUnit XML reports (reference vs candidate stack); exit 1 on any difference or stale `--allow` entry, 2 on unreadable input | `python3 scripts/junit_diff.py a.xml b.xml`; backlog T3.1 |
 | `test_junit_diff.py` | ci-gate | Contract tests and fixtures (`scripts/tests/junit_diff/`) for `junit_diff.py` | `make script-tests` |
+| `s3-tests/reference_parity.py` | ci-gate | Plans the reference parity targets, lists a run's failures for the single rerun, and compares every candidate's s3-tests JUnit against the pinned reference per node id with a tolerated-difference file | `.github/workflows/reference-parity.yml`; backlog T3.2 |
+| `s3-tests/test_reference_parity.py` | ci-gate | Contract tests for `reference_parity.py` | `make script-tests` |
+| `test_reference_binary.py` | ci-gate | Contract tests for `.github/actions/reference-binary/reference_binary.py` (pin resolution and verified unpacking) | `make script-tests` |
 | `e2e-run.sh` | ci-gate | Boots a rustfs server and runs the `s3s-e2e` black-box conformance tool against it | ci.yml `e2e-tests` jobs; `docs/testing/README.md` |
 | `run_ecstore_validation_suite.sh` | dev-tool | ecstore black-box validation suite (`quick`/`full`/`destructive`/`fuzz` profiles) | `docs/testing/README.md`, `docs/testing/ecstore-validation-suite-design.md` |
 | `run_e2e_tests.sh` | dev-tool | Local `e2e_test` crate runner (starts a server, applies filters, cleans up) | `crates/e2e_test/README.md` |
@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Plan and judge the s3-tests reference parity lane.
+
+Subcommands:
+  targets    validate the target list and print it as a JSON matrix
+  export-env validate one target's extra environment and print KEY=VALUE lines
+  failed     print the failed test names of one run, in implemented_tests.txt form
+  compare    compare every candidate against the reference, per pytest node id
+
+`compare` reads <reports>/<target>/selected-nodeids.txt, junit.xml and the
+optional rerun-junit.xml of every target. Each node id is reduced to pass /
+fail / skip / missing; a case that failed and then passed its single rerun is a
+pass and is listed as flaky. A tolerated entry must match a real difference.
+
+Exit codes: 0 no untolerated difference and no stale entry; 1 a difference, a
+stale tolerated entry or an incomplete run; 2 missing or malformed input.
+"""
+
+import argparse
+import json
+from pathlib import Path
+import re
+import sys
+import tomllib
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import junit_diff  # noqa: E402  (sibling script, not a package)
+
+PASS, FAIL, SKIP, MISSING = junit_diff.PASS, junit_diff.FAIL, junit_diff.SKIP, junit_diff.MISSING
+NAME = re.compile(r"[a-z][a-z0-9-]{0,39}")
+ENV_KEY = re.compile(r"RUSTFS_[A-Z0-9_]+")
+ENV_VALUE = re.compile(r"[A-Za-z0-9._,:/=+-]*")
+ISSUE = re.compile(r"https://github\.com/rustfs/[A-Za-z0-9_.-]+/issues/[1-9][0-9]*")
+NODEID = re.compile(r"[A-Za-z0-9_/]+\.py(::[A-Za-z0-9_]+)+(\[[^\]\s]*\])?")
+TEST_FILE_PREFIX = "s3tests/functional/test_s3.py::"
+BINARIES = ("reference", "candidate")
+
+
+class InputError(Exception):
+    """Input that cannot be trusted; maps to exit code 2."""
+
+
+# --- target list -----------------------------------------------------------
+
+def load_targets(path):
+    try:
+        data = tomllib.loads(Path(path).read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise InputError(f"{path}: target list is missing") from exc
+    except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
+        raise InputError(f"{path}: cannot read target list: {exc}") from exc
+    if data.get("schema") != 1 or type(data.get("schema")) is not int or set(data) != {"schema", "targets"}:
+        raise InputError(f"{path}: unsupported target list schema")
+    return validate_targets(data["targets"], str(path))
+
+
+def validate_targets(targets, where="targets"):
+    if not isinstance(targets, list) or len(targets) < 2:
+        raise InputError(f"{where}: need one reference and at least one candidate")
+    result, names = [], set()
+    for item in targets:
+        if not isinstance(item, dict) or not set(item) <= {"name", "binary", "env"}:
+            raise InputError(f"{where}: invalid target entry {item!r}")
+        name, binary, env = item.get("name"), item.get("binary"), item.get("env", {})
+        if not isinstance(name, str) or not NAME.fullmatch(name) or name in names:
+            raise InputError(f"{where}: invalid or duplicate target name {name!r}")
+        if binary not in BINARIES:
+            raise InputError(f"{where}: target {name} has unknown binary {binary!r}")
+        validate_env(env, f"{where}: target {name}")
+        names.add(name)
+        result.append({"name": name, "binary": binary, "env": dict(env)})
+    if [t["binary"] for t in result].count("reference") != 1:
+        raise InputError(f"{where}: exactly one target must use the reference binary")
+    return result
+
+
+def validate_env(env, where):
+    if not isinstance(env, dict):
+        raise InputError(f"{where}: env must be a table")
+    for key, value in env.items():
+        if not isinstance(key, str) or not ENV_KEY.fullmatch(key):
+            raise InputError(f"{where}: env key {key!r} must match RUSTFS_[A-Z0-9_]+")
+        if not isinstance(value, str) or not ENV_VALUE.fullmatch(value):
+            raise InputError(f"{where}: env value for {key} has unsupported characters")
+    return env
+
+
+# --- node ids and reports --------------------------------------------------
+
+def nodeid_to_case(nodeid):
+    """Map a pytest node id to the classname::name id that junit_diff uses."""
+    if not NODEID.fullmatch(nodeid):
+        raise InputError(f"invalid pytest node id {nodeid!r}")
+    path, *parts = nodeid.split("::")
+    module = path[: -len(".py")].replace("/", ".")
+    classname = ".".join([module, *parts[:-1]])
+    return junit_diff.case_id(classname, parts[-1])
+
+
+def read_selected(path):
+    try:
+        lines = Path(path).read_text(encoding="utf-8").splitlines()
+    except FileNotFoundError as exc:
+        raise InputError(f"{path}: selected node id list is missing") from exc
+    except (OSError, UnicodeDecodeError) as exc:
+        raise InputError(f"{path}: cannot read selected node ids: {exc}") from exc
+    nodeids = [line.strip() for line in lines if line.strip()]
+    if not nodeids:
+        raise InputError(f"{path}: selected node id list is empty")
+    if len(set(nodeids)) != len(nodeids):
+        raise InputError(f"{path}: duplicate selected node id")
+    by_case = {}
+    for nodeid in nodeids:
+        case = nodeid_to_case(nodeid)
+        if case in by_case:
+            raise InputError(f"{path}: node ids {by_case[case]} and {nodeid} collide")
+        by_case[case] = nodeid
+    return nodeids, by_case
+
+
+def target_states(directory):
+    """Return (nodeids, {nodeid: state}, [flaky nodeids]) for one target run."""
+    directory = Path(directory)
+    if not directory.is_dir():
+        raise InputError(f"{directory}: report directory is missing")
+    nodeids, by_case = read_selected(directory / "selected-nodeids.txt")
+    try:
+        first, _ = junit_diff.parse_report(str(directory / "junit.xml"))
+    except junit_diff.InputError as exc:
+        raise InputError(str(exc)) from exc
+    unexpected = sorted(set(first) - set(by_case))
+    if unexpected:
+        raise InputError(f"{directory}: junit.xml has unselected cases: {', '.join(unexpected[:5])}")
+    states = {by_case[case]: state for case, state in first.items()}
+    flaky = []
+    rerun_path = directory / "rerun-junit.xml"
+    if rerun_path.exists():
+        try:
+            rerun, _ = junit_diff.parse_report(str(rerun_path))
+        except junit_diff.InputError as exc:
+            raise InputError(str(exc)) from exc
+        for case, state in sorted(rerun.items()):
+            nodeid = by_case.get(case)
+            if nodeid is None or states.get(nodeid) != FAIL:
+                raise InputError(f"{rerun_path}: {case} was rerun without failing first")
+            if state == PASS:
+                flaky.append(nodeid)
+            states[nodeid] = PASS if state == PASS else FAIL
+    for nodeid in nodeids:
+        states.setdefault(nodeid, MISSING)
+    return nodeids, states, flaky
+
+
+def failed_names(directory):
+    _, states, _ = target_states(directory)
+    names = []
+    for nodeid, state in states.items():
+        if state == FAIL:
+            if not nodeid.startswith(TEST_FILE_PREFIX):
+                raise InputError(f"cannot express {nodeid} as an implemented_tests.txt entry")
+            names.append(nodeid[len(TEST_FILE_PREFIX):])
+    return sorted(names)
+
+
+# --- tolerated differences ---------------------------------------------------
+
+def read_tolerated(path, candidates):
+    try:
+        lines = Path(path).read_text(encoding="utf-8").splitlines()
+    except FileNotFoundError as exc:
+        raise InputError(f"{path}: tolerated-difference file is missing") from exc
+    except (OSError, UnicodeDecodeError) as exc:
+        raise InputError(f"{path}: cannot read tolerated differences: {exc}") from exc
+    entries = {}
+    for number, line in enumerate(lines, 1):
+        if not line.strip() or line.lstrip().startswith("#"):
+            continue
+        fields = [part.strip() for part in line.split(" | ")]
+        if len(fields) != 4 or not all(fields):
+            raise InputError(f"{path}:{number}: expected '<target> | <node id> | <issue URL> | <reason>'")
+        target, nodeid, issue, _reason = fields
+        if target not in candidates:
+            raise InputError(f"{path}:{number}: {target!r} is not a candidate target")
+        nodeid_to_case(nodeid)
+        if not ISSUE.fullmatch(issue):
+            raise InputError(f"{path}:{number}: tracking issue must be a https://github.com/rustfs/<repo>/issues/<n> URL")
+        if (target, nodeid) in entries:
+            raise InputError(f"{path}:{number}: duplicate entry for {target} {nodeid}")
+        entries[(target, nodeid)] = issue
+    return entries
+
+
+# --- comparison ----------------------------------------------------------------
+
+def count(states):
+    result = {PASS: 0, FAIL: 0, SKIP: 0, MISSING: 0}
+    for state in states.values():
+        result[state] += 1
+    return result
+
+
+def compare(targets, reports, tolerated_path):
+    reference = next(t["name"] for t in targets if t["binary"] == "reference")
+    candidates = [t["name"] for t in targets if t["binary"] == "candidate"]
+    tolerated = read_tolerated(tolerated_path, set(candidates))
+    runs = {t["name"]: target_states(Path(reports) / t["name"]) for t in targets}
+    selected = runs[reference][0]
+    for name, (nodeids, _, _) in runs.items():
+        if nodeids != selected:
+            raise InputError(f"{name}: selected node ids differ from the {reference} run")
+    ref_states = runs[reference][1]
+    differences = []
+    for name in candidates:
+        states = runs[name][1]
+        for nodeid in selected:
+            if ref_states[nodeid] != states[nodeid]:
+                issue = tolerated.get((name, nodeid))
+                differences.append({"target": name, "nodeid": nodeid, "reference": ref_states[nodeid],
+                                    "candidate": states[nodeid], "tolerated": issue})
+    seen = {(d["target"], d["nodeid"]) for d in differences}
+    stale = [{"target": t, "nodeid": n, "issue": i} for (t, n), i in sorted(tolerated.items()) if (t, n) not in seen]
+    incomplete = {name: [n for n in selected if run[1][n] == MISSING] for name, run in runs.items()}
+    return {
+        "reference": reference,
+        "candidates": candidates,
+        "selected": len(selected),
+        "counts": {name: count(run[1]) for name, run in runs.items()},
+        "flaky": {name: run[2] for name, run in runs.items()},
+        "differences": differences,
+        "stale_tolerated": stale,
+        "incomplete": {name: ids for name, ids in incomplete.items() if ids},
+    }
+
+
+def failed(summary):
+    return bool(summary["stale_tolerated"] or summary["incomplete"]
+                or any(not d["tolerated"] for d in summary["differences"]))
+
+
+def render(summary):
+    out = ["## s3-tests reference parity", ""]
+    out.append(f"Reference target `{summary['reference']}`; {summary['selected']} selected node ids per target.")
+    out += ["", "| Target | pass | fail | skip | missing | flaky (passed on rerun) |", "|---|---:|---:|---:|---:|---:|"]
+    for name, c in summary["counts"].items():
+        out.append(f"| `{name}` | {c[PASS]} | {c[FAIL]} | {c[SKIP]} | {c[MISSING]} | {len(summary['flaky'][name])} |")
+    diffs = summary["differences"]
+    out += ["", f"### Differences ({len(diffs)})", ""]
+    if diffs:
+        out += ["| Target | Node id | reference | candidate | Tolerated |", "|---|---|---|---|---|"]
+        for d in diffs:
+            out.append(f"| `{d['target']}` | `{d['nodeid']}` | {d['reference']} | {d['candidate']} | {d['tolerated'] or 'no'} |")
+    else:
+        out.append("No differences.")
+    if summary["stale_tolerated"]:
+        out += ["", "### Stale tolerated entries (no matching difference; remove them)", ""]
+        out += [f"- `{s['target']}` `{s['nodeid']}` ({s['issue']})" for s in summary["stale_tolerated"]]
+    if summary["incomplete"]:
+        out += ["", "### Incomplete runs (selected node ids without a result)", ""]
+        out += [f"- `{name}`: {len(ids)} missing, first `{ids[0]}`" for name, ids in summary["incomplete"].items()]
+    flaky = [(name, ids) for name, ids in summary["flaky"].items() if ids]
+    if flaky:
+        out += ["", "### Flaky (failed, then passed the single rerun; not a difference)", ""]
+        out += [f"- `{name}`: " + ", ".join(f"`{i}`" for i in ids) for name, ids in flaky]
+    out += ["", "Result: **" + ("FAIL" if failed(summary) else "PASS") + "**", ""]
+    return "\n".join(out)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    sub = parser.add_subparsers(dest="mode", required=True)
+    tgt = sub.add_parser("targets")
+    tgt.add_argument("--config", required=True)
+    env = sub.add_parser("export-env")
+    env.add_argument("--target-json", required=True)
+    fail = sub.add_parser("failed")
+    fail.add_argument("--report", required=True, help="one target's report directory")
+    cmp_ = sub.add_parser("compare")
+    cmp_.add_argument("--targets-json", required=True)
+    cmp_.add_argument("--reports", required=True)
+    cmp_.add_argument("--tolerated", required=True)
+    cmp_.add_argument("--markdown", help="write the summary table here (appended)")
+    cmp_.add_argument("--json", help="write the machine-readable summary here")
+    args = parser.parse_args(argv)
+    try:
+        if args.mode == "targets":
+            print(json.dumps(load_targets(args.config), separators=(",", ":")))
+        elif args.mode == "export-env":
+            target = json.loads(args.target_json)
+            if not isinstance(target, dict):
+                raise InputError("target must be a JSON object")
+            for key, value in sorted(validate_env(target.get("env", {}), "target").items()):
+                print(f"{key}={value}")
+        elif args.mode == "failed":
+            for name in failed_names(args.report):
+                print(name)
+        else:
+            targets = validate_targets(json.loads(args.targets_json))
+            summary = compare(targets, args.reports, args.tolerated)
+            text = render(summary)
+            print(text)
+            if args.markdown:
+                with open(args.markdown, "a", encoding="utf-8") as handle:
+                    handle.write(text + "\n")
+            if args.json:
+                with open(args.json, "w", encoding="utf-8") as handle:
+                    json.dump(summary, handle, indent=2, sort_keys=True)
+                    handle.write("\n")
+            return 1 if failed(summary) else 0
+    except (InputError, json.JSONDecodeError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,332 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contract tests for reference_parity.py: target list, rerun merge, tolerated entries and exit codes."""
+
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+SCRIPT = Path(__file__).resolve().parent / "reference_parity.py"
+REPO = Path(__file__).resolve().parents[2]
+PREFIX = "s3tests/functional/test_s3.py::"
+CLASSNAME = "s3tests.functional.test_s3"
+TARGETS = [{"name": "reference", "binary": "reference", "env": {}},
+           {"name": "candidate-legacy", "binary": "candidate", "env": {}}]
+ISSUE = "https://github.com/rustfs/rustfs/issues/1"
+
+
+def junit(states):
+    cases = []
+    for name, state in states.items():
+        body = {"pass": "", "fail": "<failure message='x'/>", "error": "<error message='x'/>",
+                "skip": "<skipped/>"}[state]
+        cases.append(f'<testcase classname="{CLASSNAME}" name="{name}">{body}</testcase>')
+    return f'<testsuites><testsuite name="pytest">{"".join(cases)}</testsuite></testsuites>'
+
+
+def run(*args):
+    proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
+    return proc.returncode, proc.stdout, proc.stderr
+
+
+class Lane:
+    """A temporary reports directory with one sub-directory per target."""
+
+    def __init__(self, case):
+        self.temp = tempfile.TemporaryDirectory()
+        case.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.reports = self.root / "reports"
+        self.tolerated = self.root / "tolerated.txt"
+        self.tolerated.write_text("# none\n")
+
+    def target(self, name, states, selected=None, rerun=None):
+        directory = self.reports / name
+        directory.mkdir(parents=True)
+        names = selected if selected is not None else list(states)
+        (directory / "selected-nodeids.txt").write_text("".join(PREFIX + n + "\n" for n in names))
+        (directory / "junit.xml").write_text(junit(states))
+        if rerun is not None:
+            (directory / "rerun-junit.xml").write_text(junit(rerun))
+        return directory
+
+    def compare(self, targets=None):
+        summary = self.root / "summary.json"
+        code, out, err = run("compare", "--targets-json", json.dumps(targets or TARGETS),
+                             "--reports", str(self.reports), "--tolerated", str(self.tolerated),
+                             "--json", str(summary))
+        data = json.loads(summary.read_text()) if summary.exists() else None
+        return code, out, err, data
+
+
+BASE = {"test_a": "pass", "test_b": "pass", "test_c": "skip"}
+
+
+class CompareTests(unittest.TestCase):
+    # --- negative cases ---
+
+    def test_pass_versus_fail_is_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"], [{"target": "candidate-legacy", "nodeid": PREFIX + "test_a",
+                                                "reference": "pass", "candidate": "fail", "tolerated": None}])
+        self.assertIn("Result: **FAIL**", out)
+
+    def test_error_counts_as_fail(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_b": "error"})
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"][0]["candidate"], "fail")
+
+    def test_reference_fail_candidate_pass_is_also_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", {**BASE, "test_a": "fail"})
+        lane.target("candidate-legacy", BASE)
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"][0]["reference"], "fail")
+
+    def test_skip_versus_pass_is_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_c": "pass"})
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"][0]["nodeid"], PREFIX + "test_c")
+
+    def test_case_absent_from_candidate_report_is_missing_and_incomplete(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {"test_a": "pass", "test_c": "skip"}, selected=list(BASE))
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"][0]["candidate"], "missing")
+        self.assertEqual(data["incomplete"], {"candidate-legacy": [PREFIX + "test_b"]})
+        self.assertIn("Incomplete runs", out)
+
+    def test_case_missing_from_both_reports_still_fails(self):
+        lane = Lane(self)
+        partial = {"test_a": "pass", "test_c": "skip"}
+        lane.target("reference", partial, selected=list(BASE))
+        lane.target("candidate-legacy", partial, selected=list(BASE))
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["differences"], [])
+        self.assertEqual(set(data["incomplete"]), {"reference", "candidate-legacy"})
+
+    def test_failure_that_also_fails_the_rerun_stays_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_a": "fail"})
+        code, _, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["flaky"]["candidate-legacy"], [])
+
+    def test_rerun_of_a_case_that_did_not_fail_is_rejected(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_b": "pass"})
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("rerun without failing first", err)
+
+    def test_missing_target_report_fails_rather_than_skips(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        code, _, err, data = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIsNone(data)
+        self.assertIn("candidate-legacy: report directory is missing", err)
+
+    def test_missing_junit_fails(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        (lane.target("candidate-legacy", BASE) / "junit.xml").unlink()
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("cannot read", err)
+
+    def test_different_selected_lists_are_rejected(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {"test_a": "pass", "test_b": "pass"})
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("selected node ids differ", err)
+
+    def test_unselected_case_in_junit_is_rejected(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_extra": "pass"}, selected=list(BASE))
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("unselected cases", err)
+
+    def test_stale_tolerated_entry_fails(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", BASE)
+        lane.tolerated.write_text(f"candidate-legacy | {PREFIX}test_a | {ISSUE} | known drift\n")
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 1)
+        self.assertEqual(data["stale_tolerated"][0]["nodeid"], PREFIX + "test_a")
+        self.assertIn("Stale tolerated entries", out)
+
+    def test_tolerated_entry_for_another_target_does_not_cover_the_difference(self):
+        lane = Lane(self)
+        targets = TARGETS + [{"name": "candidate-gateway", "binary": "candidate", "env": {}}]
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+        lane.target("candidate-gateway", {**BASE, "test_a": "fail"})
+        lane.tolerated.write_text(f"candidate-gateway | {PREFIX}test_a | {ISSUE} | gateway gap\n")
+        code, _, _, data = lane.compare(targets)
+        self.assertEqual(code, 1)
+        untolerated = [d["target"] for d in data["differences"] if not d["tolerated"]]
+        self.assertEqual(untolerated, ["candidate-legacy"])
+
+    def test_tolerated_entry_without_issue_url_is_rejected(self):
+        for line in (f"candidate-legacy | {PREFIX}test_a | rustfs#1 | reason",
+                     f"candidate-legacy | {PREFIX}test_a | {ISSUE} | ",
+                     f"candidate-legacy | {PREFIX}test_a | {ISSUE}",
+                     f"reference | {PREFIX}test_a | {ISSUE} | reason",
+                     f"candidate-legacy | test_a | {ISSUE} | reason"):
+            with self.subTest(line=line):
+                lane = Lane(self)
+                lane.target("reference", BASE)
+                lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+                lane.tolerated.write_text(line + "\n")
+                code, _, err, _ = lane.compare()
+                self.assertEqual(code, 2, err)
+
+    def test_duplicate_tolerated_entry_is_rejected(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+        entry = f"candidate-legacy | {PREFIX}test_a | {ISSUE} | reason\n"
+        lane.tolerated.write_text(entry * 2)
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("duplicate entry", err)
+
+    def test_missing_tolerated_file_fails(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", BASE)
+        lane.tolerated.unlink()
+        code, _, err, _ = lane.compare()
+        self.assertEqual(code, 2)
+        self.assertIn("tolerated-difference file is missing", err)
+
+    # --- positive cases ---
+
+    def test_identical_runs_pass(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", BASE)
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 0)
+        self.assertEqual(data["counts"]["candidate-legacy"], {"pass": 2, "fail": 0, "skip": 1, "missing": 0})
+        self.assertIn("No differences.", out)
+        self.assertIn("Result: **PASS**", out)
+
+    def test_failure_that_passes_the_rerun_is_flaky_not_a_difference(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_a": "pass"})
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 0)
+        self.assertEqual(data["flaky"]["candidate-legacy"], [PREFIX + "test_a"])
+        self.assertIn("Flaky", out)
+
+    def test_tolerated_difference_passes_and_is_labelled(self):
+        lane = Lane(self)
+        lane.target("reference", BASE)
+        lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
+        lane.tolerated.write_text(f"# comment\ncandidate-legacy | {PREFIX}test_a | {ISSUE} | known drift\n")
+        code, out, _, data = lane.compare()
+        self.assertEqual(code, 0)
+        self.assertEqual(data["differences"][0]["tolerated"], ISSUE)
+        self.assertIn(ISSUE, out)
+
+
+class TargetTests(unittest.TestCase):
+    def write(self, text):
+        temp = tempfile.TemporaryDirectory()
+        self.addCleanup(temp.cleanup)
+        path = Path(temp.name) / "targets.toml"
+        path.write_text(text)
+        return path
+
+    def test_invalid_target_lists_are_rejected(self):
+        ref = '[[targets]]\nname = "reference"\nbinary = "reference"\n'
+        cand = '[[targets]]\nname = "candidate-legacy"\nbinary = "candidate"\n'
+        for label, text in {
+            "no schema": ref + cand,
+            "only reference": "schema = 1\n" + ref,
+            "two references": "schema = 1\n" + ref + ref.replace('"reference"\nbinary', '"other"\nbinary'),
+            "no reference": "schema = 1\n" + cand + cand.replace("candidate-legacy", "candidate-two"),
+            "duplicate name": "schema = 1\n" + ref + cand + cand,
+            "unknown binary": "schema = 1\n" + ref + cand.replace('binary = "candidate"', 'binary = "main"'),
+            "bad name": "schema = 1\n" + ref + cand.replace("candidate-legacy", "Candidate Legacy"),
+            "foreign env key": "schema = 1\n" + ref + cand + 'env = { PATH = "/tmp" }\n',
+            "env injection": "schema = 1\n" + ref + cand + 'env = { RUSTFS_X = "a\\nB=c" }\n',
+            "unknown key": "schema = 1\n" + ref + cand + 'stack = "gateway"\n',
+        }.items():
+            with self.subTest(label):
+                code, _, _ = run("targets", "--config", str(self.write(text)))
+                self.assertEqual(code, 2)
+
+    def test_missing_target_file_fails(self):
+        code, _, err = run("targets", "--config", "/nonexistent/targets.toml")
+        self.assertEqual(code, 2)
+        self.assertIn("target list is missing", err)
+
+    def test_export_env_rejects_foreign_keys(self):
+        code, _, _ = run("export-env", "--target-json", json.dumps({"env": {"LD_PRELOAD": "x"}}))
+        self.assertEqual(code, 2)
+
+    def test_repository_target_list_is_valid(self):
+        code, out, err = run("targets", "--config", str(REPO / ".config/reference-parity-targets.toml"))
+        self.assertEqual(code, 0, err)
+        self.assertEqual(json.loads(out), TARGETS)
+
+    def test_export_env_prints_validated_lines(self):
+        code, out, _ = run("export-env", "--target-json", json.dumps({"env": {"RUSTFS_S3_STACK": "gateway"}}))
+        self.assertEqual((code, out), (0, "RUSTFS_S3_STACK=gateway\n"))
+
+
+class FailedTests(unittest.TestCase):
+    def test_failed_lists_only_failures_in_test_list_form(self):
+        lane = Lane(self)
+        directory = lane.target("reference", {**BASE, "test_a": "fail", "test_d": "error"})
+        code, out, _ = run("failed", "--report", str(directory))
+        self.assertEqual((code, out), (0, "test_a\ntest_d\n"))
+
+    def test_failed_rejects_a_missing_report(self):
+        code, _, err = run("failed", "--report", "/nonexistent/report")
+        self.assertEqual(code, 2)
+        self.assertIn("report directory is missing", err)
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Exercise the reference-binary pin resolver and unpacker without network access."""
+
+import hashlib
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+import zipfile
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / ".github/actions/reference-binary/reference_binary.py"
+PIN = '''schema = 1
+repository = "rustfs/rustfs"
+version = "1.0.5"
+tag = "1.0.5"
+
+[platforms.linux-x86_64-gnu]
+asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
+sha256 = "{sha}"
+executable = "rustfs"
+'''
+SHA = "a" * 64
+
+
+def run(*args):
+    proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
+    return proc.returncode, proc.stdout, proc.stderr
+
+
+class Base(unittest.TestCase):
+    def setUp(self):
+        temp = tempfile.TemporaryDirectory()
+        self.addCleanup(temp.cleanup)
+        self.root = Path(temp.name)
+
+    def pin(self, text):
+        path = self.root / "pin.toml"
+        path.write_text(text)
+        return path
+
+    def archive(self, members):
+        path = self.root / "rustfs.zip"
+        with zipfile.ZipFile(path, "w") as bundle:
+            for name, data in members.items():
+                bundle.writestr(name, data)
+        return path, hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+class ResolveTests(Base):
+    def test_missing_pin_file_fails(self):
+        code, _, err = run("resolve", "--pin", str(self.root / "absent.toml"), "--platform", "linux-x86_64-gnu")
+        self.assertEqual(code, 1)
+        self.assertIn("reference pin file is missing", err)
+
+    def test_unpinned_platform_fails(self):
+        code, _, err = run("resolve", "--pin", str(self.pin(PIN.format(sha=SHA))), "--platform", "linux-aarch64-gnu")
+        self.assertEqual(code, 1)
+        self.assertIn("no pin for platform", err)
+
+    def test_malformed_pins_fail(self):
+        good = PIN.format(sha=SHA)
+        for label, text in {
+            "short digest": good.replace(SHA, "a" * 63),
+            "uppercase digest": good.replace(SHA, "A" * 64),
+            "no digest": good.replace(f'sha256 = "{SHA}"\n', ""),
+            "path in asset": good.replace('asset = "rustfs-', 'asset = "../rustfs-'),
+            "non-zip asset": good.replace("v1.0.5.zip", "v1.0.5.deb"),
+            "asset of another version": good.replace("v1.0.5.zip", "v1.0.4.zip"),
+            "path in executable": good.replace('executable = "rustfs"', 'executable = "bin/rustfs"'),
+            "wrong schema": good.replace("schema = 1", "schema = 2"),
+            "boolean schema": good.replace("schema = 1", "schema = true"),
+            "no repository": good.replace('repository = "rustfs/rustfs"\n', ""),
+            "unknown key": good + 'url = "https://example.invalid"\n',
+            "not toml": "schema = \n",
+        }.items():
+            with self.subTest(label):
+                code, _, _ = run("resolve", "--pin", str(self.pin(text)), "--platform", "linux-x86_64-gnu")
+                self.assertEqual(code, 1)
+
+    def test_resolves_identity_as_step_outputs(self):
+        code, out, err = run("resolve", "--pin", str(self.pin(PIN.format(sha=SHA))), "--platform", "linux-x86_64-gnu")
+        self.assertEqual(code, 0, err)
+        self.assertEqual(out.splitlines(), ["repository=rustfs/rustfs", "version=1.0.5", "tag=1.0.5",
+                                            "asset=rustfs-linux-x86_64-gnu-v1.0.5.zip", f"sha256={SHA}",
+                                            "executable=rustfs"])
+
+    def test_repository_pin_is_valid(self):
+        code, out, err = run("resolve", "--pin", str(ROOT / ".config/reference-binary.toml"),
+                             "--platform", "linux-x86_64-gnu")
+        self.assertEqual(code, 0, err)
+        self.assertIn("version=1.0.5", out.splitlines())
+
+
+class UnpackTests(Base):
+    def unpack(self, archive, sha):
+        return run("unpack", "--archive", str(archive), "--sha256", sha, "--executable", "rustfs",
+                   "--destination", str(self.root / "out"))
+
+    def test_digest_mismatch_fails_and_extracts_nothing(self):
+        archive, _ = self.archive({"rustfs": b"binary"})
+        code, _, err = self.unpack(archive, SHA)
+        self.assertEqual(code, 1)
+        self.assertIn("does not match the pinned digest", err)
+        self.assertFalse((self.root / "out").exists())
+
+    def test_missing_archive_fails(self):
+        code, _, err = self.unpack(self.root / "absent.zip", SHA)
+        self.assertEqual(code, 1)
+        self.assertIn("missing or unreadable", err)
+
+    def test_archive_without_top_level_executable_fails(self):
+        archive, sha = self.archive({"bin/rustfs": b"binary"})
+        code, _, err = self.unpack(archive, sha)
+        self.assertEqual(code, 1)
+        self.assertIn("exactly one top-level", err)
+
+    def test_empty_executable_fails(self):
+        archive, sha = self.archive({"rustfs": b""})
+        code, _, _ = self.unpack(archive, sha)
+        self.assertEqual(code, 1)
+
+    def test_non_zip_archive_fails(self):
+        archive = self.root / "rustfs.zip"
+        archive.write_bytes(b"not a zip")
+        code, _, err = self.unpack(archive, hashlib.sha256(b"not a zip").hexdigest())
+        self.assertEqual(code, 1)
+        self.assertIn("not a valid zip", err)
+
+    def test_extracts_only_the_executable(self):
+        archive, sha = self.archive({"rustfs": b"binary", "../escape": b"x", "README": b"r"})
+        code, out, err = self.unpack(archive, sha)
+        self.assertEqual(code, 0, err)
+        target = self.root / "out" / "rustfs"
+        self.assertEqual(out.strip(), f"path={target}")
+        self.assertEqual(target.read_bytes(), b"binary")
+        self.assertTrue(target.stat().st_mode & 0o111)
+        self.assertEqual(sorted(p.name for p in (self.root / "out").iterdir()), ["rustfs"])
+        self.assertFalse((self.root / "escape").exists())
+
+
+if __name__ == "__main__":
+    unittest.main()