mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 20:06:37 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 86d8509826 | |||
| bc07cfd115 | |||
| bce5922aef |
@@ -0,0 +1,2 @@
|
||||
sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
|
||||
sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
|
||||
@@ -0,0 +1 @@
|
||||
sha256=9b9bc336b43b70d0e06e0adb5455bf035bb18945d85d60936eb6fe4d48e0e680
|
||||
@@ -0,0 +1,2 @@
|
||||
sha256-darwin=55534a97fbd376f64c8f6c341d319017d11ff77cad6da8629a1a7f6a874e0315
|
||||
sha256-linux=c06fb8c19aed6f388b9dc61cb8251b7a44f8561a9bf764ad2b9e635598f8dc17
|
||||
@@ -0,0 +1 @@
|
||||
sha256=655a3f3c1d042e694339d15caba7580518320322d1bac0f09450b37e6c09e2e7
|
||||
@@ -0,0 +1 @@
|
||||
sha256=ec27cde6ce6400723c4b372bfbd2ac61709c744294e4810af765e8a808d8e31d
|
||||
@@ -75,6 +75,11 @@ embedded-secrets-check: ## Check no private key material or credential literal i
|
||||
@echo "🔑 Checking embedded secret material guard..."
|
||||
./scripts/check_embedded_secrets.sh
|
||||
|
||||
.PHONY: test-wiring-check
|
||||
test-wiring-check: ## Check tests stay registered and selected by their intended runners
|
||||
@echo "🧪 Checking test wiring..."
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||
@echo "🩺 Checking log-analyzer rule anchors..."
|
||||
|
||||
@@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
@echo "✅ Fast development checks passed!"
|
||||
|
||||
@@ -35,6 +35,8 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_pinned_paired_abba_bench.sh
|
||||
./scripts/test_manual_transition_runbooks.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/s3-tests/test_report_compat.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
|
||||
|
||||
+48
-14
@@ -38,10 +38,11 @@ e2e-vault = { max-threads = 1 }
|
||||
# replacement_privileged_e2e_test when explicitly run as root on Linux). They
|
||||
# are correct in isolation but resource-heavy; serialize them under nextest's
|
||||
# process boundary (serial_test's #[serial] does not cross it) so several 4-disk
|
||||
# servers never run at once. ci-7's nightly picks these up via the e2e suite;
|
||||
# servers never run at once. The e2e-full merge/main lane picks these up;
|
||||
# they are deliberately NOT in the fast PR `e2e-smoke` filter.
|
||||
e2e-reliability = { max-threads = 1 }
|
||||
e2e-inline-boundaries = { max-threads = 1 }
|
||||
e2e-cluster-nightly = { max-threads = 1 }
|
||||
|
||||
# --- default profile (local): serialize the flaky groups, never retry --------
|
||||
[[profile.default.overrides]]
|
||||
@@ -161,7 +162,7 @@ retries = 2
|
||||
# Serialize the 4-disk reliability / degraded-read e2e tests under the ci
|
||||
# profile too (see the e2e-reliability test-group note near the top). Not a
|
||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||
# run concurrently when ci-7's nightly runs the full e2e suite.
|
||||
# run concurrently when e2e-full runs the suite.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
@@ -230,8 +231,8 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the nightly profile derives its set as "the replication module MINUS this
|
||||
# allowlist", so any new replication test lands in nightly by default (never
|
||||
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
||||
# regexes byte-identical. Count invariant: 20 here + 49 nightly = 69 total
|
||||
# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
|
||||
# regexes byte-identical. The committed profile selection digests make changes
|
||||
# visible in CI; current counts live in docs/testing/e2e-suite-inventory.md.
|
||||
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
||||
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
||||
# SSRF egress guard rejected on every PR after repl-1 (#4712). That is fixed —
|
||||
@@ -327,9 +328,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
# the STS dual-node test actually exercises its path (it skips gracefully with
|
||||
# a visible log line when awscurl is absent), and routes scheduled failures
|
||||
# through .github/actions/schedule-failure-issue (ci-8). Explicit division of
|
||||
# labor with ci-5's future e2e-full merge gate: these tests run ONLY here, not
|
||||
# double-run there. TODO(ci-7): fold this interim repl-owned lane into the ci
|
||||
# domain's consolidated scheduled e2e workflow once it exists.
|
||||
# labor with e2e-full: these tests run only in the consolidated nightly
|
||||
# workflow, not in the merge/main lane.
|
||||
[profile.e2e-repl-nightly]
|
||||
default-filter = """
|
||||
package(e2e_test)
|
||||
@@ -343,26 +343,60 @@ fail-fast = false
|
||||
# workflow as the failure-triage artifact.
|
||||
path = "junit.xml"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-nightly profile — destructive multi-process cluster fault domains
|
||||
# ---------------------------------------------------------------------------
|
||||
# These seven modules are deliberately outside e2e-full's merge budget. Each
|
||||
# starts a real multi-process or multi-disk topology and exercises node/disk
|
||||
# loss, quorum, cleanup, notification fan-in, or admin-timeout behavior. The
|
||||
# consolidated nightly workflow runs them serially to avoid resource
|
||||
# starvation; failures are never retried.
|
||||
[profile.e2e-nightly]
|
||||
default-filter = """
|
||||
package(e2e_test)
|
||||
& test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
[profile.e2e-nightly.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
[[profile.e2e-nightly.overrides]]
|
||||
filter = 'package(e2e_test)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-protocols profile — serial protocol lane
|
||||
# ---------------------------------------------------------------------------
|
||||
# The suite owns fixed ports, so the nightly workflow runs this exact profile
|
||||
# with one nextest worker.
|
||||
[profile.e2e-protocols]
|
||||
default-filter = 'package(e2e_test) & test(/^protocols::/)'
|
||||
fail-fast = false
|
||||
|
||||
[profile.e2e-protocols.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-full profile — merge-gate full single-node e2e lane (backlog#1149 ci-5)
|
||||
# ---------------------------------------------------------------------------
|
||||
# The merge gate (ci.yml `e2e-full` job: push main + merge_group +
|
||||
# workflow_dispatch). Runs the never-automated user-visible suites — KMS (40),
|
||||
# object_lock (33), multipart_auth (109), quota, checksum, encryption,
|
||||
# workflow_dispatch). Runs the user-visible KMS, object-lock, multipart-auth,
|
||||
# quota, checksum, encryption,
|
||||
# security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
|
||||
# skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
|
||||
# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
|
||||
#
|
||||
# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
|
||||
# * protocols:: — FTPS/SFTP/WebDAV, still pinned to --test-threads=1 by fixed
|
||||
# ports; they join a scheduled lane once ci-6 randomises the ports (ci-7).
|
||||
# * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
|
||||
# with one worker because the suite owns fixed ports.
|
||||
# * the 7 cluster suites that spin up a RustFSTestClusterEnvironment
|
||||
# (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster,
|
||||
# namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression,
|
||||
# object_lambda) — too heavy for the merge budget; they run in ci-7's
|
||||
# nightly 4-node lane.
|
||||
# object_lambda) — too heavy for the merge budget; they run in the
|
||||
# e2e-nightly serial cluster-fault lane.
|
||||
# * replication_extension_test — repl-1 already splits it into the PR
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (49 slow) lanes and reserves
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
|
||||
# it for those, so e2e-full does not double-run it.
|
||||
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
|
||||
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
|
||||
|
||||
@@ -46,10 +46,9 @@ lists when upstream changes.
|
||||
the PR.
|
||||
- **Weekly + manual**: `.github/workflows/e2e-s3tests.yml` runs the full
|
||||
upstream suite (`TEST_SCOPE=all`) against a Docker deployment (single node
|
||||
or a 4-node distributed cluster behind HAProxy). It fails only on
|
||||
regressions in the implemented whitelist and publishes a classification
|
||||
report (`compat-report.md`, also shown in the job summary) listing promotion
|
||||
candidates and unclassified tests.
|
||||
or a 4-node distributed cluster behind HAProxy). The canonical gate policy
|
||||
and compatibility-report behavior are documented in
|
||||
[`scripts/s3-tests/README.md`](../../scripts/s3-tests/README.md).
|
||||
|
||||
## Running Tests Locally
|
||||
|
||||
|
||||
@@ -125,6 +125,9 @@ jobs:
|
||||
- name: Check no embedded secret material
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
|
||||
@@ -160,6 +160,9 @@ jobs:
|
||||
- name: Check no embedded secret material
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
@@ -686,9 +689,9 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
# Build the e2e test graph once. The archive is reused by the security
|
||||
# count-floor check and the smoke run below, avoiding a second compile of
|
||||
# the same e2e_test target on cold runners (backlog#1645).
|
||||
# Build the e2e test graph once. The archive is reused by the smoke
|
||||
# selection guard, security exact-count check, and run below, avoiding a
|
||||
# second compile of the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Archive e2e smoke test binaries
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
@@ -696,6 +699,7 @@ jobs:
|
||||
run: |
|
||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
@@ -760,7 +764,7 @@ jobs:
|
||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||
# dispatch. protocols / the 6 cluster suites / replication / #[ignore] are
|
||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
@@ -820,6 +824,13 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
- name: Verify e2e full membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-full -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"
|
||||
|
||||
# Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
|
||||
# default-filter in .config/nextest.toml is the single wiring mechanism —
|
||||
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
||||
@@ -832,7 +843,9 @@ jobs:
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-full-junit-${{ github.run_number }}
|
||||
path: target/nextest/e2e-full/junit.xml
|
||||
path: |
|
||||
target/nextest/e2e-full/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
retention-days: 7
|
||||
|
||||
e2e-tests-rio-v2:
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Nightly full replication e2e lane (backlog#1147 repl-1, deps: ci-4).
|
||||
# Consolidated nightly e2e lane for replication, cluster faults, and protocols.
|
||||
#
|
||||
# The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the
|
||||
# FAST replication tests. This scheduled lane runs the remaining heavier
|
||||
@@ -28,15 +28,12 @@
|
||||
# add ad-hoc cargo-test steps here; change the filterset instead. The
|
||||
# authoritative membership and count come from
|
||||
# `cargo nextest list -p e2e_test --profile e2e-repl-nightly`; the PR/nightly
|
||||
# count invariant is maintained next to the filtersets in .config/nextest.toml
|
||||
# (deliberately not duplicated here).
|
||||
# selection digest is committed under .config/.
|
||||
#
|
||||
# Explicit division of labor: the nightly subset runs ONLY here, never double-run
|
||||
# in ci-5's future e2e-full merge gate. TODO(ci-7): once the ci domain's
|
||||
# consolidated scheduled e2e workflow exists, fold this interim repl-owned lane
|
||||
# into it rather than growing a second scheduled entrypoint.
|
||||
# Explicit division of labor: these subsets run only here and never double-run
|
||||
# in the e2e-full merge gate.
|
||||
|
||||
name: e2e-replication-nightly
|
||||
name: e2e-nightly
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -50,6 +47,10 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
repl-nightly:
|
||||
name: Replication e2e (nightly)
|
||||
@@ -97,9 +98,20 @@ jobs:
|
||||
# demand otherwise, but a single explicit build avoids several parallel
|
||||
# nextest test processes racing to build it at once.
|
||||
- name: Build rustfs binary
|
||||
run: cargo build -p rustfs --bins
|
||||
run: |
|
||||
cargo build -p rustfs --bins
|
||||
: > target/debug/rustfs.features
|
||||
|
||||
- name: Verify replication e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-repl-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-repl-nightly "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run replication e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
|
||||
- name: Upload nextest junit report
|
||||
@@ -107,13 +119,112 @@ jobs:
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}
|
||||
path: target/nextest/e2e-repl-nightly/junit.xml
|
||||
path: |
|
||||
target/nextest/e2e-repl-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-logs/
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
|
||||
cluster-nightly:
|
||||
name: Cluster fault e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-e2e-nightly
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build rustfs binary
|
||||
run: |
|
||||
cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
: > target/debug/rustfs.features
|
||||
|
||||
- name: Verify cluster fault e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-nightly "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run cluster fault e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
run: cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
- name: Upload cluster fault diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
${{ runner.temp }}/rustfs-e2e-nightly-logs/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
protocols-nightly:
|
||||
name: Protocol e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-e2e-protocols
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
||||
- name: Verify protocol e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-protocols -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-protocols "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run protocol e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
run: >-
|
||||
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
|
||||
|
||||
- name: Upload protocol diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-protocols/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
||||
${{ runner.temp }}/rustfs-protocol-e2e-logs/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [repl-nightly]
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
# Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
|
||||
# manual workflow_dispatch runs stay quiet so a debugging run never files a
|
||||
# spurious alert.
|
||||
|
||||
@@ -18,10 +18,9 @@
|
||||
# runs only the implemented_tests.txt whitelist. This workflow complements it:
|
||||
#
|
||||
# - Scheduled weekly full sweep (TEST_SCOPE=all): runs the ENTIRE upstream
|
||||
# suite and reports promotion candidates (tests that newly pass) and
|
||||
# unclassified tests. The job fails only on regressions in the implemented
|
||||
# whitelist or on infrastructure errors — expected failures from
|
||||
# not-yet-implemented features do not turn the run red.
|
||||
# suite and reports promotion candidates. Regressions, unclassified tests,
|
||||
# incomplete execution, and infrastructure errors fail the job; classified
|
||||
# failures for not-yet-implemented features remain informational.
|
||||
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
|
||||
#
|
||||
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
|
||||
@@ -45,13 +44,6 @@
|
||||
# The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
|
||||
# via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.
|
||||
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||
# reading this file, which has already misled at least one audit — hence this
|
||||
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||
#
|
||||
name: e2e-s3tests
|
||||
|
||||
on:
|
||||
@@ -81,6 +73,19 @@ on:
|
||||
description: "Stop after N failures. '0' to run everything."
|
||||
required: false
|
||||
default: "0"
|
||||
shard-count:
|
||||
description: "Exact-node-ID shard count for a targeted manual run"
|
||||
required: false
|
||||
default: "1"
|
||||
type: choice
|
||||
options:
|
||||
- "1"
|
||||
- "2"
|
||||
- "4"
|
||||
shard-index:
|
||||
description: "Zero-based shard index for a targeted manual run"
|
||||
required: false
|
||||
default: "0"
|
||||
markexpr:
|
||||
description: "Optional pytest -m expression"
|
||||
required: false
|
||||
@@ -111,6 +116,8 @@ env:
|
||||
XDIST: ${{ github.event.inputs.xdist || '4' }}
|
||||
MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
|
||||
MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
|
||||
S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }}
|
||||
TEST_TIMEOUT: "300"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
|
||||
@@ -127,19 +134,22 @@ defaults:
|
||||
|
||||
jobs:
|
||||
s3tests:
|
||||
name: s3tests (${{ matrix.test-mode }}, shard ${{ matrix.shard-index }})
|
||||
# GitHub-hosted: reliably provides Docker + docker compose + python3/pip.
|
||||
# See the header note (ci-1) for why the self-hosted sm-standard-4 label
|
||||
# was abandoned. TODO(ci-8): scheduled-failure alerting (auto-open issue)
|
||||
# is added by the ci-8 composite action; do not implement it here.
|
||||
# was abandoned. Scheduled failures are handled by alert-on-failure below.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 2
|
||||
matrix:
|
||||
# Scheduled sweeps cover both topologies; manual runs use the input.
|
||||
test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
|
||||
shard-index: ${{ github.event_name == 'schedule' && fromJSON('[0, 1, 2, 3]') || fromJSON(format('[{0}]', github.event.inputs.shard-index || '0')) }}
|
||||
env:
|
||||
TEST_MODE: ${{ matrix.test-mode }}
|
||||
S3_SHARD_INDEX: ${{ matrix.shard-index }}
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
@@ -181,6 +191,7 @@ jobs:
|
||||
- name: Start single RustFS
|
||||
if: env.TEST_MODE == 'single'
|
||||
run: |
|
||||
SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
|
||||
docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
|
||||
docker rm -f rustfs-single >/dev/null 2>&1 || true
|
||||
# The four disks share one physical device on the runner (a single
|
||||
@@ -193,6 +204,7 @@ jobs:
|
||||
-e RUSTFS_ADDRESS=0.0.0.0:9000 \
|
||||
-e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
|
||||
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
||||
-e RUSTFS_SSE_S3_MASTER_KEY="${SSE_KEY}" \
|
||||
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-v /tmp/rustfs-single:/data \
|
||||
@@ -201,6 +213,7 @@ jobs:
|
||||
- name: Start 4-node distributed cluster
|
||||
if: env.TEST_MODE == 'multi'
|
||||
run: |
|
||||
SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
|
||||
# A real distributed deployment: every node lists all endpoints in
|
||||
# RUSTFS_VOLUMES so data is erasure-coded ACROSS nodes. Do not use
|
||||
# node-local volume paths here — that would create four independent
|
||||
@@ -213,6 +226,7 @@ jobs:
|
||||
RUSTFS_ADDRESS: "0.0.0.0:9000"
|
||||
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
RUSTFS_SSE_S3_MASTER_KEY: "${SSE_KEY}"
|
||||
RUSTFS_VOLUMES: "http://rustfs{1...4}:9000/data/rustfs{0...3}"
|
||||
# Each node's four disks share one physical device inside its
|
||||
# container, so bypass the local physical-disk-independence guard
|
||||
@@ -294,7 +308,6 @@ jobs:
|
||||
|
||||
- name: Run ceph s3-tests
|
||||
run: |
|
||||
set +e
|
||||
DEPLOY_MODE=existing \
|
||||
TEST_MODE="${TEST_MODE}" \
|
||||
TEST_SCOPE="${TEST_SCOPE}" \
|
||||
@@ -302,26 +315,6 @@ jobs:
|
||||
MAXFAIL="${MAXFAIL}" \
|
||||
MARKEXPR="${MARKEXPR}" \
|
||||
./scripts/s3-tests/run.sh
|
||||
RC=$?
|
||||
set -e
|
||||
|
||||
if [ "${TEST_SCOPE}" = "implemented" ]; then
|
||||
# Whitelist run: every failure is a regression.
|
||||
exit "${RC}"
|
||||
fi
|
||||
|
||||
# Full sweep: failures outside the implemented whitelist are
|
||||
# inventory (promotion candidates / unimplemented features), not a
|
||||
# gate. Fail only on whitelist regressions or infrastructure errors.
|
||||
JUNIT="artifacts/s3tests-${TEST_MODE}/junit.xml"
|
||||
if [ ! -f "${JUNIT}" ]; then
|
||||
echo "No junit.xml produced — infrastructure failure (exit ${RC})" >&2
|
||||
exit "${RC}"
|
||||
fi
|
||||
python3 scripts/s3-tests/report_compat.py \
|
||||
--junit "${JUNIT}" \
|
||||
--lists-dir scripts/s3-tests \
|
||||
--fail-on-regression
|
||||
|
||||
- name: Publish compatibility report
|
||||
if: always()
|
||||
@@ -346,7 +339,7 @@ jobs:
|
||||
if: always() && env.ACT != 'true'
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: s3tests-${{ env.TEST_MODE }}
|
||||
name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
|
||||
path: artifacts/**
|
||||
|
||||
alert-on-failure:
|
||||
|
||||
+11
-24
@@ -12,27 +12,22 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||
# reading this file, which has already misled at least one audit — hence this
|
||||
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||
#
|
||||
name: Fuzz
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
# PR trigger is intentionally narrow: only changes to the fuzz harness
|
||||
# itself gate a PR. Broad crate paths (ecstore/filemeta/utils/policy/…)
|
||||
# are covered by the nightly `schedule` run below, which fuzzes against
|
||||
# whatever landed on main. Widening these paths previously queued a
|
||||
# ~45min fuzz-build on nearly every PR and is why this workflow was
|
||||
# disabled; do not re-add crate paths here.
|
||||
# Run when the harness or any directly fuzzed production crate changes.
|
||||
paths:
|
||||
- "fuzz/**"
|
||||
- "scripts/fuzz/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/policy/**"
|
||||
- "crates/security-governance/**"
|
||||
- "crates/utils/**"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/workflows/fuzz.yml"
|
||||
schedule:
|
||||
- cron: "0 2 * * *"
|
||||
@@ -81,7 +76,7 @@ jobs:
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 45
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -121,12 +116,7 @@ jobs:
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: fuzz-prebuilt-binaries-${{ github.run_number }}
|
||||
path: |
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/archive_extract
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/bucket_validation
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/local_metadata
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/path_containment
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/policy_ingress
|
||||
path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
compression-level: 0
|
||||
@@ -192,10 +182,7 @@ jobs:
|
||||
nightly-fuzz-corpus:
|
||||
name: "Nightly / ${{ matrix.target }}"
|
||||
needs: fuzz-build
|
||||
# TODO(ci-8): when the schedule-failure-issue composite action lands,
|
||||
# add a step here (or a dependent job) that opens/updates a GitHub issue
|
||||
# on nightly failure. ci-8 is the single alerting mechanism for all
|
||||
# scheduled workflows; do not self-roll alerting in this workflow.
|
||||
# Scheduled failures are handled by alert-on-failure below.
|
||||
if: >
|
||||
github.event_name == 'schedule' ||
|
||||
(github.event_name == 'workflow_dispatch' &&
|
||||
|
||||
+26
-21
@@ -48,16 +48,14 @@ cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
|
||||
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
|
||||
|
||||
# Protocols suite — fixed ports, MUST be single-threaded, gated by build features
|
||||
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
|
||||
cargo test -p e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
|
||||
```
|
||||
|
||||
The protocols suite has its own contract (fixed bind ports 9022–9301,
|
||||
`--test-threads=1`, feature-gated scheduling) documented in
|
||||
single-worker execution, feature-gated scheduling) documented in
|
||||
[`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
|
||||
selects which features the spawned binary is built with; leave it unset to run
|
||||
every protocol entry.
|
||||
every protocol entry. Use the exact profile command under
|
||||
[Troubleshooting](#troubleshooting) for CI-equivalent execution.
|
||||
|
||||
### `#[ignore]` semantics
|
||||
|
||||
@@ -159,27 +157,26 @@ construction (random port + isolated temp dir) and need no serialization.
|
||||
## CI map
|
||||
|
||||
`e2e_test` is **excluded** from the main `cargo nextest run --profile ci --all`
|
||||
pass ([`.github/workflows/ci.yml`](../../.github/workflows/ci.yml) line 158,
|
||||
`--exclude e2e_test`) — the whole crate is too slow to gate every PR. Subsets
|
||||
join CI through the nextest profile system only (never as ad-hoc jobs):
|
||||
pass (`--exclude e2e_test`) — the whole crate is too slow to gate every PR.
|
||||
Subsets join CI through nextest profiles; the fixed-port protocol suite uses
|
||||
the same profile for membership and execution with one nightly worker.
|
||||
|
||||
| Suite | Runs where | Status |
|
||||
| --- | --- | --- |
|
||||
| Smoke subset (`e2e-smoke` profile) | `e2e-tests` job, every PR | **Active** (backlog#1149 ci-4) |
|
||||
| Full single-node suite (`e2e-full` profile) | `e2e-full` job, merge queue + main | **Active** (backlog#1149 ci-5) |
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | — | Not in CI yet (backlog#1149 ci-5) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | — | Not in CI yet (backlog#1149 ci-7) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + dual-node) | `e2e-repl-nightly` profile, scheduled workflow | **Active** (backlog#1147 repl-1) |
|
||||
| `reliant/*` (pre-started server) | — | Manual only |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
| `reliant/*` | 19 tests in PR smoke; remaining default tests in `e2e-full` | **Active** except `#[ignore]` |
|
||||
|
||||
Links: [`ci.yml`](../../.github/workflows/ci.yml) `e2e-tests` (line 347),
|
||||
`test-ilm-integration-serial` (line 196). The `e2e-smoke` `default-filter` in
|
||||
[`.config/nextest.toml`](../../.config/nextest.toml) is the **single wiring
|
||||
mechanism** — extend that filter (or add a sibling profile) to admit more
|
||||
tests; do not add e2e jobs to `ci.yml`. repl-1 / ilm-3 are landing in parallel
|
||||
and may add lanes; keep the table above easy to extend.
|
||||
The profile filters in [`.config/nextest.toml`](../../.config/nextest.toml) are
|
||||
the wiring source of truth. Committed test-ID digests under
|
||||
`.config/e2e-*-selection.txt` make every membership change explicit.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
@@ -188,9 +185,15 @@ and may add lanes; keep the table above easy to extend.
|
||||
```bash
|
||||
# Smoke (e2e-tests job) — includes the 20 fast replication tests
|
||||
cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
# Replication nightly lane (16 slow + dual-node tests; install awscurl for the
|
||||
# STS dual-node test, else it skips gracefully)
|
||||
# Full single-node merge/main lane
|
||||
cargo nextest run --profile e2e-full -p e2e_test
|
||||
# Cluster fault nightly lane
|
||||
cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
# Replication nightly lane; install awscurl so STS paths do not skip
|
||||
cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
# Fixed-port protocol nightly lane
|
||||
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
|
||||
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
|
||||
# ILM serial lane
|
||||
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
|
||||
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
|
||||
@@ -273,4 +276,6 @@ current subset is.
|
||||
`docs/testing/e2e-suite-inventory.md` records the per-module test counts as
|
||||
listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
|
||||
moving e2e tests so acceptance numbers in the test-strategy issues
|
||||
(backlog#1147–#1155) stay auditable.
|
||||
(backlog#1147–#1155) stay auditable. When a profile membership change is
|
||||
intentional, review its JSON listing before updating the matching
|
||||
`.config/e2e-*-selection.txt` test-ID digest.
|
||||
|
||||
@@ -11,10 +11,17 @@ test process directly.
|
||||
|
||||
## Running Tests
|
||||
|
||||
Use the canonical CI-equivalent protocol command in the parent
|
||||
[`e2e_test` README](../../README.md#troubleshooting).
|
||||
|
||||
For targeted debugging of the core suite only:
|
||||
|
||||
```bash
|
||||
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
|
||||
```
|
||||
|
||||
This targeted command does not cover the full `e2e-protocols` profile.
|
||||
|
||||
`RUSTFS_BUILD_FEATURES` controls which features the test rustfs binary is
|
||||
built with. When this variable is set, the protocol test runner schedules
|
||||
only entries whose protocol is present in the requested feature list. Leave
|
||||
@@ -133,4 +140,3 @@ property without consulting any external doc.
|
||||
Bind ports 9023 (SFTP) and 9100 (S3). Spawns rustfs with
|
||||
`RUSTFS_SFTP_IDLE_TIMEOUT=5`, sleeps 10 s past the timeout, then issues an
|
||||
SFTP request and asserts the server has closed the session.
|
||||
|
||||
|
||||
@@ -922,14 +922,10 @@ mod prepared_get_object_metadata_tests {
|
||||
.expect("test should find an object whose initial fanout covers both data shards")
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)"
|
||||
)]
|
||||
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
|
||||
fn bounded_initial_parity_disk_index(bucket: &str, object: &str) -> usize {
|
||||
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
||||
.get(3)
|
||||
.expect("4-disk test geometry should leave one bounded spare disk")
|
||||
.get(2)
|
||||
.expect("4-disk test geometry should schedule one parity disk initially")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1087,7 +1083,7 @@ mod prepared_get_object_metadata_tests {
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
||||
],
|
||||
async {
|
||||
let slow_parity_disk = bounded_spare_disk_index(bucket, &object);
|
||||
let slow_parity_disk = bounded_initial_parity_disk_index(bucket, &object);
|
||||
let barrier =
|
||||
rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
|
||||
@@ -297,10 +297,16 @@ impl ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
|
||||
use crate::disk::{DiskOption, format::FormatV3, new_disk};
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk};
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
|
||||
use crate::store::init_format::{load_format_erasure, save_format_file};
|
||||
use crate::store::init_local_disks_with_instance_ctx;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
|
||||
let format = FormatV3::new(1, 1);
|
||||
@@ -347,6 +353,51 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
|
||||
let mut pool_endpoints = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("multi-pool heal test disk should be created");
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
|
||||
.expect("test endpoint should parse");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
pool_endpoints.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("heal-owner-pool-{pool_index}"),
|
||||
platform: "test".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(pool_endpoints);
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
.await
|
||||
.expect("multi-pool local disks should initialize");
|
||||
let shutdown = CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx(
|
||||
"127.0.0.1:0".parse().expect("test address should parse"),
|
||||
endpoint_pools,
|
||||
shutdown.clone(),
|
||||
instance_ctx,
|
||||
)
|
||||
.await
|
||||
.expect("multi-pool test store should initialize");
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(temp_dir, store, shutdown)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_pool_scope_selects_only_requested_pool() {
|
||||
let store = minimal_heal_store().await;
|
||||
@@ -506,6 +557,204 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn unscoped_heal_object_suspended_owner_semantics() {
|
||||
let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
|
||||
let active_object = "active-owner";
|
||||
let suspended_only_object = "suspended-only";
|
||||
let duplicate_object = "duplicate-owner";
|
||||
let marker_object = "marker-owner";
|
||||
let quorum_object = "quorum-owner";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created in all pools");
|
||||
|
||||
let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("active owner object should be written");
|
||||
let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
|
||||
missing_active_disk
|
||||
.delete(
|
||||
&bucket,
|
||||
active_object,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("active owner shard should be removed for repair");
|
||||
assert!(
|
||||
missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
|
||||
"the active owner fixture must start with one missing metadata copy"
|
||||
);
|
||||
|
||||
let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
|
||||
store.pools[1]
|
||||
.put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("suspended owner object should be written");
|
||||
for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
|
||||
let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
|
||||
store.pools[pool_index]
|
||||
.put_object(
|
||||
&bucket,
|
||||
duplicate_object,
|
||||
&mut duplicate_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("duplicate owner object should be written");
|
||||
}
|
||||
let history_version = Uuid::new_v4();
|
||||
let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
marker_object,
|
||||
&mut history_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(history_version.to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned marker history should be written");
|
||||
store.pools[0]
|
||||
.delete_object(
|
||||
&bucket,
|
||||
marker_object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete marker should be written");
|
||||
let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("quorum boundary object should be written");
|
||||
{
|
||||
let mut pool_meta = store.pool_meta.write().await;
|
||||
let mut next = PoolMeta::new(&store.pools, &pool_meta);
|
||||
next.pools[1].decommission = Some(PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
});
|
||||
*pool_meta = next;
|
||||
}
|
||||
|
||||
let (_, duplicate_owner) = store
|
||||
.get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("duplicate owner should resolve");
|
||||
assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
|
||||
let (_, active_duplicate_owner) = store
|
||||
.get_latest_object_info_with_idx(
|
||||
&bucket,
|
||||
duplicate_object,
|
||||
&ObjectOptions {
|
||||
skip_decommissioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("active duplicate owner should resolve");
|
||||
assert_eq!(
|
||||
active_duplicate_owner, 0,
|
||||
"suspended duplicate must be excluded from active owner selection"
|
||||
);
|
||||
let (marker_info, marker_owner) = store
|
||||
.get_latest_object_info_with_idx(
|
||||
&bucket,
|
||||
marker_object,
|
||||
&ObjectOptions {
|
||||
skip_decommissioned: true,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("latest delete marker should resolve");
|
||||
assert_eq!(marker_owner, 0);
|
||||
assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
|
||||
|
||||
let (active_result, active_err) = store
|
||||
.handle_heal_object(&bucket, active_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("unscoped active-owner heal should complete");
|
||||
assert_eq!(active_result.object, active_object);
|
||||
assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
|
||||
assert!(
|
||||
missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
|
||||
"active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
|
||||
);
|
||||
assert!(
|
||||
store.pools[1]
|
||||
.get_object_info(&bucket, active_object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"the suspended pool must not be written for an active-owner object"
|
||||
);
|
||||
|
||||
let (suspended_result, suspended_err) = store
|
||||
.handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("unscoped suspended-only heal should return a terminal result");
|
||||
assert!(suspended_result.object.is_empty());
|
||||
assert!(matches!(suspended_err, Some(Error::FileNotFound)));
|
||||
assert!(
|
||||
store.pools[1]
|
||||
.get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok(),
|
||||
"suspended-only data must remain untouched when unscoped heal reports absent"
|
||||
);
|
||||
|
||||
let (_, explicit_err) = store
|
||||
.handle_heal_object(
|
||||
&bucket,
|
||||
suspended_only_object,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("explicit suspended-owner heal should return a mapped error");
|
||||
assert!(matches!(explicit_err, Some(Error::SlowDown)));
|
||||
|
||||
let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
let surviving_quorum_disk = original_quorum_disks[3].clone();
|
||||
*store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
|
||||
let (_, quorum_err) = store
|
||||
.handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("quorum boundary heal should return a mapped result");
|
||||
*store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
|
||||
assert!(
|
||||
matches!(quorum_err, Some(Error::ErasureReadQuorum)),
|
||||
"quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handle_heal_format_continues_after_a_pool_error() {
|
||||
let canonical_format = FormatV3::new(1, 3);
|
||||
|
||||
@@ -373,11 +373,6 @@ impl ErasureSetHealer {
|
||||
set_disk_id: &str,
|
||||
buckets: &[String],
|
||||
) -> Result<(ResumeManager, CheckpointManager)> {
|
||||
if self.replacement_task_id.is_none() && CheckpointManager::is_blocked(&self.disk, task_id).await {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Resume task {task_id} has a blocked checkpoint"),
|
||||
});
|
||||
}
|
||||
// check if resume state exists
|
||||
let has_resume_state = if self.replacement_task_id.is_some() {
|
||||
ResumeManager::has_replacement_intent(&self.disk, task_id).await
|
||||
|
||||
@@ -51,7 +51,6 @@ const RESUME_STATE_FILE: &str = "ahm_resume_state.json";
|
||||
const REPLACEMENT_INTENT_FILE: &str = "ahm_replacement_intent.json";
|
||||
const RESUME_PROGRESS_FILE: &str = "ahm_progress.json";
|
||||
pub(super) const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json";
|
||||
pub(super) const RESUME_CHECKPOINT_BLOCKED_FILE: &str = "ahm_checkpoint.blocked";
|
||||
const REPLACEMENT_COMPLETION_PROOF_FILE: &str = "ahm_replacement_completion_proof.json";
|
||||
const REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
|
||||
const REPLACEMENT_INTENT_SEAL_FILE: &str = "ahm_replacement_intent_seal";
|
||||
|
||||
@@ -18,14 +18,13 @@ use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::{Mutex as AsyncMutex, RwLock};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use super::super::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes};
|
||||
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt, RUSTFS_META_BUCKET};
|
||||
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
|
||||
use super::{
|
||||
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_BLOCKED_FILE, RESUME_CHECKPOINT_FILE,
|
||||
delete_resume_file, path_to_str, validate_resume_task_id,
|
||||
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_FILE, delete_resume_file, path_to_str,
|
||||
validate_resume_task_id,
|
||||
};
|
||||
|
||||
const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state";
|
||||
@@ -117,108 +116,17 @@ pub struct CheckpointManager {
|
||||
disk: DiskStore,
|
||||
checkpoint: Arc<RwLock<ResumeCheckpoint>>,
|
||||
throttle: Mutex<PersistThrottle>,
|
||||
save_lock: AsyncMutex<()>,
|
||||
last_saved: Mutex<Option<EcstoreDiskBytes>>,
|
||||
}
|
||||
|
||||
impl CheckpointManager {
|
||||
fn blocked_path(task_id: &str) -> std::path::PathBuf {
|
||||
Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}"))
|
||||
}
|
||||
|
||||
/// Return whether a checkpoint was permanently isolated after a malformed
|
||||
/// or unsupported snapshot was observed.
|
||||
pub(crate) async fn is_blocked(disk: &DiskStore, task_id: &str) -> bool {
|
||||
if validate_resume_task_id(task_id).is_err() {
|
||||
return false;
|
||||
}
|
||||
let blocked_path = Self::blocked_path(task_id);
|
||||
let Ok(path) = path_to_str(&blocked_path) else {
|
||||
return false;
|
||||
};
|
||||
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
|
||||
Ok(_) => true,
|
||||
Err(crate::heal::DiskError::FileNotFound) => false,
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate the checkpoint while enumerating resumable state. This reads
|
||||
/// the checkpoint once and also isolates malformed or unsupported data.
|
||||
pub(crate) async fn is_resumable(disk: &DiskStore, task_id: &str) -> bool {
|
||||
if validate_resume_task_id(task_id).is_err() || Self::is_blocked(disk, task_id).await {
|
||||
return false;
|
||||
}
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
let Ok(path) = path_to_str(&file_path) else {
|
||||
return false;
|
||||
};
|
||||
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
|
||||
Ok(bytes) if bytes.is_empty() => true,
|
||||
Ok(bytes) => Self::load_from_data(disk.clone(), task_id, bytes.to_vec()).await.is_ok(),
|
||||
Err(crate::heal::DiskError::FileNotFound) => true,
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn block_invalid_snapshot(disk: &DiskStore, task_id: &str) {
|
||||
// This marker is intentionally version-agnostic: an unsupported reader
|
||||
// must stop selector retries until an operator cleans up the snapshot.
|
||||
let blocked_path = Self::blocked_path(task_id);
|
||||
let Ok(path) = path_to_str(&blocked_path) else {
|
||||
return;
|
||||
};
|
||||
let result = EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
Some(EcstoreDiskBytes::from_static(b"blocked")),
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(EcstoreConditionalFileUpdate::Updated | EcstoreConditionalFileUpdate::Mismatch) => {}
|
||||
Ok(EcstoreConditionalFileUpdate::Missing) => warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
state = "blocked_marker_write_failed",
|
||||
error = "marker target disappeared",
|
||||
"Heal checkpoint could not persist its blocked marker"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
state = "blocked_marker_write_failed",
|
||||
error = %error,
|
||||
"Heal checkpoint could not persist its blocked marker"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// create new checkpoint manager
|
||||
pub async fn new(disk: DiskStore, task_id: String) -> Result<Self> {
|
||||
validate_resume_task_id(&task_id)?;
|
||||
let checkpoint_volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}");
|
||||
if let Err(error) = EcstoreDiskAPI::make_volume(disk.as_ref(), &checkpoint_volume).await
|
||||
&& error != crate::heal::DiskError::VolumeExists
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to create checkpoint volume: {error}"),
|
||||
});
|
||||
}
|
||||
let checkpoint = ResumeCheckpoint::new(task_id);
|
||||
let manager = Self {
|
||||
disk,
|
||||
checkpoint: Arc::new(RwLock::new(checkpoint)),
|
||||
throttle: Mutex::new(PersistThrottle::new()),
|
||||
save_lock: AsyncMutex::new(()),
|
||||
last_saved: Mutex::new(None),
|
||||
};
|
||||
|
||||
// save initial checkpoint
|
||||
@@ -232,7 +140,6 @@ impl CheckpointManager {
|
||||
error = %e,
|
||||
"Heal checkpoint persistence failed"
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
Ok(manager)
|
||||
}
|
||||
@@ -241,22 +148,11 @@ impl CheckpointManager {
|
||||
pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result<Self> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?;
|
||||
Self::load_from_data(disk, task_id, checkpoint_data).await
|
||||
}
|
||||
|
||||
async fn load_from_data(disk: DiskStore, task_id: &str, checkpoint_data: Vec<u8>) -> Result<Self> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let mut checkpoint: ResumeCheckpoint = match serde_json::from_slice(&checkpoint_data) {
|
||||
Ok(checkpoint) => checkpoint,
|
||||
Err(error) => {
|
||||
Self::block_invalid_snapshot(&disk, task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize checkpoint: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
let mut checkpoint: ResumeCheckpoint =
|
||||
serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize checkpoint: {e}"),
|
||||
})?;
|
||||
if checkpoint.task_id != task_id {
|
||||
Self::block_invalid_snapshot(&disk, task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Resume checkpoint task id does not match filename".to_string(),
|
||||
});
|
||||
@@ -267,7 +163,6 @@ impl CheckpointManager {
|
||||
// identities. Discard the stale sets and position, then stamp the
|
||||
// current schema so the scan restarts cleanly.
|
||||
if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA {
|
||||
Self::block_invalid_snapshot(&disk, task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!(
|
||||
"Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
|
||||
@@ -299,8 +194,6 @@ impl CheckpointManager {
|
||||
disk,
|
||||
checkpoint: Arc::new(RwLock::new(checkpoint)),
|
||||
throttle: Mutex::new(PersistThrottle::new()),
|
||||
save_lock: AsyncMutex::new(()),
|
||||
last_saved: Mutex::new(Some(EcstoreDiskBytes::from(checkpoint_data))),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -311,7 +204,7 @@ impl CheckpointManager {
|
||||
}
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
match path_to_str(&file_path) {
|
||||
Ok(path_str) => match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(data) => !data.is_empty(),
|
||||
Err(_) => false,
|
||||
},
|
||||
@@ -399,7 +292,6 @@ impl CheckpointManager {
|
||||
|
||||
let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
delete_resume_file(&self.disk, &checkpoint_file).await?;
|
||||
delete_resume_file(&self.disk, &Self::blocked_path(&task_id)).await?;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
@@ -415,126 +307,21 @@ impl CheckpointManager {
|
||||
|
||||
/// save checkpoint to disk
|
||||
async fn save_checkpoint(&self) -> Result<()> {
|
||||
// Serialize saves and take the snapshot only after acquiring the lock:
|
||||
// a slower writer must not publish a snapshot taken before a newer one.
|
||||
let _save_guard = self.save_lock.lock().await;
|
||||
let checkpoint = self.checkpoint.read().await.clone();
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
validate_resume_task_id(&checkpoint.task_id)?;
|
||||
let checkpoint_data =
|
||||
EcstoreDiskBytes::from(serde_json::to_vec(&checkpoint).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize checkpoint: {e}"),
|
||||
})?);
|
||||
let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize checkpoint: {e}"),
|
||||
})?;
|
||||
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE));
|
||||
|
||||
let path_str = path_to_str(&file_path)?;
|
||||
let last_saved = self
|
||||
.last_saved
|
||||
.lock()
|
||||
.map_err(|_| Error::TaskExecutionFailed {
|
||||
message: "Checkpoint save state lock is poisoned; refusing to save".to_string(),
|
||||
})?
|
||||
.clone();
|
||||
let update = EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path_str,
|
||||
last_saved.clone(),
|
||||
Some(checkpoint_data.clone()),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save checkpoint: {e}"),
|
||||
})?;
|
||||
|
||||
let expected = match update {
|
||||
EcstoreConditionalFileUpdate::Updated => None,
|
||||
EcstoreConditionalFileUpdate::Missing => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
|
||||
});
|
||||
}
|
||||
EcstoreConditionalFileUpdate::Mismatch => {
|
||||
// A healthy manager normally completes the CAS above without
|
||||
// another read or JSON parse. Inspect only after a mismatch so
|
||||
// corruption and future schemas cannot be overwritten blindly.
|
||||
let existing = match HealDiskExt::read_all(self.disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(existing) => existing,
|
||||
Err(crate::heal::DiskError::FileNotFound) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
|
||||
});
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to inspect checkpoint after CAS mismatch: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
if existing.is_empty() && last_saved.is_none() {
|
||||
Some(existing)
|
||||
} else {
|
||||
let current: ResumeCheckpoint = match serde_json::from_slice(&existing) {
|
||||
Ok(current) => current,
|
||||
Err(error) => {
|
||||
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Existing checkpoint is corrupt: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
if current.task_id != checkpoint.task_id {
|
||||
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Existing checkpoint task id does not match filename".to_string(),
|
||||
});
|
||||
}
|
||||
if current.schema_version > CURRENT_CHECKPOINT_SCHEMA {
|
||||
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!(
|
||||
"Existing checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
|
||||
current.schema_version
|
||||
),
|
||||
});
|
||||
}
|
||||
if last_saved.as_ref().is_none_or(|saved| saved.as_ref() != existing.as_ref()) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Checkpoint changed since this manager loaded it; refusing to overwrite newer progress"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
Some(existing)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(expected) = expected {
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path_str,
|
||||
Some(expected),
|
||||
Some(checkpoint_data.clone()),
|
||||
)
|
||||
self.disk
|
||||
.write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into())
|
||||
.await
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save checkpoint after CAS mismatch: {e}"),
|
||||
})? {
|
||||
EcstoreConditionalFileUpdate::Updated => {}
|
||||
EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Checkpoint changed while saving; refusing to overwrite newer progress".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut last_saved = self.last_saved.lock().map_err(|_| Error::TaskExecutionFailed {
|
||||
message: "Checkpoint save state lock is poisoned after save".to_string(),
|
||||
})?;
|
||||
*last_saved = Some(checkpoint_data);
|
||||
message: format!("Failed to save checkpoint: {e}"),
|
||||
})?;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
@@ -554,7 +341,7 @@ impl CheckpointManager {
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
|
||||
let path_str = path_to_str(&file_path)?;
|
||||
HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str)
|
||||
disk.read_all(RUSTFS_META_BUCKET, path_str)
|
||||
.await
|
||||
.map(|bytes| bytes.to_vec())
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
|
||||
@@ -1675,264 +1675,6 @@ async fn future_resume_and_checkpoint_schemas_are_rejected() {
|
||||
temp_dir.close().expect("remove schema test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_save_does_not_replace_a_non_empty_truncated_snapshot() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
let truncated = b"{\"schema_version\":5,\"task_id\":";
|
||||
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, truncated.as_slice().into())
|
||||
.await
|
||||
.expect("write truncated checkpoint fixture");
|
||||
|
||||
let error = manager
|
||||
.update_position(2, 7)
|
||||
.await
|
||||
.expect_err("a truncated checkpoint must fail closed during save");
|
||||
assert!(error.to_string().contains("Existing checkpoint is corrupt"));
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read truncated checkpoint fixture"),
|
||||
truncated.as_slice()
|
||||
);
|
||||
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove checkpoint save test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_save_does_not_replace_a_future_schema_snapshot() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
let mut future = ResumeCheckpoint::new(task_id.clone());
|
||||
future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
|
||||
let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, future_bytes.clone().into())
|
||||
.await
|
||||
.expect("write future checkpoint fixture");
|
||||
|
||||
let error = manager
|
||||
.update_position(2, 7)
|
||||
.await
|
||||
.expect_err("a future schema must fail closed during save");
|
||||
assert!(error.to_string().contains("Existing checkpoint schema"));
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read future checkpoint fixture"),
|
||||
future_bytes
|
||||
);
|
||||
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove future schema test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_checkpoint_manager_rebuilds_an_empty_snapshot() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, EcstoreDiskBytes::new())
|
||||
.await
|
||||
.expect("write empty checkpoint fixture");
|
||||
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("a new manager must rebuild an empty checkpoint");
|
||||
manager
|
||||
.update_position(3, 11)
|
||||
.await
|
||||
.expect("rebuilt checkpoint must remain writable");
|
||||
assert!(CheckpointManager::has_checkpoint(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove empty checkpoint test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deleted_checkpoint_is_not_recreated_by_an_old_manager() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
manager.cleanup().await.expect("delete checkpoint fixture");
|
||||
|
||||
let error = manager
|
||||
.update_position(1, 2)
|
||||
.await
|
||||
.expect_err("an old manager must not resurrect a deleted checkpoint");
|
||||
assert!(error.to_string().contains("removed after this manager saved it"));
|
||||
assert!(!CheckpointManager::has_checkpoint(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove deleted checkpoint test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_empty_blocked_marker_still_blocks_resume_selection() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &blocked_path, EcstoreDiskBytes::new())
|
||||
.await
|
||||
.expect("write empty blocked marker fixture");
|
||||
|
||||
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
assert!(!CheckpointManager::is_resumable(&disk, &task_id).await);
|
||||
// Recovery requires replacing/cleaning the snapshot, then removing the
|
||||
// marker; ordinary selector retries are intentionally not an unlock path.
|
||||
manager.cleanup().await.expect("clean blocked checkpoint");
|
||||
assert!(!CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove empty blocked marker test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resumable_selector_skips_healthy_tasks_with_blocked_markers() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let tasks = [
|
||||
(ResumeUtils::generate_task_id(), EcstoreDiskBytes::new()),
|
||||
(ResumeUtils::generate_task_id(), EcstoreDiskBytes::from_static(b"blocked")),
|
||||
];
|
||||
for (task_id, marker) in &tasks {
|
||||
ResumeManager::new(
|
||||
disk.clone(),
|
||||
task_id.clone(),
|
||||
"erasure_set".to_string(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["bucket".to_string()],
|
||||
)
|
||||
.await
|
||||
.expect("create healthy resume state");
|
||||
CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create healthy checkpoint");
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
let checkpoint_bytes = disk
|
||||
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read healthy checkpoint before blocking");
|
||||
let marker_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &marker_path, marker.clone())
|
||||
.await
|
||||
.expect("write blocked marker");
|
||||
|
||||
assert!(
|
||||
ResumeUtils::get_resumable_tasks(&disk)
|
||||
.await
|
||||
.expect("filter blocked healthy task")
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read healthy checkpoint after blocking"),
|
||||
checkpoint_bytes
|
||||
);
|
||||
}
|
||||
temp_dir.close().expect("remove blocked selector test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_checkpoint_manager_cannot_overwrite_newer_progress() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let first = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create first checkpoint manager");
|
||||
let second = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("load second checkpoint manager");
|
||||
|
||||
second
|
||||
.update_position(4, 20)
|
||||
.await
|
||||
.expect("persist newer checkpoint progress");
|
||||
let error = first
|
||||
.update_position(1, 3)
|
||||
.await
|
||||
.expect_err("stale checkpoint manager must not overwrite newer progress");
|
||||
assert!(error.to_string().contains("newer progress"));
|
||||
|
||||
let persisted = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("load newer checkpoint progress")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(persisted.current_bucket_index, 4);
|
||||
assert_eq!(persisted.current_object_index, 20);
|
||||
temp_dir.close().expect("remove stale manager test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resumable_selector_isolates_future_and_corrupt_checkpoints() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let future_task = ResumeUtils::generate_task_id();
|
||||
let corrupt_task = ResumeUtils::generate_task_id();
|
||||
for task_id in [&future_task, &corrupt_task] {
|
||||
ResumeManager::new(
|
||||
disk.clone(),
|
||||
task_id.to_string(),
|
||||
"erasure_set".to_string(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["bucket".to_string()],
|
||||
)
|
||||
.await
|
||||
.expect("create resumable state fixture");
|
||||
}
|
||||
|
||||
let future_path = format!("{BUCKET_META_PREFIX}/{future_task}_{RESUME_CHECKPOINT_FILE}");
|
||||
let mut future = ResumeCheckpoint::new(future_task.clone());
|
||||
future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
|
||||
let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &future_path, future_bytes.clone().into())
|
||||
.await
|
||||
.expect("write future checkpoint fixture");
|
||||
let corrupt_path = format!("{BUCKET_META_PREFIX}/{corrupt_task}_{RESUME_CHECKPOINT_FILE}");
|
||||
let corrupt_bytes = b"{truncated";
|
||||
disk.write_all(RUSTFS_META_BUCKET, &corrupt_path, corrupt_bytes.as_slice().into())
|
||||
.await
|
||||
.expect("write corrupt checkpoint fixture");
|
||||
|
||||
assert!(
|
||||
ResumeUtils::get_resumable_tasks(&disk)
|
||||
.await
|
||||
.expect("filter malformed resumable tasks")
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
ResumeUtils::get_resumable_tasks(&disk)
|
||||
.await
|
||||
.expect("filter blocked resumable tasks")
|
||||
.is_empty()
|
||||
);
|
||||
for (task_id, path, bytes) in [
|
||||
(&future_task, future_path, future_bytes),
|
||||
(&corrupt_task, corrupt_path, corrupt_bytes.to_vec()),
|
||||
] {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, &path)
|
||||
.await
|
||||
.expect("read isolated checkpoint bytes"),
|
||||
bytes
|
||||
);
|
||||
let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
|
||||
assert!(
|
||||
!disk
|
||||
.read_all(RUSTFS_META_BUCKET, &blocked_path)
|
||||
.await
|
||||
.expect("read checkpoint blocked marker")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
temp_dir.close().expect("remove selector isolation test directory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_persist_throttle_batches_until_threshold() {
|
||||
let mut throttle = PersistThrottle::new();
|
||||
|
||||
@@ -21,7 +21,7 @@ use uuid::Uuid;
|
||||
use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
|
||||
use super::replacement::{ReplacementPhase, ReplacementRecoveryRecord};
|
||||
use super::{
|
||||
CheckpointManager, EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
|
||||
EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
|
||||
REPLACEMENT_INTENT_FILE, RESUME_STATE_FILE, ResumeManager, ResumeStateFile, is_replacement_intent, path_to_str,
|
||||
replacement_recovery_corruption_for_state_load, replacement_recovery_dir, validate_resume_task_id,
|
||||
};
|
||||
@@ -67,7 +67,6 @@ impl ResumeUtils {
|
||||
// Extract task ID from filename: {task_id}_ahm_resume_state.json
|
||||
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
&& CheckpointManager::is_resumable(disk, task_id).await
|
||||
{
|
||||
task_ids.push(task_id.to_string());
|
||||
}
|
||||
|
||||
@@ -622,7 +622,8 @@ mod test {
|
||||
let _resolver_lock = DNS_RESOLVER_TEST_LOCK.lock().unwrap();
|
||||
reset_dns_resolver_inner();
|
||||
|
||||
let err = resolve_domain("rustfs-resolver-provenance.invalid").unwrap_err();
|
||||
// DNS labels are limited to 63 bytes, so the system resolver rejects this before lookup.
|
||||
let err = resolve_domain("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.invalid").unwrap_err();
|
||||
|
||||
assert_ne!(err.kind(), std::io::ErrorKind::Other, "system resolver error was wrapped: {err}");
|
||||
}
|
||||
|
||||
@@ -5,12 +5,10 @@
|
||||
> ```bash
|
||||
> cargo nextest list -p e2e_test --message-format json | jq -r '.["rust-suites"][]?.testcases | to_entries[] | select(.value.ignored == false) | .key | split("::")[0]' | sort | uniq -c
|
||||
> ```
|
||||
> Modules marked ✅ are in the PR smoke profile `e2e-smoke`
|
||||
> (`.config/nextest.toml`); admission criteria: `crates/e2e_test/README.md`.
|
||||
> 🌙 marks tests in the scheduled `e2e-repl-nightly` profile (backlog#1147
|
||||
> repl-1): `replication_extension_test` splits 20 fast tests into the PR smoke
|
||||
> lane and 28 slow / `_real_dual_node` / `_real_three_node` / `_real_single_node` tests into the
|
||||
> nightly lane (`.github/workflows/e2e-replication-nightly.yml`).
|
||||
> Modules marked ✅ are in the PR smoke profile `e2e-smoke`; 🌙 marks the
|
||||
> cluster, protocol, and replication subsets in the consolidated nightly
|
||||
> workflow. The `e2e-full` merge/main profile covers the remaining default
|
||||
> single-node tests. Committed test-ID digests are enforced before each run.
|
||||
> Note: counts exclude `#[ignore]`d tests (nextest lists them separately).
|
||||
> Managed-SSE (SSE-S3/SSE-KMS) replication contracts assert successful
|
||||
> re-encryption on the target (backlog#1783); SSE-C replication still pins a
|
||||
@@ -19,19 +17,21 @@
|
||||
| module | tests | PR smoke |
|
||||
|---|---|---|
|
||||
| admin_auth_test | 4 | ✅ |
|
||||
| admin_iam_crud_test | 2 | ✅ |
|
||||
| admin_iam_crud_test | 3 | ✅ |
|
||||
| admin_pools_test | 1 | ✅ |
|
||||
| admin_timeout_regression_test | 1 | |
|
||||
| admin_timeout_regression_test | 1 | 🌙 |
|
||||
| anonymous_access_test | 4 | ✅ |
|
||||
| api_rate_limit_test | 3 | |
|
||||
| archive_download_integrity_test | 13 | |
|
||||
| bucket_logging_test | 3 | |
|
||||
| bucket_policy_check_test | 1 | ✅ |
|
||||
| bucket_stats_regression_test | 3 | |
|
||||
| chaos | 2 | |
|
||||
| checksum_upload_test | 7 | |
|
||||
| cluster_concurrency_test | 2 | |
|
||||
| cluster_multidrive_pool_test | 2 | |
|
||||
| common | 12 | |
|
||||
| compression_test | 1 | |
|
||||
| cluster_concurrency_test | 2 | 🌙 |
|
||||
| cluster_multidrive_pool_test | 2 | 🌙 |
|
||||
| common | 14 | |
|
||||
| compression_test | 6 | ✅ |
|
||||
| connection_cap_test | 2 | |
|
||||
| console_smoke_test | 1 | ✅ |
|
||||
| content_encoding_test | 3 | ✅ |
|
||||
@@ -45,48 +45,58 @@
|
||||
| delete_marker_migration_semantics_test | 2 | ✅ |
|
||||
| delete_object_no_content_length_test | 1 | |
|
||||
| delete_objects_versioning_test | 2 | ✅ |
|
||||
| delete_regression_test | 5 | |
|
||||
| distributed_startup_regression_test | 3 | |
|
||||
| existing_object_tag_policy_test | 4 | |
|
||||
| fake_s3_target | 4 | ✅ |
|
||||
| fake_s3_target | 6 | ✅ |
|
||||
| fault_proxy | 7 | |
|
||||
| get_codec_streaming_compat_test | 1 | |
|
||||
| get_stream_failure_observability_test | 1 | |
|
||||
| group_delete_test | 1 | |
|
||||
| head_object_consistency_test | 1 | ✅ |
|
||||
| head_object_range_test | 1 | ✅ |
|
||||
| heal_erasure_disk_rebuild_test | 3 | |
|
||||
| inline_fast_path_cluster_test | 14 | |
|
||||
| heal_erasure_disk_rebuild_test | 4 | 🌙 |
|
||||
| inline_fast_path_cluster_test | 16 | |
|
||||
| internode_rpc_signature_e2e_test | 5 | |
|
||||
| kms | 41 | |
|
||||
| kms | 48 | |
|
||||
| leading_slash_key_test | 2 | ✅ |
|
||||
| lifecycle_regression_test | 4 | |
|
||||
| list_buckets_auth_test | 1 | ✅ |
|
||||
| list_buckets_double_slash_test | 3 | ✅ |
|
||||
| list_buckets_iam_filter_test | 1 | ✅ |
|
||||
| list_object_versions_metadata_extension_test | 1 | |
|
||||
| list_object_versions_regression_test | 2 | ✅ |
|
||||
| list_objects_duplicates_test | 3 | ✅ |
|
||||
| list_objects_v2_metadata_extension_test | 1 | |
|
||||
| list_objects_v2_pagination_test | 12 | ✅ |
|
||||
| listing_regression_test | 4 | |
|
||||
| mc_mirror_small_bucket_test | 1 | |
|
||||
| multipart_auth_test | 75 | |
|
||||
| multipart_storage_class_test | 3 | ✅ |
|
||||
| namespace_lock_quorum_test | 2 | |
|
||||
| namespace_lock_quorum_test | 2 | 🌙 |
|
||||
| negative_sigv4_test | 6 | ✅ |
|
||||
| notification_startup_regression_test | 2 | |
|
||||
| notification_webhook_test | 3 | ✅ |
|
||||
| object_lambda_test | 16 | |
|
||||
| object_lock | 33 | |
|
||||
| object_lambda_test | 16 | 🌙 |
|
||||
| object_lock | 34 | |
|
||||
| overwrite_cleanup_regression_test | 1 | |
|
||||
| presigned_negative_test | 7 | ✅ |
|
||||
| protocols | 16 | |
|
||||
| protocols | 16 | 🌙 |
|
||||
| quota_test | 14 | |
|
||||
| reliability_disk_fault_test | 3 | |
|
||||
| reliant | 24 | 18 ✅ |
|
||||
| replication_extension_test | 50 | 20 ✅ +30 🌙 |
|
||||
| reliability_disk_fault_test | 4 | |
|
||||
| reliant | 25 | 19 ✅ |
|
||||
| replication_extension_test | 75 | 20 ✅ +55 🌙 |
|
||||
| security_boundary_test | 4 | |
|
||||
| ssec_copy_test | 2 | ✅ |
|
||||
| server_startup_failfast_test | 1 | |
|
||||
| snowball_auto_extract_test | 6 | |
|
||||
| special_chars_test | 14 | ✅ |
|
||||
| stale_multipart_cleanup_cluster_test | 1 | |
|
||||
| ssec_copy_test | 2 | ✅ |
|
||||
| stale_multipart_cleanup_cluster_test | 1 | 🌙 |
|
||||
| storage_class_capability_test | 4 | ✅ |
|
||||
| sts_query_compat_test | 3 | ✅ |
|
||||
| sts_query_compat_test | 6 | ✅ |
|
||||
| tier_transition_regression_test | 3 | |
|
||||
| tls_gen | 3 | |
|
||||
| tls_hot_reload_test | 1 | ✅ |
|
||||
| version_id_regression_test | 10 | ✅ |
|
||||
|
||||
**Total listed: 530 tests across 70 modules · PR smoke subset: 148 tests / 33 modules** (31 full modules + 18 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 30 tests** · updated 2026-08-09.
|
||||
**Total listed: 577 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 455 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-21.
|
||||
|
||||
@@ -48,11 +48,8 @@ fn materialize_case(path: String, prefix: Option<String>, flags: &[String]) -> (
|
||||
.fold((path, prefix), |(path, prefix), flag| apply_flag(path, prefix, flag))
|
||||
}
|
||||
|
||||
fn has_dot_segments(path: &str) -> bool {
|
||||
path.split(['/', '\\']).any(|segment| {
|
||||
let trimmed = segment.trim();
|
||||
trimmed == "." || trimmed == ".."
|
||||
})
|
||||
fn has_parent_segments(path: &str) -> bool {
|
||||
path.split(['/', '\\']).any(|segment| segment == "..")
|
||||
}
|
||||
|
||||
fuzz_target!(|data: &[u8]| {
|
||||
@@ -66,8 +63,8 @@ fuzz_target!(|data: &[u8]| {
|
||||
|
||||
if let Ok(key) = normalize_extract_entry_key(&path, prefix.as_deref(), is_dir) {
|
||||
assert!(
|
||||
!has_dot_segments(&key),
|
||||
"accepted archive entry retained dot segments: path={:?} prefix={:?} key={:?}",
|
||||
!has_parent_segments(&key),
|
||||
"accepted archive entry retained parent segments: path={:?} prefix={:?} key={:?}",
|
||||
path,
|
||||
prefix,
|
||||
key
|
||||
|
||||
@@ -69,7 +69,11 @@ fuzz_target!(|data: &[u8]| {
|
||||
&& let Some(object) = value.as_object()
|
||||
{
|
||||
let mut legacy_doc = Map::new();
|
||||
if let Some(policy) = object.get("Policy").or_else(|| object.get("policy")) {
|
||||
if let Some(policy) = object
|
||||
.get("Policy")
|
||||
.or_else(|| object.get("policy"))
|
||||
.filter(|policy| serde_json::from_value::<Policy>((*policy).clone()).is_ok())
|
||||
{
|
||||
legacy_doc.insert("version".to_string(), json!(1));
|
||||
legacy_doc.insert("policy".to_string(), policy.clone());
|
||||
legacy_doc.insert("create_date".to_string(), json!("2025-03-07T12:00:00Z"));
|
||||
|
||||
@@ -17,6 +17,220 @@
|
||||
//! This binary shares RustFS's existing subcommand dispatcher and provides the
|
||||
//! documented entry point for offline tooling such as `inspect bucket-meta`.
|
||||
|
||||
fn main() {
|
||||
use std::fs;
|
||||
use std::io::{Read as _, Write as _};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use rustfs::connect::offline::{OfflineEnrollment, OfflineKeyStore};
|
||||
|
||||
/// Owner read/write only. The response names the key being enrolled and the
|
||||
/// challenge it answers; neither belongs to anyone else on the machine.
|
||||
#[cfg(unix)]
|
||||
const RESPONSE_MODE: u32 = 0o600;
|
||||
|
||||
const USAGE: &str = "\
|
||||
Usage: rustfs-cli connect offline enroll --challenge <path|-> --output <path> [--key-dir <path>]
|
||||
|
||||
Answers a Connect offline enrolment challenge without a network. Reads the
|
||||
challenge from a file or from stdin when the path is `-`, verifies it against the
|
||||
enrolment root compiled into this binary, mints the key being enrolled on first
|
||||
use, and writes the signed response.
|
||||
|
||||
No secret is ever accepted on the command line.
|
||||
";
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let arguments: Vec<String> = std::env::args().skip(1).collect();
|
||||
|
||||
// Offline enrolment is handled before the server dispatcher is reached, and
|
||||
// the reason is the surface's whole point: `run_process` builds a Tokio
|
||||
// runtime and enters the server's async main. An air-gapped enrolment must
|
||||
// not start a runtime, a task, or anything that could open a socket, so the
|
||||
// two paths cannot share an entry.
|
||||
if matches!(
|
||||
arguments.first().map(String::as_str),
|
||||
Some("connect") if matches!(arguments.get(1).map(String::as_str), Some("offline"))
|
||||
) {
|
||||
return match run_offline(&arguments[2..]) {
|
||||
Ok(()) => ExitCode::SUCCESS,
|
||||
Err(message) => {
|
||||
eprintln!("rustfs-cli: {message}");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
rustfs::startup_entrypoint::run_process();
|
||||
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
|
||||
fn run_offline(arguments: &[String]) -> Result<(), String> {
|
||||
match arguments.first().map(String::as_str) {
|
||||
Some("enroll") => enroll(&arguments[1..]),
|
||||
Some(other) => Err(format!("unknown offline subcommand `{other}`\n\n{USAGE}")),
|
||||
None => Err(format!("missing offline subcommand\n\n{USAGE}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn enroll(arguments: &[String]) -> Result<(), String> {
|
||||
let mut challenge_path: Option<String> = None;
|
||||
let mut output_path: Option<String> = None;
|
||||
let mut key_directory: Option<String> = None;
|
||||
|
||||
let mut index = 0;
|
||||
while index < arguments.len() {
|
||||
let flag = arguments[index].as_str();
|
||||
let take_value = |name: &str| -> Result<String, String> {
|
||||
arguments
|
||||
.get(index + 1)
|
||||
.cloned()
|
||||
.ok_or_else(|| format!("`{name}` needs a value\n\n{USAGE}"))
|
||||
};
|
||||
|
||||
match flag {
|
||||
"--challenge" => challenge_path = Some(take_value("--challenge")?),
|
||||
"--output" => output_path = Some(take_value("--output")?),
|
||||
"--key-dir" => key_directory = Some(take_value("--key-dir")?),
|
||||
"-h" | "--help" => {
|
||||
println!("{USAGE}");
|
||||
return Ok(());
|
||||
}
|
||||
other => return Err(format!("unknown option `{other}`\n\n{USAGE}")),
|
||||
}
|
||||
|
||||
index += 2;
|
||||
}
|
||||
|
||||
let challenge_path = challenge_path.ok_or_else(|| format!("`--challenge` is required\n\n{USAGE}"))?;
|
||||
let output_path = output_path.ok_or_else(|| format!("`--output` is required\n\n{USAGE}"))?;
|
||||
let key_directory = key_directory.unwrap_or_else(|| ".".to_string());
|
||||
|
||||
let challenge = read_challenge(&challenge_path)?;
|
||||
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_err(|_| "the system clock is before the Unix epoch".to_string())?
|
||||
.as_secs() as i64;
|
||||
|
||||
let verified = OfflineEnrollment::verify_challenge(&challenge, now).map_err(|error| error.to_string())?;
|
||||
|
||||
// First use mints the key; a retry answers with the one already enrolled,
|
||||
// because the operator may already be carrying a response naming it.
|
||||
let key = OfflineKeyStore::new(&key_directory)
|
||||
.load_or_create()
|
||||
.map_err(|error| error.to_string())?;
|
||||
|
||||
let mut device_nonce = [0u8; 32];
|
||||
getrandom(&mut device_nonce)?;
|
||||
|
||||
let response = OfflineEnrollment::build_response(&verified, &key, &device_nonce, now).map_err(|error| error.to_string())?;
|
||||
|
||||
write_response(Path::new(&output_path), &response)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reads the challenge from a file, or from stdin when the path is `-`.
|
||||
///
|
||||
/// A challenge is not a secret — it is signed, public, and carried in by hand —
|
||||
/// so accepting a path is safe. The response's key never arrives this way.
|
||||
fn read_challenge(path: &str) -> Result<Vec<u8>, String> {
|
||||
if path == "-" {
|
||||
let mut buffer = Vec::new();
|
||||
std::io::stdin()
|
||||
.read_to_end(&mut buffer)
|
||||
.map_err(|error| format!("cannot read the challenge from stdin: {error}"))?;
|
||||
return Ok(buffer);
|
||||
}
|
||||
|
||||
fs::read(path).map_err(|error| format!("cannot read the challenge at {path}: {error}"))
|
||||
}
|
||||
|
||||
/// Writes the response durably and atomically at mode 0600.
|
||||
///
|
||||
/// Not the no-clobber publish `IdentityStore` performs for a key: an operator
|
||||
/// who reruns an enrolment expects the response file to be replaced, whereas a
|
||||
/// second key would strand the first. Same durability, deliberately different
|
||||
/// publication rule.
|
||||
fn write_response(path: &Path, response: &[u8]) -> Result<(), String> {
|
||||
let parent = path.parent().filter(|parent| !parent.as_os_str().is_empty());
|
||||
let temporary: PathBuf = match parent {
|
||||
Some(parent) => parent.join(format!(".{}.tmp", file_name(path))),
|
||||
None => PathBuf::from(format!(".{}.tmp", file_name(path))),
|
||||
};
|
||||
|
||||
let mut options = fs::OpenOptions::new();
|
||||
options.write(true).create(true).truncate(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt as _;
|
||||
options.mode(RESPONSE_MODE);
|
||||
}
|
||||
|
||||
let write = (|| -> std::io::Result<()> {
|
||||
let mut file = options.open(&temporary)?;
|
||||
file.write_all(response)?;
|
||||
|
||||
// The umask can only narrow the creation mode, so set the exact mode
|
||||
// before the bytes become durable.
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
file.set_permissions(fs::Permissions::from_mode(RESPONSE_MODE))?;
|
||||
}
|
||||
|
||||
file.sync_all()
|
||||
})();
|
||||
|
||||
if let Err(error) = write {
|
||||
let _ = fs::remove_file(&temporary);
|
||||
return Err(format!("cannot write the response to {}: {error}", path.display()));
|
||||
}
|
||||
|
||||
fs::rename(&temporary, path).map_err(|error| {
|
||||
let _ = fs::remove_file(&temporary);
|
||||
format!("cannot publish the response at {}: {error}", path.display())
|
||||
})?;
|
||||
|
||||
if let Some(parent) = parent {
|
||||
sync_directory(parent);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn file_name(path: &Path) -> String {
|
||||
path.file_name()
|
||||
.map(|name| name.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|| "response".to_string())
|
||||
}
|
||||
|
||||
/// Fsync the directory so the renamed entry survives power loss. Directories
|
||||
/// cannot be opened for syncing on Windows, where this is a no-op.
|
||||
fn sync_directory(directory: &Path) {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
if let Ok(handle) = fs::File::open(directory) {
|
||||
let _ = handle.sync_all();
|
||||
}
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
let _ = directory;
|
||||
}
|
||||
|
||||
/// Fills `buffer` with operating-system randomness.
|
||||
///
|
||||
/// The device nonce must be unpredictable: it is what stops a captured response
|
||||
/// being replayed as a fresh one. Sourced through p256's pinned rand_core 0.6
|
||||
/// rather than the workspace `rand` 0.10, matching `identity.rs`; the two are
|
||||
/// different crate versions and only the pinned one is on p256's own path.
|
||||
fn getrandom(buffer: &mut [u8]) -> Result<(), String> {
|
||||
use p256::elliptic_curve::rand_core::{OsRng, RngCore as _};
|
||||
|
||||
OsRng
|
||||
.try_fill_bytes(buffer)
|
||||
.map_err(|error| format!("the operating system random source failed: {error}"))
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@
|
||||
|
||||
pub mod identity;
|
||||
pub mod identity_store;
|
||||
pub mod offline;
|
||||
|
||||
pub use identity::{DeviceIdentity, IdentityError, RegistrationProof, RegistrationTranscript};
|
||||
pub use identity_store::{IdentityStore, StoreError};
|
||||
pub use offline::{EnrollmentError, OfflineEnrollment, OfflineKeyStore, VerifiedChallenge};
|
||||
|
||||
@@ -0,0 +1,684 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Challenge verification and response production for offline enrolment.
|
||||
//!
|
||||
//! Two invariants carry the security of this surface and both are easy to break
|
||||
//! by accident:
|
||||
//!
|
||||
//! - Every signature is checked over the octets that arrived, never over a
|
||||
//! re-serialised document. Parsing happens only to route the verification, and
|
||||
//! nothing a parse yields is believed until the signature over those same
|
||||
//! octets has verified.
|
||||
//! - The enrolment root is the constant in this file. It is never taken from a
|
||||
//! challenge, a configuration file, or an operator prompt, so there is no
|
||||
//! trust-on-first-use path an operator could be talked into.
|
||||
//!
|
||||
//! The order of the checks in [`OfflineEnrollment::verify_challenge`] is frozen
|
||||
//! by `verificationOrder.enrollmentChallenge` in
|
||||
//! `protocol/agent/v1/fixtures/offline-enrollment/trust-model.json`, and the
|
||||
//! signature encoding, the domain separation tags, and every rejection reason
|
||||
//! are frozen beside it. Reordering the checks changes which reason a given
|
||||
//! artifact produces, which is itself part of the contract.
|
||||
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
|
||||
use p256::ecdsa::signature::{Signer as _, Verifier as _};
|
||||
use p256::ecdsa::{Signature, SigningKey, VerifyingKey};
|
||||
use p256::pkcs8::DecodePrivateKey as _;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use time::{Date, Month, OffsetDateTime, PrimitiveDateTime, Time};
|
||||
|
||||
use crate::connect::identity::DeviceIdentity;
|
||||
|
||||
/// The hosted enrolment root, compiled in. Both halves are pinned: the
|
||||
/// fingerprint identifies the root, and the point is what actually verifies the
|
||||
/// first link, so a build cannot be pointed at a different key by supplying one.
|
||||
const PINNED_ROOT_KEY_ID: &str = "df22e2806112debbe953672aafa186d699af0e97dd3fd2b09fa8359005fe348f";
|
||||
const PINNED_ROOT_PUBLIC_KEY: &str = "BFfx-K-FfEA5nK_Rz3IHacvRCkJyQ7JOd1geLyU6HKRZDgNezmVuKhvJ22VhemyjV__Gshk8JGGqOBzYPMD0p6s";
|
||||
|
||||
/// Domain separation tags. A document that verifies under one of these must not
|
||||
/// be accepted for another artifact type, so the tag is part of the signature
|
||||
/// input rather than a property of the caller.
|
||||
const TAG_TRUST_LINK: &[u8] = b"rustfs-offline-trust-link-v1";
|
||||
const TAG_CHALLENGE: &[u8] = b"rustfs-offline-enrollment-challenge-v1";
|
||||
const TAG_RESPONSE: &[u8] = b"rustfs-offline-enrollment-response-v1";
|
||||
|
||||
/// The single octet between the tag and the signed document.
|
||||
const DOMAIN_SEPARATOR: u8 = 0x00;
|
||||
|
||||
const SIGNATURE_ALGORITHM: &str = "ES256";
|
||||
const PROTOCOL_VERSION: &str = "v1";
|
||||
const FORMAT_TRUST_LINK: &str = "rustfs.connect.offline.trustLink/1";
|
||||
const FORMAT_CHALLENGE: &str = "rustfs.connect.offline.enrollmentChallenge/1";
|
||||
const FORMAT_RESPONSE: &str = "rustfs.connect.offline.enrollmentResponse/1";
|
||||
|
||||
/// DER SubjectPublicKeyInfo header for an uncompressed P-256 point. A keyId is
|
||||
/// the SHA-256 of this prefix followed by the 65 octet point, so the prefix is
|
||||
/// also how a device public key is recovered from its own DER encoding.
|
||||
const SPKI_PREFIX: [u8; 26] = [
|
||||
0x30, 0x59, 0x30, 0x13, 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x02, 0x01, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03,
|
||||
0x01, 0x07, 0x03, 0x42, 0x00,
|
||||
];
|
||||
|
||||
/// Order of the P-256 group, and half of it. `r` and `s` must lie in `[1, n)`,
|
||||
/// and `s` additionally in `[1, n/2]`: ECDSA admits both `s` and `n - s`, and a
|
||||
/// signature with two spellings cannot serve as an artifact identity. Every
|
||||
/// ECDSA library accepts the malleated form, so the encoding layer rejects it.
|
||||
const GROUP_ORDER: [u8; 32] = [
|
||||
0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbc, 0xe6, 0xfa, 0xad, 0xa7,
|
||||
0x17, 0x9e, 0x84, 0xf3, 0xb9, 0xca, 0xc2, 0xfc, 0x63, 0x25, 0x51,
|
||||
];
|
||||
const MAX_S: [u8; 32] = [
|
||||
0x7f, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xde, 0x73, 0x7d, 0x56, 0xd3,
|
||||
0x8b, 0xcf, 0x42, 0x79, 0xdc, 0xe5, 0x61, 0x7e, 0x31, 0x92, 0xa8,
|
||||
];
|
||||
|
||||
const SCALAR_OCTETS: usize = 32;
|
||||
const SIGNATURE_OCTETS: usize = 64;
|
||||
/// 64 octets as unpadded base64url. The length is checked before decoding so
|
||||
/// that `=` padding, the standard alphabet, DER, and a truncated value are all
|
||||
/// refused rather than repaired.
|
||||
const SIGNATURE_VALUE_CHARS: usize = 86;
|
||||
|
||||
const PUBLIC_KEY_OCTETS: usize = 65;
|
||||
const PUBLIC_KEY_CHARS: usize = 87;
|
||||
/// SEC1 tag of an uncompressed point. Compressed and hybrid forms are refused.
|
||||
const UNCOMPRESSED_POINT: u8 = 0x04;
|
||||
|
||||
const TIMESTAMP_CHARS: usize = 20;
|
||||
|
||||
/// The chain is exactly two links: a pinned root issues the intermediate, and
|
||||
/// the intermediate issues the signing key. Roles are positional and the
|
||||
/// enumeration is closed.
|
||||
const CHAIN_LINK_COUNT: usize = 2;
|
||||
const CHAIN_ROLES: [&str; CHAIN_LINK_COUNT] = ["intermediate", "signing"];
|
||||
|
||||
/// Skew allowed on the challenge window. A device may have no synchronised
|
||||
/// clock at all, so its own reading of "now" is advisory.
|
||||
const CLOCK_SKEW_TOLERANCE: i64 = 300;
|
||||
|
||||
/// Longest life a challenge may claim. The issuer sets both ends of its own
|
||||
/// window, so the protocol bound is applied on top of the declared expiry
|
||||
/// rather than trusted from it.
|
||||
const MAX_CHALLENGE_LIFETIME: i64 = 604_800;
|
||||
|
||||
/// A challenge that verified, with the fields the response has to echo.
|
||||
///
|
||||
/// Construction is the proof: a value of this type only exists after the chain
|
||||
/// closed on the pinned root and the challenge signature verified over the
|
||||
/// received octets.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct VerifiedChallenge {
|
||||
pub challenge_id: String,
|
||||
pub organization_name: String,
|
||||
pub cluster_name: String,
|
||||
pub nonce: String,
|
||||
pub issued_at: String,
|
||||
pub expires_at: String,
|
||||
pub connect_key_id: String,
|
||||
/// The signature value of the challenge, verbatim. It binds a response to
|
||||
/// the one challenge it answers, so it is carried rather than recomputed.
|
||||
pub challenge_proof: String,
|
||||
}
|
||||
|
||||
/// Why an offline enrolment artifact was refused.
|
||||
///
|
||||
/// The variants are the frozen `reason` vocabulary of
|
||||
/// `fixtures/offline-enrollment/error-codes.json`, which spans both halves of
|
||||
/// the exchange. The device half implemented here produces the encoding, chain,
|
||||
/// version, and freshness reasons; the reasons that describe a response being
|
||||
/// evaluated against stored state — [`Self::ChallengeUnknown`],
|
||||
/// [`Self::ChallengeProofInvalid`], [`Self::DeviceProofInvalid`],
|
||||
/// [`Self::EnrollmentReplayed`], [`Self::OrganizationMismatch`], and
|
||||
/// [`Self::ClusterMismatch`] — are Connect's to raise and are named here so the
|
||||
/// two sides share one vocabulary.
|
||||
///
|
||||
/// No variant carries a payload: a rejection must never disclose key material,
|
||||
/// signature octets, nonces, or document bytes.
|
||||
#[derive(Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum EnrollmentError {
|
||||
#[error("protocolVersion is missing, malformed, or names an unsupported major version")]
|
||||
UnsupportedProtocol,
|
||||
|
||||
#[error("formatVersion is not a supported offline enrollment format")]
|
||||
UnsupportedFormat,
|
||||
|
||||
#[error("the signature is not 64 octets of fixed-width r||s in unpadded base64url")]
|
||||
SignatureMalformed,
|
||||
|
||||
#[error("the signature is not in its canonical low-S form")]
|
||||
SignatureNotCanonical,
|
||||
|
||||
#[error("the signature does not verify over the received octets")]
|
||||
SignatureInvalid,
|
||||
|
||||
#[error("the trust chain is not issued by a root pinned in this build")]
|
||||
EnrollmentRootUnknown,
|
||||
|
||||
#[error("a trust link is invalid, misordered, or outside its validity at the challenge issuedAt")]
|
||||
TrustChainInvalid,
|
||||
|
||||
#[error("connectKeyId is not the subject of the last trust link")]
|
||||
ConnectKeyUnchained,
|
||||
|
||||
#[error("no issued challenge matches this challengeId")]
|
||||
ChallengeUnknown,
|
||||
|
||||
#[error("the challenge is not yet valid at the evaluation time")]
|
||||
ChallengeNotYetValid,
|
||||
|
||||
#[error("the challenge has expired at the evaluation time")]
|
||||
ChallengeExpired,
|
||||
|
||||
#[error("the response nonce or challengeProof is not the one issued for this challenge")]
|
||||
ChallengeProofInvalid,
|
||||
|
||||
#[error("the response does not prove possession of the device key it presents")]
|
||||
DeviceProofInvalid,
|
||||
|
||||
#[error("the challenge was already consumed")]
|
||||
EnrollmentReplayed,
|
||||
|
||||
#[error("the response names a different organization than the challenge it answers")]
|
||||
OrganizationMismatch,
|
||||
|
||||
#[error("the response names a different cluster than the challenge it answers")]
|
||||
ClusterMismatch,
|
||||
|
||||
/// The artifact could not be read as a signed enrolment document at all: the
|
||||
/// envelope, the base64 of the signed octets, or a field the frozen order
|
||||
/// reads before the signature verifies did not parse. The frozen reason set
|
||||
/// has no code for a structurally unreadable document, so this variant maps
|
||||
/// to none of them.
|
||||
#[error("the offline enrollment document is not well formed")]
|
||||
MalformedDocument,
|
||||
|
||||
/// A fault on this side of the exchange rather than in the artifact: the
|
||||
/// device key did not round-trip through its own PKCS#8 encoding, or the
|
||||
/// caller named an instant outside the representable calendar. Fails closed
|
||||
/// because a half-produced response must never reach removable media.
|
||||
#[error("the enrollment response could not be produced on this device")]
|
||||
ResponseNotProduced,
|
||||
}
|
||||
|
||||
impl EnrollmentError {
|
||||
/// The frozen `reason` an operator and Connect both branch on.
|
||||
///
|
||||
/// The `Display` message is prose and may be reworded; this is the stable
|
||||
/// identifier, so nothing should parse the message instead. The two
|
||||
/// variants with no frozen counterpart deliberately return codes outside
|
||||
/// the frozen set rather than borrowing the nearest one, so a document that
|
||||
/// simply failed to parse can never be reported as a signature or freshness
|
||||
/// failure.
|
||||
pub fn reason(&self) -> &'static str {
|
||||
match self {
|
||||
Self::UnsupportedProtocol => "UNSUPPORTED_PROTOCOL",
|
||||
Self::UnsupportedFormat => "UNSUPPORTED_FORMAT",
|
||||
Self::SignatureMalformed => "SIGNATURE_MALFORMED",
|
||||
Self::SignatureNotCanonical => "SIGNATURE_NOT_CANONICAL",
|
||||
Self::SignatureInvalid => "SIGNATURE_INVALID",
|
||||
Self::EnrollmentRootUnknown => "ENROLLMENT_ROOT_UNKNOWN",
|
||||
Self::TrustChainInvalid => "TRUST_CHAIN_INVALID",
|
||||
Self::ConnectKeyUnchained => "CONNECT_KEY_UNCHAINED",
|
||||
Self::ChallengeUnknown => "CHALLENGE_UNKNOWN",
|
||||
Self::ChallengeNotYetValid => "CHALLENGE_NOT_YET_VALID",
|
||||
Self::ChallengeExpired => "CHALLENGE_EXPIRED",
|
||||
Self::ChallengeProofInvalid => "CHALLENGE_PROOF_INVALID",
|
||||
Self::DeviceProofInvalid => "DEVICE_PROOF_INVALID",
|
||||
Self::EnrollmentReplayed => "ENROLLMENT_REPLAYED",
|
||||
Self::OrganizationMismatch => "ORGANIZATION_MISMATCH",
|
||||
Self::ClusterMismatch => "CLUSTER_MISMATCH",
|
||||
Self::MalformedDocument => "MALFORMED_DOCUMENT",
|
||||
Self::ResponseNotProduced => "RESPONSE_NOT_PRODUCED",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed document, in the shape both directions carry it. `bytes` is
|
||||
/// standard padded base64 of the exact octets that were signed; nothing else is
|
||||
/// ever used as the signature input.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct SignedDocument {
|
||||
bytes: String,
|
||||
signature: DocumentSignature,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct DocumentSignature {
|
||||
algorithm: String,
|
||||
key_id: String,
|
||||
value: String,
|
||||
}
|
||||
|
||||
/// The three fields the frozen order permits reading before anything verifies.
|
||||
/// They route the verification and are not facts until it has.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ChallengeRouting {
|
||||
connect_key_id: String,
|
||||
issued_at: String,
|
||||
trust_chain: Vec<SignedDocument>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ChallengeDocument {
|
||||
format_version: String,
|
||||
protocol_version: String,
|
||||
challenge_id: String,
|
||||
organization_name: String,
|
||||
cluster_name: String,
|
||||
nonce: String,
|
||||
issued_at: String,
|
||||
expires_at: String,
|
||||
connect_key_id: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct TrustLink {
|
||||
format_version: String,
|
||||
protocol_version: String,
|
||||
role: String,
|
||||
issuer_key_id: String,
|
||||
subject_key_id: String,
|
||||
subject_public_key: String,
|
||||
not_before: String,
|
||||
not_after: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ResponseDocument<'a> {
|
||||
format_version: &'a str,
|
||||
protocol_version: &'a str,
|
||||
challenge_id: &'a str,
|
||||
organization_name: &'a str,
|
||||
cluster_name: &'a str,
|
||||
challenge_nonce: &'a str,
|
||||
challenge_proof: &'a str,
|
||||
device_key_id: String,
|
||||
device_public_key: String,
|
||||
device_nonce: String,
|
||||
produced_at: String,
|
||||
}
|
||||
|
||||
/// The device half of the offline enrolment exchange: bytes in, bytes out.
|
||||
pub struct OfflineEnrollment;
|
||||
|
||||
impl OfflineEnrollment {
|
||||
/// Verify an enrolment challenge and return what a response must echo.
|
||||
///
|
||||
/// `now_unix` is the device's reading of the current time, which the clock
|
||||
/// skew tolerance treats as advisory.
|
||||
pub fn verify_challenge(document: &[u8], now_unix: i64) -> Result<VerifiedChallenge, EnrollmentError> {
|
||||
let envelope: SignedDocument = serde_json::from_slice(document).map_err(|_| EnrollmentError::MalformedDocument)?;
|
||||
|
||||
// Step 1: the encoding is checked before anything is decoded from it, so
|
||||
// a DER, padded, truncated, out-of-range, or high-S signature is refused
|
||||
// on its spelling rather than handed to a library that would accept it.
|
||||
let signature = decode_signature(&envelope.signature)?;
|
||||
|
||||
// The octets that were transmitted. They are never re-serialised: every
|
||||
// later step signs and parses this same buffer.
|
||||
let bytes = BASE64_STANDARD
|
||||
.decode(envelope.bytes.as_bytes())
|
||||
.map_err(|_| EnrollmentError::MalformedDocument)?;
|
||||
|
||||
// Step 2: routing only.
|
||||
let routing: ChallengeRouting = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::MalformedDocument)?;
|
||||
let issued_at = parse_timestamp(&routing.issued_at)?;
|
||||
|
||||
// Steps 3 to 5.
|
||||
let connect_key = verify_trust_chain(&routing.trust_chain, &routing.connect_key_id, issued_at)?;
|
||||
|
||||
// Step 6. The verification key comes from the chain, so `signature.keyId`
|
||||
// is a label rather than an input: a value naming some other key simply
|
||||
// fails to verify here.
|
||||
if !verifies(&connect_key, TAG_CHALLENGE, &bytes, &signature) {
|
||||
return Err(EnrollmentError::SignatureInvalid);
|
||||
}
|
||||
|
||||
// Step 7: only now is the document read as a fact.
|
||||
let challenge: ChallengeDocument = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::MalformedDocument)?;
|
||||
if challenge.protocol_version != PROTOCOL_VERSION {
|
||||
return Err(EnrollmentError::UnsupportedProtocol);
|
||||
}
|
||||
if challenge.format_version != FORMAT_CHALLENGE {
|
||||
return Err(EnrollmentError::UnsupportedFormat);
|
||||
}
|
||||
|
||||
// Step 8.
|
||||
let expires_at = parse_timestamp(&challenge.expires_at)?;
|
||||
check_challenge_window(issued_at, expires_at, now_unix)?;
|
||||
|
||||
Ok(VerifiedChallenge {
|
||||
challenge_id: challenge.challenge_id,
|
||||
organization_name: challenge.organization_name,
|
||||
cluster_name: challenge.cluster_name,
|
||||
nonce: challenge.nonce,
|
||||
issued_at: challenge.issued_at,
|
||||
expires_at: challenge.expires_at,
|
||||
connect_key_id: challenge.connect_key_id,
|
||||
challenge_proof: envelope.signature.value,
|
||||
})
|
||||
}
|
||||
|
||||
/// Build the signed response an operator carries back to Connect.
|
||||
///
|
||||
/// `device_nonce` is the response's own replay value and must come from a
|
||||
/// cryptographic source. The private key never appears in the result: only
|
||||
/// the public point, its fingerprint, and a signature over the document
|
||||
/// that presents them, which is what makes presenting the key safe.
|
||||
pub fn build_response(
|
||||
challenge: &VerifiedChallenge,
|
||||
key: &DeviceIdentity,
|
||||
device_nonce: &[u8; 32],
|
||||
produced_at_unix: i64,
|
||||
) -> Result<Vec<u8>, EnrollmentError> {
|
||||
let issued_at = parse_timestamp(&challenge.issued_at)?;
|
||||
let expires_at = parse_timestamp(&challenge.expires_at)?;
|
||||
// Connect re-checks producedAt against the same window, so a response
|
||||
// outside it is refused here rather than written to media and rejected
|
||||
// after the operator has carried it out.
|
||||
check_challenge_window(issued_at, expires_at, produced_at_unix)?;
|
||||
|
||||
let point = device_public_point(key)?;
|
||||
let produced_at = format_timestamp(produced_at_unix)?;
|
||||
|
||||
let document = ResponseDocument {
|
||||
format_version: FORMAT_RESPONSE,
|
||||
protocol_version: PROTOCOL_VERSION,
|
||||
challenge_id: &challenge.challenge_id,
|
||||
organization_name: &challenge.organization_name,
|
||||
cluster_name: &challenge.cluster_name,
|
||||
challenge_nonce: &challenge.nonce,
|
||||
challenge_proof: &challenge.challenge_proof,
|
||||
device_key_id: key_id(&point),
|
||||
device_public_key: BASE64_URL_NO_PAD.encode(point),
|
||||
device_nonce: BASE64_URL_NO_PAD.encode(device_nonce),
|
||||
produced_at,
|
||||
};
|
||||
|
||||
// Serialised once. These octets are what is signed and what is carried,
|
||||
// so no second serialisation can disagree with the signature.
|
||||
let bytes = serde_json::to_vec(&document).map_err(|_| EnrollmentError::ResponseNotProduced)?;
|
||||
let signature = sign(key, TAG_RESPONSE, &bytes)?;
|
||||
|
||||
let envelope = SignedDocument {
|
||||
bytes: BASE64_STANDARD.encode(&bytes),
|
||||
signature: DocumentSignature {
|
||||
algorithm: SIGNATURE_ALGORITHM.to_owned(),
|
||||
key_id: document.device_key_id,
|
||||
value: signature,
|
||||
},
|
||||
};
|
||||
|
||||
serde_json::to_vec(&envelope).map_err(|_| EnrollmentError::ResponseNotProduced)
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk the chain from the pinned root to the signing key, returning the key
|
||||
/// `connect_key_id` names once the chain vouches for it.
|
||||
fn verify_trust_chain(
|
||||
chain: &[SignedDocument],
|
||||
connect_key_id: &str,
|
||||
challenge_issued_at: i64,
|
||||
) -> Result<VerifyingKey, EnrollmentError> {
|
||||
// The pinned root gate runs before the chain's shape is examined, so a
|
||||
// chain that is internally consistent under a foreign root — exactly what
|
||||
// trust on first use would have accepted — is refused for its root rather
|
||||
// than for its length.
|
||||
let first = chain.first().ok_or(EnrollmentError::EnrollmentRootUnknown)?;
|
||||
let root = decode_trust_link(first)?;
|
||||
if root.0.issuer_key_id != PINNED_ROOT_KEY_ID {
|
||||
return Err(EnrollmentError::EnrollmentRootUnknown);
|
||||
}
|
||||
|
||||
let [_, second] = chain else {
|
||||
return Err(EnrollmentError::TrustChainInvalid);
|
||||
};
|
||||
let links = [root, decode_trust_link(second)?];
|
||||
|
||||
let mut issuer_key_id = PINNED_ROOT_KEY_ID.to_owned();
|
||||
let (mut issuer_key, _) = decode_public_key(PINNED_ROOT_PUBLIC_KEY).ok_or(EnrollmentError::EnrollmentRootUnknown)?;
|
||||
|
||||
for (index, ((link, link_bytes), entry)) in links.iter().zip(chain).enumerate() {
|
||||
if link.format_version != FORMAT_TRUST_LINK
|
||||
|| link.protocol_version != PROTOCOL_VERSION
|
||||
|| link.role != CHAIN_ROLES[index]
|
||||
|| link.issuer_key_id != issuer_key_id
|
||||
// A link that names itself as its own issuer would let a stolen
|
||||
// intermediate mint its own root.
|
||||
|| link.subject_key_id == link.issuer_key_id
|
||||
{
|
||||
return Err(EnrollmentError::TrustChainInvalid);
|
||||
}
|
||||
|
||||
let (subject_key, subject_point) =
|
||||
decode_public_key(&link.subject_public_key).ok_or(EnrollmentError::TrustChainInvalid)?;
|
||||
if key_id(&subject_point) != link.subject_key_id {
|
||||
return Err(EnrollmentError::TrustChainInvalid);
|
||||
}
|
||||
|
||||
let signature = decode_signature(&entry.signature)?;
|
||||
if !verifies(&issuer_key, TAG_TRUST_LINK, link_bytes, &signature) {
|
||||
return Err(EnrollmentError::TrustChainInvalid);
|
||||
}
|
||||
|
||||
// The issuer controls both ends of a link's window, so it is evaluated
|
||||
// with no skew tolerance, and against the challenge's issuedAt rather
|
||||
// than against the device clock: a challenge carries the chain that was
|
||||
// valid when it was issued.
|
||||
let not_before = parse_timestamp(&link.not_before)?;
|
||||
let not_after = parse_timestamp(&link.not_after)?;
|
||||
if challenge_issued_at < not_before || challenge_issued_at > not_after {
|
||||
return Err(EnrollmentError::TrustChainInvalid);
|
||||
}
|
||||
|
||||
issuer_key_id = link.subject_key_id.clone();
|
||||
issuer_key = subject_key;
|
||||
}
|
||||
|
||||
if issuer_key_id != connect_key_id {
|
||||
return Err(EnrollmentError::ConnectKeyUnchained);
|
||||
}
|
||||
|
||||
Ok(issuer_key)
|
||||
}
|
||||
|
||||
/// Decode a link and keep the octets it was signed over: the signature is
|
||||
/// checked against these, never against a re-encoding of the parsed link.
|
||||
fn decode_trust_link(entry: &SignedDocument) -> Result<(TrustLink, Vec<u8>), EnrollmentError> {
|
||||
let bytes = BASE64_STANDARD
|
||||
.decode(entry.bytes.as_bytes())
|
||||
.map_err(|_| EnrollmentError::MalformedDocument)?;
|
||||
let link = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::TrustChainInvalid)?;
|
||||
Ok((link, bytes))
|
||||
}
|
||||
|
||||
/// Check a signature's spelling and range, then admit it.
|
||||
///
|
||||
/// `r` and `s` are compared against the group order here rather than left to
|
||||
/// the ECDSA library, because a library that accepts high-S — every library
|
||||
/// does — would let a malleated copy of an artifact pass as a second artifact.
|
||||
fn decode_signature(signature: &DocumentSignature) -> Result<Signature, EnrollmentError> {
|
||||
if signature.algorithm != SIGNATURE_ALGORITHM {
|
||||
return Err(EnrollmentError::SignatureMalformed);
|
||||
}
|
||||
|
||||
let value = signature.value.as_bytes();
|
||||
if value.len() != SIGNATURE_VALUE_CHARS || !value.iter().all(|byte| is_base64url(*byte)) {
|
||||
return Err(EnrollmentError::SignatureMalformed);
|
||||
}
|
||||
|
||||
let decoded = BASE64_URL_NO_PAD
|
||||
.decode(value)
|
||||
.map_err(|_| EnrollmentError::SignatureMalformed)?;
|
||||
let octets: [u8; SIGNATURE_OCTETS] = decoded
|
||||
.as_slice()
|
||||
.try_into()
|
||||
.map_err(|_| EnrollmentError::SignatureMalformed)?;
|
||||
|
||||
// Big-endian octets of equal length order lexicographically exactly as the
|
||||
// integers they spell, so a slice comparison is the range check.
|
||||
let (r, s) = octets.split_at(SCALAR_OCTETS);
|
||||
let out_of_range = |scalar: &[u8]| scalar.iter().all(|byte| *byte == 0) || scalar >= &GROUP_ORDER[..];
|
||||
if out_of_range(r) || out_of_range(s) {
|
||||
return Err(EnrollmentError::SignatureMalformed);
|
||||
}
|
||||
if s > &MAX_S[..] {
|
||||
return Err(EnrollmentError::SignatureNotCanonical);
|
||||
}
|
||||
|
||||
Signature::from_slice(&octets).map_err(|_| EnrollmentError::SignatureMalformed)
|
||||
}
|
||||
|
||||
fn verifies(key: &VerifyingKey, tag: &[u8], bytes: &[u8], signature: &Signature) -> bool {
|
||||
key.verify(&signature_input(tag, bytes), signature).is_ok()
|
||||
}
|
||||
|
||||
fn signature_input(tag: &[u8], bytes: &[u8]) -> Vec<u8> {
|
||||
let mut input = Vec::with_capacity(tag.len() + 1 + bytes.len());
|
||||
input.extend_from_slice(tag);
|
||||
input.push(DOMAIN_SEPARATOR);
|
||||
input.extend_from_slice(bytes);
|
||||
input
|
||||
}
|
||||
|
||||
fn sign(key: &DeviceIdentity, tag: &[u8], bytes: &[u8]) -> Result<String, EnrollmentError> {
|
||||
// `DeviceIdentity` publishes no general signing operation, so the key is
|
||||
// rebuilt from its own PKCS#8 encoding; the encoding is wiped when the
|
||||
// wrapper drops.
|
||||
let pkcs8 = key.to_pkcs8_der().map_err(|_| EnrollmentError::ResponseNotProduced)?;
|
||||
let signing_key = SigningKey::from_pkcs8_der(pkcs8.as_slice()).map_err(|_| EnrollmentError::ResponseNotProduced)?;
|
||||
|
||||
let signature: Signature = signing_key.sign(&signature_input(tag, bytes));
|
||||
let canonical = signature.normalize_s().unwrap_or(signature);
|
||||
|
||||
Ok(BASE64_URL_NO_PAD.encode(canonical.to_bytes()))
|
||||
}
|
||||
|
||||
/// The device's public point, recovered from the DER encoding the identity
|
||||
/// publishes so that one prefix constant governs both the fingerprint and the
|
||||
/// wire form.
|
||||
fn device_public_point(key: &DeviceIdentity) -> Result<[u8; PUBLIC_KEY_OCTETS], EnrollmentError> {
|
||||
key.public_key_der()
|
||||
.strip_prefix(&SPKI_PREFIX)
|
||||
.and_then(|point| <[u8; PUBLIC_KEY_OCTETS]>::try_from(point).ok())
|
||||
.ok_or(EnrollmentError::ResponseNotProduced)
|
||||
}
|
||||
|
||||
/// Decode an uncompressed SEC1 point and check that it is on the curve.
|
||||
///
|
||||
/// The length and alphabet are checked before decoding so that a padded or
|
||||
/// standard-alphabet spelling is refused, and the point tag is checked so that
|
||||
/// the compressed and hybrid forms — which no keyId would match — cannot be
|
||||
/// spelled at all.
|
||||
fn decode_public_key(value: &str) -> Option<(VerifyingKey, [u8; PUBLIC_KEY_OCTETS])> {
|
||||
let value = value.as_bytes();
|
||||
if value.len() != PUBLIC_KEY_CHARS || !value.iter().all(|byte| is_base64url(*byte)) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let point: [u8; PUBLIC_KEY_OCTETS] = BASE64_URL_NO_PAD.decode(value).ok()?.try_into().ok()?;
|
||||
if point[0] != UNCOMPRESSED_POINT {
|
||||
return None;
|
||||
}
|
||||
|
||||
VerifyingKey::from_sec1_bytes(&point).ok().map(|key| (key, point))
|
||||
}
|
||||
|
||||
/// Lowercase SHA-256 hex of the DER SubjectPublicKeyInfo built from a 65 octet
|
||||
/// uncompressed point.
|
||||
fn key_id(point: &[u8]) -> String {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(SPKI_PREFIX);
|
||||
digest.update(point);
|
||||
hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
fn is_base64url(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_'
|
||||
}
|
||||
|
||||
/// Parse `YYYY-MM-DDTHH:MM:SSZ` into a Unix instant.
|
||||
///
|
||||
/// The shape is checked before the fields are read: offsets other than `Z` and
|
||||
/// fractional seconds are refused rather than normalised, so two producers
|
||||
/// cannot spell the same instant two ways.
|
||||
fn parse_timestamp(value: &str) -> Result<i64, EnrollmentError> {
|
||||
let octets = value.as_bytes();
|
||||
if octets.len() != TIMESTAMP_CHARS
|
||||
|| octets[4] != b'-'
|
||||
|| octets[7] != b'-'
|
||||
|| octets[10] != b'T'
|
||||
|| octets[13] != b':'
|
||||
|| octets[16] != b':'
|
||||
|| octets[19] != b'Z'
|
||||
{
|
||||
return Err(EnrollmentError::MalformedDocument);
|
||||
}
|
||||
|
||||
let field = |range: std::ops::Range<usize>| -> Result<u32, EnrollmentError> {
|
||||
let text = &value[range];
|
||||
if !text.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(EnrollmentError::MalformedDocument);
|
||||
}
|
||||
text.parse().map_err(|_| EnrollmentError::MalformedDocument)
|
||||
};
|
||||
|
||||
let month = Month::try_from(field(5..7)? as u8).map_err(|_| EnrollmentError::MalformedDocument)?;
|
||||
let date = Date::from_calendar_date(field(0..4)? as i32, month, field(8..10)? as u8)
|
||||
.map_err(|_| EnrollmentError::MalformedDocument)?;
|
||||
let clock = Time::from_hms(field(11..13)? as u8, field(14..16)? as u8, field(17..19)? as u8)
|
||||
.map_err(|_| EnrollmentError::MalformedDocument)?;
|
||||
|
||||
Ok(PrimitiveDateTime::new(date, clock).assume_utc().unix_timestamp())
|
||||
}
|
||||
|
||||
fn format_timestamp(unix: i64) -> Result<String, EnrollmentError> {
|
||||
let moment = OffsetDateTime::from_unix_timestamp(unix).map_err(|_| EnrollmentError::ResponseNotProduced)?;
|
||||
Ok(format!(
|
||||
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z",
|
||||
moment.year(),
|
||||
u8::from(moment.month()),
|
||||
moment.day(),
|
||||
moment.hour(),
|
||||
moment.minute(),
|
||||
moment.second()
|
||||
))
|
||||
}
|
||||
|
||||
/// `at` must fall within `[issuedAt - 300, expiresAt + 300]`.
|
||||
///
|
||||
/// The declared expiry is capped at the protocol's maximum challenge lifetime
|
||||
/// because the issuer sets both ends of its own window; a challenge claiming a
|
||||
/// longer life expires at the bound.
|
||||
fn check_challenge_window(issued_at: i64, expires_at: i64, at: i64) -> Result<(), EnrollmentError> {
|
||||
if at < issued_at.saturating_sub(CLOCK_SKEW_TOLERANCE) {
|
||||
return Err(EnrollmentError::ChallengeNotYetValid);
|
||||
}
|
||||
|
||||
let effective_expiry = expires_at.min(issued_at.saturating_add(MAX_CHALLENGE_LIFETIME));
|
||||
if at > effective_expiry.saturating_add(CLOCK_SKEW_TOLERANCE) {
|
||||
return Err(EnrollmentError::ChallengeExpired);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! On-disk home of the offline enrollment key.
|
||||
//!
|
||||
//! An air-gapped device enrols with a key that is not its online device
|
||||
//! identity: the online key is minted during a registration exchange this
|
||||
//! device cannot perform, and an operator who carries an enrolment response out
|
||||
//! on removable media is enrolling exactly one key that Connect will pin. Losing
|
||||
//! it means asking for a fresh challenge, so it is written durably and published
|
||||
//! exactly once.
|
||||
//!
|
||||
//! The durability protocol is not reimplemented here. [`IdentityStore`] already
|
||||
//! seals a P-256 key at mode 0600, fsyncs it, and publishes it through a
|
||||
//! no-clobber link so a retry or a concurrent start converges on one key; it is
|
||||
//! pointed at a directory of this key's own rather than generalised into a
|
||||
//! key-store abstraction that would have to describe both lifecycles.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::super::identity::DeviceIdentity;
|
||||
use super::super::identity_store::{IdentityStore, StoreError};
|
||||
|
||||
/// Subdirectory holding the offline enrolment key, kept apart from the online
|
||||
/// device identity so neither can be read in place of the other.
|
||||
const OFFLINE_DIRECTORY: &str = "offline";
|
||||
|
||||
/// The offline enrolment key of one deployment.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct OfflineKeyStore {
|
||||
inner: IdentityStore,
|
||||
}
|
||||
|
||||
impl OfflineKeyStore {
|
||||
pub fn new(directory: impl AsRef<Path>) -> Self {
|
||||
Self {
|
||||
inner: IdentityStore::new(directory.as_ref().join(OFFLINE_DIRECTORY)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn key_path(&self) -> PathBuf {
|
||||
self.inner.key_path()
|
||||
}
|
||||
|
||||
/// Return the stored key, or `None` when this deployment has never enrolled
|
||||
/// offline. Reading never creates one, so a deployment that only ever
|
||||
/// registers online holds no offline key.
|
||||
pub fn load(&self) -> Result<Option<DeviceIdentity>, StoreError> {
|
||||
self.inner.load()
|
||||
}
|
||||
|
||||
/// Return the stored key, generating and publishing one the first time.
|
||||
///
|
||||
/// A second enrolment attempt returns the original key rather than minting a
|
||||
/// replacement: the operator may already be carrying a response for it, and
|
||||
/// two keys would mean the response and the device disagree about which one
|
||||
/// Connect pinned.
|
||||
pub fn load_or_create(&self) -> Result<DeviceIdentity, StoreError> {
|
||||
self.inner.load_or_create()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Offline enrolment: joining a Connect tenant without a network.
|
||||
//!
|
||||
//! An air-gapped cluster cannot perform the registration exchange, so an
|
||||
//! operator carries a signed challenge in and a signed response out. The device
|
||||
//! half of that exchange lives here: verifying the challenge against a root
|
||||
//! whose fingerprint is compiled into this binary, minting the key being
|
||||
//! enrolled, and signing the response.
|
||||
//!
|
||||
//! Nothing here opens a socket. That is the point of the surface, and it is
|
||||
//! asserted rather than assumed: the enrolment path takes bytes and returns
|
||||
//! bytes.
|
||||
//!
|
||||
//! The trust model, the signing convention, and every rejection reason are
|
||||
//! frozen by `protocol/agent/v1/fixtures/offline-enrollment/` and by
|
||||
//! `docs/adr/0009-offline-signing.md` on the Connect side.
|
||||
|
||||
pub mod enrollment;
|
||||
pub mod key_store;
|
||||
|
||||
pub use enrollment::{EnrollmentError, OfflineEnrollment, VerifiedChallenge};
|
||||
pub use key_store::OfflineKeyStore;
|
||||
@@ -0,0 +1,951 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Offline enrollment conformance against the frozen Connect fixtures.
|
||||
//!
|
||||
//! The device half of the air-gapped exchange verifies a challenge Connect
|
||||
//! signed and produces a response Connect will verify. Neither side can talk to
|
||||
//! the other while it does so, which means every disagreement about encoding,
|
||||
//! trust, or clock windows surfaces as a failed enrollment in the field rather
|
||||
//! than as an error at development time. The fixtures under
|
||||
//! `protocol/agent/v1/fixtures/offline-enrollment/` are the shared statement of
|
||||
//! what both sides must do, so this suite replays them rather than restating
|
||||
//! them: accept vectors must be accepted with the fields the document carries,
|
||||
//! reject vectors must fail with the single reason `error-codes.json` freezes,
|
||||
//! and the signature encoding rules in `trust-model.json` must hold even where
|
||||
//! the underlying ECDSA library is happy.
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
|
||||
use rustfs::connect::identity::DeviceIdentity;
|
||||
use rustfs::connect::offline::{EnrollmentError, OfflineEnrollment, VerifiedChallenge};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
/// DER prefix of a P-256 `SubjectPublicKeyInfo`, frozen by
|
||||
/// `trust-model.json` as `signature.subjectPublicKeyInfoDerPrefix`. The 65
|
||||
/// octet uncompressed point follows it, so a SEC1 point published in a fixture
|
||||
/// becomes a decodable public key by concatenation.
|
||||
const SPKI_PREFIX_HEX: &str = "3059301306072a8648ce3d020106082a8648ce3d030107034200";
|
||||
|
||||
/// `clockSkew.toleranceSeconds` in `trust-model.json`.
|
||||
const SKEW_TOLERANCE_SECONDS: i64 = 300;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture access
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn fixture_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../protocol/agent/v1/fixtures/offline-enrollment")
|
||||
}
|
||||
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
|
||||
/// Read one fixture file and refuse it unless its bytes match the digest
|
||||
/// `MANIFEST.sha256` freezes.
|
||||
///
|
||||
/// Every vector in this suite arrives through here. A fixture edited on this
|
||||
/// side therefore fails the tests that depend on it instead of quietly
|
||||
/// redefining what conformance means, which is the failure mode a
|
||||
/// fixture-driven suite is otherwise blind to.
|
||||
fn read_fixture(name: &str) -> Vec<u8> {
|
||||
let dir = fixture_dir();
|
||||
let manifest = fs::read_to_string(dir.join("MANIFEST.sha256")).expect("read MANIFEST.sha256");
|
||||
|
||||
let expected = manifest
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.find_map(|line| {
|
||||
let (digest, file) = line
|
||||
.split_once(" ")
|
||||
.unwrap_or_else(|| panic!("malformed manifest line: {line}"));
|
||||
(file == name).then(|| digest.to_string())
|
||||
})
|
||||
.unwrap_or_else(|| panic!("{name} is not listed in MANIFEST.sha256"));
|
||||
|
||||
let bytes = fs::read(dir.join(name)).unwrap_or_else(|error| panic!("read {name}: {error}"));
|
||||
assert_eq!(sha256_hex(&bytes), expected, "{name} does not match the digest MANIFEST.sha256 freezes");
|
||||
bytes
|
||||
}
|
||||
|
||||
fn fixture_json(name: &str) -> Value {
|
||||
serde_json::from_slice(&read_fixture(name)).unwrap_or_else(|error| panic!("{name} parses: {error}"))
|
||||
}
|
||||
|
||||
fn accept_vectors() -> Value {
|
||||
fixture_json("accept-vectors.json")
|
||||
}
|
||||
|
||||
fn reject_vectors() -> Value {
|
||||
fixture_json("reject-vectors.json")
|
||||
}
|
||||
|
||||
fn trust_model() -> Value {
|
||||
fixture_json("trust-model.json")
|
||||
}
|
||||
|
||||
fn vector_list(fixture: &Value) -> Vec<Value> {
|
||||
fixture["vectors"].as_array().expect("fixture carries a vector list").clone()
|
||||
}
|
||||
|
||||
fn field<'a>(value: &'a Value, key: &str) -> &'a str {
|
||||
value[key]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("expected a string at '{key}' in {value}"))
|
||||
}
|
||||
|
||||
/// The octets an operator carries in on removable media.
|
||||
///
|
||||
/// The fixture's `document` object *is* the transmitted artifact: a padded
|
||||
/// base64 `bytes` field holding the raw signed octets, plus the detached
|
||||
/// signature over them. Only `bytes` is covered by the signature, so
|
||||
/// re-serialising the surrounding envelope here cannot change what a verifier
|
||||
/// checks.
|
||||
fn envelope(document: &Value) -> Vec<u8> {
|
||||
serde_json::to_vec(document).expect("envelope serialises")
|
||||
}
|
||||
|
||||
/// The raw octets the signature covers, exactly as transmitted.
|
||||
fn signed_octets(document: &Value) -> Vec<u8> {
|
||||
BASE64_STANDARD
|
||||
.decode(field(document, "bytes"))
|
||||
.expect("document bytes are padded base64")
|
||||
}
|
||||
|
||||
/// The parsed signed document. Parsing is a convenience for the assertions
|
||||
/// below; the implementation under test is required to verify before it parses.
|
||||
fn signed_document(document: &Value) -> Value {
|
||||
serde_json::from_slice(&signed_octets(document)).expect("signed document parses")
|
||||
}
|
||||
|
||||
fn unix(rfc3339: &str) -> i64 {
|
||||
chrono::DateTime::parse_from_rfc3339(rfc3339)
|
||||
.unwrap_or_else(|error| panic!("'{rfc3339}' is not RFC 3339: {error}"))
|
||||
.timestamp()
|
||||
}
|
||||
|
||||
fn hex_to_bytes(hex: &str) -> Vec<u8> {
|
||||
(0..hex.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).expect("valid hex"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Turn a fixture's unpadded-base64url SEC1 point into a usable verifying key.
|
||||
fn verifying_key(sec1_base64url: &str) -> p256::ecdsa::VerifyingKey {
|
||||
let point = BASE64_URL_NO_PAD.decode(sec1_base64url).expect("public key is base64url");
|
||||
assert_eq!(point.len(), 65, "the protocol freezes a 65 octet uncompressed SEC1 point");
|
||||
|
||||
let mut der = hex_to_bytes(SPKI_PREFIX_HEX);
|
||||
der.extend_from_slice(&point);
|
||||
<p256::ecdsa::VerifyingKey as p256::pkcs8::DecodePublicKey>::from_public_key_der(&der).expect("public key decodes")
|
||||
}
|
||||
|
||||
fn published_key(role_or_name: &str) -> Value {
|
||||
fixture_json("trust-chain.json")["keys"]
|
||||
.as_array()
|
||||
.expect("trust chain publishes keys")
|
||||
.iter()
|
||||
.find(|key| field(key, "name") == role_or_name)
|
||||
.unwrap_or_else(|| panic!("trust-chain.json publishes no key named '{role_or_name}'"))
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// `signatureInput = domainSeparationTag || 0x00 || the received octets`, the
|
||||
/// rule `trust-model.json` freezes under `domainSeparation`.
|
||||
fn signing_input(artifact_tag: &str, received: &[u8]) -> Vec<u8> {
|
||||
let mut input = artifact_tag.as_bytes().to_vec();
|
||||
input.push(0x00);
|
||||
input.extend_from_slice(received);
|
||||
input
|
||||
}
|
||||
|
||||
fn domain_tag(artifact: &str) -> String {
|
||||
let model = trust_model();
|
||||
assert_eq!(
|
||||
field(&model["domainSeparation"], "separatorByte"),
|
||||
"0x00",
|
||||
"the separator byte this suite encodes is the one the trust model freezes"
|
||||
);
|
||||
field(&model["domainSeparation"]["tags"], artifact).to_string()
|
||||
}
|
||||
|
||||
/// Locate an accept vector by the name other vectors reference it by.
|
||||
fn accept_vector_named(name: &str) -> Value {
|
||||
vector_list(&accept_vectors())
|
||||
.into_iter()
|
||||
.find(|vector| field(vector, "name") == name)
|
||||
.unwrap_or_else(|| panic!("accept-vectors.json carries no vector named '{name}'"))
|
||||
}
|
||||
|
||||
/// Verify the challenge a response vector answers, at that challenge's own
|
||||
/// evaluation time.
|
||||
fn answered_challenge(response_vector: &Value) -> (Value, VerifiedChallenge) {
|
||||
let challenge_vector = accept_vector_named(field(response_vector, "answersChallenge"));
|
||||
let now = unix(field(&challenge_vector, "evaluationTime"));
|
||||
let verified = OfflineEnrollment::verify_challenge(&envelope(&challenge_vector["document"]), now)
|
||||
.expect("the answered challenge is an accept vector and must verify");
|
||||
(challenge_vector, verified)
|
||||
}
|
||||
|
||||
fn device_nonce_of(document: &Value) -> [u8; 32] {
|
||||
let raw = BASE64_URL_NO_PAD
|
||||
.decode(field(&signed_document(document), "deviceNonce"))
|
||||
.expect("deviceNonce is base64url");
|
||||
raw.try_into().expect("replay.nonceLengthBytes freezes a 32 octet nonce")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Accept vectors
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Every challenge accept vector must verify at its own evaluation time and
|
||||
/// expose exactly what the signed document says.
|
||||
///
|
||||
/// Two of these vectors sit on the skew boundary — 120 seconds before
|
||||
/// `issuedAt` and 300 seconds after `expiresAt` — so a verifier that compares
|
||||
/// against the raw window instead of the tolerated one fails here rather than
|
||||
/// in an air-gapped data centre. `challenge_proof` is pinned to the challenge's
|
||||
/// own detached signature value because that is what the response has to echo;
|
||||
/// deriving it from anything else would silently break the binding.
|
||||
#[test]
|
||||
fn every_challenge_accept_vector_verifies_and_exposes_the_signed_fields() {
|
||||
let mut verified_count = 0usize;
|
||||
|
||||
for vector in vector_list(&accept_vectors()) {
|
||||
if field(&vector, "artifact") != "challenge" {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = field(&vector, "name");
|
||||
let document = &vector["document"];
|
||||
let now = unix(field(&vector, "evaluationTime"));
|
||||
|
||||
let verified = OfflineEnrollment::verify_challenge(&envelope(document), now)
|
||||
.unwrap_or_else(|error| panic!("accept vector '{name}' must verify: {}", error.reason()));
|
||||
|
||||
let signed = signed_document(document);
|
||||
assert_eq!(verified.challenge_id, field(&signed, "challengeId"), "vector '{name}' challengeId");
|
||||
assert_eq!(
|
||||
verified.organization_name,
|
||||
field(&signed, "organizationName"),
|
||||
"vector '{name}' organizationName"
|
||||
);
|
||||
assert_eq!(verified.cluster_name, field(&signed, "clusterName"), "vector '{name}' clusterName");
|
||||
assert_eq!(verified.nonce, field(&signed, "nonce"), "vector '{name}' nonce");
|
||||
assert_eq!(verified.issued_at, field(&signed, "issuedAt"), "vector '{name}' issuedAt");
|
||||
assert_eq!(verified.expires_at, field(&signed, "expiresAt"), "vector '{name}' expiresAt");
|
||||
assert_eq!(verified.connect_key_id, field(&signed, "connectKeyId"), "vector '{name}' connectKeyId");
|
||||
assert_eq!(
|
||||
verified.challenge_proof,
|
||||
field(&document["signature"], "value"),
|
||||
"vector '{name}' must carry the challenge's own signature as the proof a response echoes"
|
||||
);
|
||||
|
||||
verified_count += 1;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
verified_count, 3,
|
||||
"accept-vectors.json publishes three challenge vectors; a fourth is a protocol change"
|
||||
);
|
||||
}
|
||||
|
||||
/// Connect's own producer wrote the response accept vectors. Rebuilding them
|
||||
/// from the challenge they answer, with the device nonce and production time
|
||||
/// they used, must reproduce every field that does not depend on which device
|
||||
/// key signed — including the discarded-unknown-field vector, whose extra
|
||||
/// `telemetryHint` must not survive into anything this side produces.
|
||||
#[test]
|
||||
fn response_accept_vectors_are_reproduced_field_for_field_by_build_response() {
|
||||
let key = DeviceIdentity::generate();
|
||||
let mut reproduced = 0usize;
|
||||
|
||||
for vector in vector_list(&accept_vectors()) {
|
||||
if field(&vector, "artifact") != "response" {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = field(&vector, "name");
|
||||
let published = signed_document(&vector["document"]);
|
||||
let (_, challenge) = answered_challenge(&vector);
|
||||
|
||||
let produced_at = unix(field(&published, "producedAt"));
|
||||
let nonce = device_nonce_of(&vector["document"]);
|
||||
|
||||
let built_envelope: Value = serde_json::from_slice(
|
||||
&OfflineEnrollment::build_response(&challenge, &key, &nonce, produced_at)
|
||||
.unwrap_or_else(|error| panic!("vector '{name}' must be reproducible: {}", error.reason())),
|
||||
)
|
||||
.expect("the built response is JSON");
|
||||
let built = signed_document(&built_envelope);
|
||||
|
||||
for shared in [
|
||||
"formatVersion",
|
||||
"protocolVersion",
|
||||
"challengeId",
|
||||
"organizationName",
|
||||
"clusterName",
|
||||
"challengeNonce",
|
||||
"challengeProof",
|
||||
"deviceNonce",
|
||||
] {
|
||||
assert_eq!(
|
||||
field(&built, shared),
|
||||
field(&published, shared),
|
||||
"vector '{name}' field {shared} must match the response Connect published"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
unix(field(&built, "producedAt")),
|
||||
produced_at,
|
||||
"vector '{name}' producedAt must be the instant it was given"
|
||||
);
|
||||
|
||||
// `versioning.additive` says an unknown optional field is discarded and
|
||||
// never echoed back; a producer that copied the challenge or a previous
|
||||
// response wholesale would carry it forward.
|
||||
assert!(
|
||||
built.get("telemetryHint").is_none(),
|
||||
"vector '{name}' must not echo an unknown optional field"
|
||||
);
|
||||
|
||||
reproduced += 1;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
reproduced, 2,
|
||||
"accept-vectors.json publishes two response vectors; a third is a protocol change"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Reject vectors
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Every challenge reject vector must fail, and fail for the one reason
|
||||
/// `error-codes.json` freezes.
|
||||
///
|
||||
/// Asserting only that verification failed would pass for an implementation
|
||||
/// that rejects everything, and would let a tampered document be reported as an
|
||||
/// expiry — a rejection reason is what an operator acts on, so it is part of the
|
||||
/// contract rather than a diagnostic detail.
|
||||
#[test]
|
||||
fn every_challenge_reject_vector_fails_with_its_frozen_reason() {
|
||||
let known_reasons: Vec<String> = fixture_json("error-codes.json")["reasons"]
|
||||
.as_array()
|
||||
.expect("error-codes.json carries reasons")
|
||||
.iter()
|
||||
.map(|entry| field(entry, "reason").to_string())
|
||||
.collect();
|
||||
|
||||
let mut rejected = 0usize;
|
||||
|
||||
for vector in vector_list(&reject_vectors()) {
|
||||
if field(&vector, "artifact") != "challenge" {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = field(&vector, "name");
|
||||
let expected = field(&vector["expected"], "reason");
|
||||
assert!(
|
||||
known_reasons.iter().any(|reason| reason == expected),
|
||||
"vector '{name}' names reason {expected}, which error-codes.json does not freeze"
|
||||
);
|
||||
|
||||
let now = unix(field(&vector, "evaluationTime"));
|
||||
let error = OfflineEnrollment::verify_challenge(&envelope(&vector["document"]), now)
|
||||
.expect_err(&format!("reject vector '{name}' must not verify"));
|
||||
|
||||
assert_eq!(error.reason(), expected, "vector '{name}' must fail as {expected}");
|
||||
rejected += 1;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
rejected, 8,
|
||||
"reject-vectors.json publishes eight challenge vectors; losing one silently narrows the suite"
|
||||
);
|
||||
}
|
||||
|
||||
/// The response reject vectors are artifacts Connect refuses. This side never
|
||||
/// verifies a response, so the device-side statement is the stronger one: given
|
||||
/// the challenge each vector answers, `build_response` must not be capable of
|
||||
/// emitting that artifact in the first place.
|
||||
///
|
||||
/// Each arm pins the specific field a compromised or careless producer would
|
||||
/// have to get wrong, so an implementation that copied values out of the wrong
|
||||
/// place — the response's own document, an operator-supplied argument, a
|
||||
/// previous exchange — fails here.
|
||||
#[test]
|
||||
fn response_reject_vectors_are_artifacts_build_response_cannot_emit() {
|
||||
let key = DeviceIdentity::generate();
|
||||
let mut covered = 0usize;
|
||||
|
||||
for vector in vector_list(&reject_vectors()) {
|
||||
if field(&vector, "artifact") != "response" {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = field(&vector, "name");
|
||||
let refused = signed_document(&vector["document"]);
|
||||
let (_, challenge) = answered_challenge(&vector);
|
||||
let produced_at = unix(field(&refused, "producedAt"));
|
||||
let nonce = device_nonce_of(&vector["document"]);
|
||||
|
||||
let outcome = OfflineEnrollment::build_response(&challenge, &key, &nonce, produced_at);
|
||||
|
||||
match field(&vector["expected"], "reason") {
|
||||
// `responseWindow` in trust-model.json: a device that emits a
|
||||
// response outside the tolerated challenge window has produced an
|
||||
// artifact Connect will refuse, so the refusal belongs here rather
|
||||
// than at the far end of a courier run.
|
||||
"CHALLENGE_EXPIRED" => {
|
||||
let error = outcome.expect_err(&format!("vector '{name}': producing this response must be refused"));
|
||||
assert_eq!(error.reason(), "CHALLENGE_EXPIRED", "vector '{name}' must refuse as CHALLENGE_EXPIRED");
|
||||
covered += 1;
|
||||
continue;
|
||||
}
|
||||
reason => {
|
||||
let built_envelope: Value = serde_json::from_slice(
|
||||
&outcome.unwrap_or_else(|error| panic!("vector '{name}' baseline must build: {}", error.reason())),
|
||||
)
|
||||
.expect("the built response is JSON");
|
||||
let built = signed_document(&built_envelope);
|
||||
|
||||
match reason {
|
||||
"ORGANIZATION_MISMATCH" => {
|
||||
assert_ne!(
|
||||
field(&refused, "organizationName"),
|
||||
challenge.organization_name,
|
||||
"vector '{name}' is only a mismatch if it names another organization"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&built, "organizationName"),
|
||||
challenge.organization_name,
|
||||
"vector '{name}': the organization must come from the challenge, never from elsewhere"
|
||||
);
|
||||
}
|
||||
"CLUSTER_MISMATCH" => {
|
||||
assert_ne!(
|
||||
field(&refused, "clusterName"),
|
||||
challenge.cluster_name,
|
||||
"vector '{name}' is only a mismatch if it names another cluster"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&built, "clusterName"),
|
||||
challenge.cluster_name,
|
||||
"vector '{name}': the cluster must come from the challenge, never from elsewhere"
|
||||
);
|
||||
}
|
||||
"CHALLENGE_PROOF_INVALID" => {
|
||||
// Two distinct vectors land here: a nonce the challenge
|
||||
// never carried, and a proof lifted from another
|
||||
// challenge. Both must be impossible to produce.
|
||||
assert_eq!(
|
||||
field(&built, "challengeNonce"),
|
||||
challenge.nonce,
|
||||
"vector '{name}': the echoed nonce must be the challenge's own"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&built, "challengeProof"),
|
||||
challenge.challenge_proof,
|
||||
"vector '{name}': the proof must be the answered challenge's signature"
|
||||
);
|
||||
assert!(
|
||||
field(&refused, "challengeNonce") != challenge.nonce
|
||||
|| field(&refused, "challengeProof") != challenge.challenge_proof,
|
||||
"vector '{name}' must differ from the challenge in nonce or proof to be rejectable"
|
||||
);
|
||||
}
|
||||
"DEVICE_PROOF_INVALID" => {
|
||||
// The refused vector presents one key and is signed by
|
||||
// another; hold the fixture to that claim, then require
|
||||
// the built response to be the opposite. Proof of
|
||||
// possession is the only thing that makes presenting a
|
||||
// key in an unauthenticated document safe.
|
||||
use p256::ecdsa::signature::Verifier as _;
|
||||
|
||||
let presented = verifying_key(field(&refused, "devicePublicKey"));
|
||||
let raw = BASE64_URL_NO_PAD
|
||||
.decode(field(&vector["document"]["signature"], "value"))
|
||||
.expect("signature is base64url");
|
||||
let signature = p256::ecdsa::Signature::from_slice(&raw).expect("signature parses");
|
||||
assert!(
|
||||
presented
|
||||
.verify(
|
||||
&signing_input(&domain_tag("enrollmentResponse"), &signed_octets(&vector["document"])),
|
||||
&signature
|
||||
)
|
||||
.is_err(),
|
||||
"vector '{name}' is only a possession failure if it does not verify under the key it presents"
|
||||
);
|
||||
|
||||
assert_response_proves_possession(&built_envelope, name);
|
||||
}
|
||||
"UNSUPPORTED_FORMAT" => {
|
||||
assert_ne!(
|
||||
field(&refused, "formatVersion"),
|
||||
field(&built, "formatVersion"),
|
||||
"vector '{name}' is only unsupported if it names another format version"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&built, "formatVersion"),
|
||||
"rustfs.connect.offline.enrollmentResponse/1",
|
||||
"vector '{name}': the format version is frozen"
|
||||
);
|
||||
}
|
||||
"UNSUPPORTED_PROTOCOL" => {
|
||||
assert_ne!(
|
||||
field(&refused, "protocolVersion"),
|
||||
field(&built, "protocolVersion"),
|
||||
"vector '{name}' is only unsupported if it names another protocol major"
|
||||
);
|
||||
assert_eq!(field(&built, "protocolVersion"), "v1", "vector '{name}': the protocol major is frozen");
|
||||
}
|
||||
"ENROLLMENT_REPLAYED" => {
|
||||
// The vector claims to be a byte-identical replay of an
|
||||
// accepted response; hold it to that, because a replay
|
||||
// vector that is not byte identical proves nothing about
|
||||
// single use.
|
||||
let accepted = accept_vector_named("response binding the device public key and the challenge proof");
|
||||
assert_eq!(
|
||||
signed_octets(&vector["document"]),
|
||||
signed_octets(&accepted["document"]),
|
||||
"vector '{name}' must be the accepted response octet for octet"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&vector["document"]["signature"], "value"),
|
||||
field(&accepted["document"]["signature"], "value"),
|
||||
"vector '{name}' must carry the accepted response's signature"
|
||||
);
|
||||
|
||||
// A fresh device nonce is a different artifact, so a
|
||||
// second enrollment is never mistaken for a replay of
|
||||
// the first.
|
||||
let other = OfflineEnrollment::build_response(&challenge, &key, &[0x5a; 32], produced_at)
|
||||
.expect("a second response builds");
|
||||
assert_ne!(
|
||||
signed_octets(&built_envelope),
|
||||
signed_octets(&serde_json::from_slice::<Value>(&other).expect("JSON")),
|
||||
"vector '{name}': a different device nonce must yield a different artifact"
|
||||
);
|
||||
}
|
||||
other => panic!("vector '{name}' names an unhandled reason {other}; extend this test"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
covered += 1;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
covered, 9,
|
||||
"reject-vectors.json publishes nine response vectors; losing one silently narrows the suite"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Signature encoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The high-S malleation is the rejection the whole encoding rule exists for.
|
||||
///
|
||||
/// `(r, n - s)` is a second valid signature over the same document under the
|
||||
/// same key. Every mainstream ECDSA library verifies it, so an implementation
|
||||
/// that hands the decoded octets straight to `p256` accepts a forged-looking
|
||||
/// duplicate of a genuine challenge — and because the 64 octets differ, that
|
||||
/// duplicate is a distinct artifact identity that slips past any deduplication
|
||||
/// keyed on the signature. This test proves the rejection came from the
|
||||
/// encoding rule and not from a failed verification: it first shows the
|
||||
/// malleated signature verifying mathematically, then requires
|
||||
/// `verify_challenge` to refuse it as SIGNATURE_NOT_CANONICAL.
|
||||
#[test]
|
||||
fn malleated_high_s_signature_is_refused_although_it_verifies_mathematically() {
|
||||
use p256::ecdsa::signature::Verifier as _;
|
||||
|
||||
let model = trust_model();
|
||||
let malleated = model["rejectedSignatureEncodings"]
|
||||
.as_array()
|
||||
.expect("trust-model.json publishes rejected encodings")
|
||||
.iter()
|
||||
.find(|entry| field(entry, "reason") == "SIGNATURE_NOT_CANONICAL")
|
||||
.expect("trust-model.json publishes the high-S malleation")
|
||||
.clone();
|
||||
assert!(
|
||||
malleated["acceptedByALenientVerifier"].as_bool() == Some(true),
|
||||
"this vector is only interesting because a lenient verifier accepts it"
|
||||
);
|
||||
|
||||
let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root");
|
||||
let genuine_value = field(&vector["document"]["signature"], "value").to_string();
|
||||
let malleated_value = field(&malleated, "value").to_string();
|
||||
assert_ne!(genuine_value, malleated_value, "the malleation must be a different encoding");
|
||||
|
||||
let genuine = BASE64_URL_NO_PAD.decode(&genuine_value).expect("signature is base64url");
|
||||
let raw = BASE64_URL_NO_PAD.decode(&malleated_value).expect("signature is base64url");
|
||||
assert_eq!(raw.len(), 64, "the malleation is well formed at 64 octets");
|
||||
assert_eq!(raw[..32], genuine[..32], "the malleation shares r with the genuine signature");
|
||||
assert_ne!(raw[32..], genuine[32..], "the malleation replaces s with n - s");
|
||||
|
||||
// Step one: the malleated pair really does verify under the signing key, so
|
||||
// a verifier cannot be excused for accepting it on mathematical grounds.
|
||||
let signature = p256::ecdsa::Signature::from_slice(&raw).expect("the malleated signature parses");
|
||||
assert!(signature.normalize_s().is_some(), "the malleated signature must be the high-S form");
|
||||
let key = verifying_key(field(&published_key("signing"), "publicKey"));
|
||||
let input = signing_input(&domain_tag("enrollmentChallenge"), &signed_octets(&vector["document"]));
|
||||
key.verify(&input, &signature)
|
||||
.expect("the malleated signature must verify mathematically, or this test proves nothing");
|
||||
|
||||
// Step two: the implementation must refuse it anyway, and say why.
|
||||
let mut tampered = vector["document"].clone();
|
||||
tampered["signature"]["value"] = Value::String(malleated_value);
|
||||
|
||||
let now = unix(field(&vector, "evaluationTime"));
|
||||
let error = OfflineEnrollment::verify_challenge(&envelope(&tampered), now)
|
||||
.expect_err("a high-S signature must be refused even though it verifies");
|
||||
assert_eq!(
|
||||
error.reason(),
|
||||
"SIGNATURE_NOT_CANONICAL",
|
||||
"a malleated signature is a canonicality failure, not a verification failure"
|
||||
);
|
||||
}
|
||||
|
||||
/// Every encoding `trust-model.json` names as rejected must fail with the
|
||||
/// reason it names — DER, padded base64url, truncation, and out-of-range
|
||||
/// scalars alongside the malleation. Three of the five are accepted by a
|
||||
/// lenient verifier, so a single blanket "signature did not verify" answer would
|
||||
/// be both wrong and undiagnosable.
|
||||
#[test]
|
||||
fn every_rejected_signature_encoding_fails_with_its_frozen_reason() {
|
||||
let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root");
|
||||
let now = unix(field(&vector, "evaluationTime"));
|
||||
let model = trust_model();
|
||||
let encodings = model["rejectedSignatureEncodings"]
|
||||
.as_array()
|
||||
.expect("trust-model.json publishes rejected encodings");
|
||||
|
||||
for entry in encodings {
|
||||
let name = field(entry, "name");
|
||||
let mut tampered = vector["document"].clone();
|
||||
tampered["signature"]["value"] = Value::String(field(entry, "value").to_string());
|
||||
|
||||
let error: EnrollmentError = OfflineEnrollment::verify_challenge(&envelope(&tampered), now)
|
||||
.err()
|
||||
.unwrap_or_else(|| panic!("rejected encoding '{name}' must not verify"));
|
||||
|
||||
assert_eq!(error.reason(), field(entry, "reason"), "rejected encoding '{name}'");
|
||||
}
|
||||
|
||||
assert_eq!(encodings.len(), 5, "trust-model.json freezes five rejected encodings");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clock window
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The tolerated window is `[issuedAt - 300, expiresAt + 300]`, inclusive at
|
||||
/// both ends. An air-gapped device has no synchronised clock, so an
|
||||
/// off-by-one here either strands a legitimate enrollment or widens the window
|
||||
/// a stolen challenge stays usable in. Both ends are checked at the exact bound
|
||||
/// and one second past it, and the reason distinguishes the two directions.
|
||||
#[test]
|
||||
fn challenge_is_accepted_at_the_exact_skew_bound_and_refused_one_second_past_it() {
|
||||
let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root");
|
||||
let document = envelope(&vector["document"]);
|
||||
let signed = signed_document(&vector["document"]);
|
||||
|
||||
let issued_at = unix(field(&signed, "issuedAt"));
|
||||
let expires_at = unix(field(&signed, "expiresAt"));
|
||||
|
||||
let earliest = issued_at - SKEW_TOLERANCE_SECONDS;
|
||||
OfflineEnrollment::verify_challenge(&document, earliest).expect("the earliest tolerated instant is inside the window");
|
||||
let error =
|
||||
OfflineEnrollment::verify_challenge(&document, earliest - 1).expect_err("one second earlier is outside the window");
|
||||
assert_eq!(error.reason(), "CHALLENGE_NOT_YET_VALID");
|
||||
|
||||
let latest = expires_at + SKEW_TOLERANCE_SECONDS;
|
||||
OfflineEnrollment::verify_challenge(&document, latest).expect("the latest tolerated instant is inside the window");
|
||||
let error = OfflineEnrollment::verify_challenge(&document, latest + 1).expect_err("one second later is outside the window");
|
||||
assert_eq!(error.reason(), "CHALLENGE_EXPIRED");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response production
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Assert a built response proves possession of the key it presents: the
|
||||
/// fingerprint matches the presented key, and the detached signature is a
|
||||
/// canonical low-S ES256 signature that verifies under that key over the exact
|
||||
/// octets transmitted.
|
||||
fn assert_response_proves_possession(built_envelope: &Value, label: &str) {
|
||||
use p256::ecdsa::signature::Verifier as _;
|
||||
|
||||
let raw = signed_octets(built_envelope);
|
||||
let built = signed_document(built_envelope);
|
||||
let signature_block = &built_envelope["signature"];
|
||||
|
||||
assert_eq!(field(signature_block, "algorithm"), "ES256", "{label}: the algorithm is frozen");
|
||||
|
||||
let value = field(signature_block, "value");
|
||||
assert_eq!(value.len(), 86, "{label}: the transfer encoding is 86 unpadded base64url characters");
|
||||
assert!(
|
||||
value.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'),
|
||||
"{label}: the signature must use the base64url alphabet with no padding"
|
||||
);
|
||||
|
||||
let bytes = BASE64_URL_NO_PAD.decode(value).expect("signature is base64url");
|
||||
assert_eq!(bytes.len(), 64, "{label}: the signature is a fixed-width r || s");
|
||||
let signature = p256::ecdsa::Signature::from_slice(&bytes).expect("signature parses");
|
||||
assert!(
|
||||
signature.normalize_s().is_none(),
|
||||
"{label}: this side must never emit the malleated high-S form it refuses to accept"
|
||||
);
|
||||
|
||||
let presented = field(&built, "devicePublicKey");
|
||||
let key = verifying_key(presented);
|
||||
key.verify(&signing_input(&domain_tag("enrollmentResponse"), &raw), &signature)
|
||||
.unwrap_or_else(|error| panic!("{label}: the response must verify under the key it presents: {error}"));
|
||||
|
||||
// `signature.keyIdAlgorithm`: the lowercase SHA-256 of the DER
|
||||
// SubjectPublicKeyInfo, not of the bare point and not of the transfer
|
||||
// encoding.
|
||||
let mut spki = hex_to_bytes(SPKI_PREFIX_HEX);
|
||||
spki.extend_from_slice(&BASE64_URL_NO_PAD.decode(presented).expect("public key is base64url"));
|
||||
let fingerprint = sha256_hex(&spki);
|
||||
assert_eq!(
|
||||
field(&built, "deviceKeyId"),
|
||||
fingerprint,
|
||||
"{label}: deviceKeyId must be the fingerprint of the key the document presents"
|
||||
);
|
||||
assert_eq!(
|
||||
field(signature_block, "keyId"),
|
||||
fingerprint,
|
||||
"{label}: the detached signature must name the same key"
|
||||
);
|
||||
}
|
||||
|
||||
/// A response is the only thing Connect will ever see from this device, so it
|
||||
/// has to carry the whole binding on its own: the challenge it answers, the
|
||||
/// proof that challenge was genuine, the key being enrolled, and possession of
|
||||
/// that key.
|
||||
#[test]
|
||||
fn built_response_binds_the_challenge_proof_and_proves_possession_of_the_device_key() {
|
||||
let vector = accept_vector_named("response binding the device public key and the challenge proof");
|
||||
let (challenge_vector, challenge) = answered_challenge(&vector);
|
||||
let key = DeviceIdentity::generate();
|
||||
let produced_at = unix(field(&signed_document(&vector["document"]), "producedAt"));
|
||||
|
||||
let bytes = OfflineEnrollment::build_response(&challenge, &key, &[0x11; 32], produced_at).expect("the response builds");
|
||||
let built_envelope: Value = serde_json::from_slice(&bytes).expect("the response is JSON");
|
||||
let built = signed_document(&built_envelope);
|
||||
|
||||
assert_response_proves_possession(&built_envelope, "built response");
|
||||
|
||||
// The proof is the challenge's own detached signature. A producer that
|
||||
// echoed the nonce alone, or hashed something, would let a response be
|
||||
// built from an unverified challenge.
|
||||
assert_eq!(
|
||||
field(&built, "challengeProof"),
|
||||
field(&challenge_vector["document"]["signature"], "value"),
|
||||
"the proof must be the signature of the challenge being answered"
|
||||
);
|
||||
assert_eq!(field(&built, "challengeNonce"), challenge.nonce);
|
||||
assert_eq!(field(&built, "challengeId"), challenge.challenge_id);
|
||||
|
||||
assert_eq!(
|
||||
field(&built, "devicePublicKey"),
|
||||
BASE64_URL_NO_PAD.encode(&key.public_key_der()[hex_to_bytes(SPKI_PREFIX_HEX).len()..]),
|
||||
"the presented key must be the key that was passed in"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&built, "deviceNonce"),
|
||||
BASE64_URL_NO_PAD.encode([0x11; 32]),
|
||||
"the device nonce must be the one that was passed in"
|
||||
);
|
||||
assert!(field(&built, "producedAt").ends_with('Z'), "producedAt is a UTC RFC 3339 instant");
|
||||
}
|
||||
|
||||
/// The response leaves the air gap on removable media and is read by anyone who
|
||||
/// handles it. A producer that serialised the key pair instead of the public
|
||||
/// key, or logged a debug rendering into the document, would put the enrolled
|
||||
/// private key on that medium — and the enrollment would still succeed, so
|
||||
/// nothing else in this suite would notice.
|
||||
#[test]
|
||||
fn built_response_carries_no_private_key_material() {
|
||||
let vector = accept_vector_named("response binding the device public key and the challenge proof");
|
||||
let (_, challenge) = answered_challenge(&vector);
|
||||
let key = DeviceIdentity::generate();
|
||||
let produced_at = unix(field(&signed_document(&vector["document"]), "producedAt"));
|
||||
|
||||
let response = OfflineEnrollment::build_response(&challenge, &key, &[0x22; 32], produced_at).expect("the response builds");
|
||||
|
||||
// The envelope carries the signed document base64-encoded, so a needle
|
||||
// present in the document is not present in the envelope octets. Both
|
||||
// layers are searched: an operator handling the medium can read either.
|
||||
let envelope_value: Value = serde_json::from_slice(&response).expect("the response is JSON");
|
||||
let mut haystack = response;
|
||||
haystack.extend_from_slice(&signed_octets(&envelope_value));
|
||||
|
||||
let pkcs8 = key.to_pkcs8_der().expect("serialise the key");
|
||||
let secret = <p256::SecretKey as p256::pkcs8::DecodePrivateKey>::from_pkcs8_der(&pkcs8).expect("the key parses");
|
||||
let scalar = secret.to_bytes();
|
||||
|
||||
// Every spelling the scalar could plausibly reach a document in: raw, and
|
||||
// the three encodings this protocol already uses elsewhere.
|
||||
let scalar_hex: String = scalar.iter().map(|byte| format!("{byte:02x}")).collect();
|
||||
for (description, needle) in [
|
||||
("the PKCS#8 encoding", pkcs8.to_vec()),
|
||||
("the raw private scalar", scalar.to_vec()),
|
||||
("the scalar in base64url", BASE64_URL_NO_PAD.encode(scalar).into_bytes()),
|
||||
("the scalar in standard base64", BASE64_STANDARD.encode(scalar).into_bytes()),
|
||||
("the scalar in hex", scalar_hex.into_bytes()),
|
||||
] {
|
||||
assert!(
|
||||
!haystack.windows(needle.len()).any(|window| window == needle.as_slice()),
|
||||
"the response must not contain {description}"
|
||||
);
|
||||
}
|
||||
|
||||
// The public half must be there, so the absence above is a statement about
|
||||
// what was excluded rather than about a haystack that would not have found
|
||||
// the private half either.
|
||||
let point = BASE64_URL_NO_PAD.encode(&key.public_key_der()[hex_to_bytes(SPKI_PREFIX_HEX).len()..]);
|
||||
assert!(
|
||||
haystack.windows(point.len()).any(|window| window == point.as_bytes()),
|
||||
"the response must still present the public key"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The offline invariant
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The whole surface exists because there is no network. This asserts that
|
||||
/// three different ways, because no single one of them is conclusive on its own.
|
||||
///
|
||||
/// 1. The process opens no descriptor across a full verify-and-respond cycle. A
|
||||
/// socket, a DNS resolver, a pooled HTTP client, or a revocation-list fetch
|
||||
/// all show up here — including one that is opened and cached rather than
|
||||
/// opened and closed, which is what a lazily built client does.
|
||||
/// 2. The cycle is a pure byte transform: the same inputs produce the same
|
||||
/// verified fields, and the evaluation instant is an argument rather than an
|
||||
/// ambient read, so nothing about the outcome can depend on reachability.
|
||||
/// 3. Repeating the cycle changes nothing observable, so a first call cannot be
|
||||
/// quietly initialising shared state that a later one reuses.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn enrollment_opens_no_descriptor_and_is_a_pure_byte_transform() {
|
||||
let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root");
|
||||
let document = envelope(&vector["document"]);
|
||||
let now = unix(field(&vector, "evaluationTime"));
|
||||
let key = DeviceIdentity::generate();
|
||||
|
||||
// Warm anything the test harness itself lazily opens before the baseline.
|
||||
let _ = open_descriptors();
|
||||
let baseline = open_descriptors();
|
||||
assert!(
|
||||
!baseline.is_empty(),
|
||||
"the descriptor table must be readable for this test to mean anything"
|
||||
);
|
||||
|
||||
let mut fields = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let challenge = OfflineEnrollment::verify_challenge(&document, now).expect("the challenge verifies");
|
||||
let response = OfflineEnrollment::build_response(&challenge, &key, &[0x33; 32], now).expect("the response builds");
|
||||
fields.push((
|
||||
challenge.challenge_id.clone(),
|
||||
challenge.nonce.clone(),
|
||||
challenge.challenge_proof.clone(),
|
||||
signed_octets(&serde_json::from_slice::<Value>(&response).expect("JSON")),
|
||||
));
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
open_descriptors(),
|
||||
baseline,
|
||||
"the enrollment path must not open a descriptor: no socket, no resolver, no cached client"
|
||||
);
|
||||
|
||||
let (first, second) = (&fields[0], &fields[1]);
|
||||
assert_eq!(first.0, second.0, "verification must be deterministic");
|
||||
assert_eq!(first.1, second.1, "verification must be deterministic");
|
||||
assert_eq!(first.2, second.2, "verification must be deterministic");
|
||||
assert_eq!(
|
||||
first.3, second.3,
|
||||
"the signed response octets are a function of the challenge, the key, the nonce, and the instant"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn open_descriptors() -> Vec<String> {
|
||||
// Linux publishes the table at /proc/self/fd; the BSDs and macOS at /dev/fd.
|
||||
let path = if PathBuf::from("/proc/self/fd").is_dir() {
|
||||
"/proc/self/fd"
|
||||
} else {
|
||||
"/dev/fd"
|
||||
};
|
||||
|
||||
let mut entries: Vec<String> = fs::read_dir(path)
|
||||
.unwrap_or_else(|error| panic!("read {path}: {error}"))
|
||||
.map(|entry| entry.expect("read dir entry").file_name().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
entries.sort();
|
||||
entries
|
||||
}
|
||||
|
||||
/// A descriptor count taken around a call cannot see a socket that was opened
|
||||
/// and closed inside it, so the invariant is also asserted where it can be
|
||||
/// stated absolutely: the implementation names no network API at all.
|
||||
///
|
||||
/// This is the shape the regression actually takes — someone adds a
|
||||
/// revocation-list fetch, a time-server check, or a "just confirm the challenge
|
||||
/// with Connect" call — and it is caught at the source rather than by observing
|
||||
/// its effects.
|
||||
#[test]
|
||||
fn enrollment_implementation_names_no_network_api() {
|
||||
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/connect/offline/enrollment.rs");
|
||||
let source = fs::read_to_string(&path).unwrap_or_else(|error| panic!("read {}: {error}", path.display()));
|
||||
|
||||
// Prose is allowed to discuss the invariant it is documenting, so only code
|
||||
// is scanned.
|
||||
let code: String = source
|
||||
.lines()
|
||||
.filter(|line| !line.trim_start().starts_with("//"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
for forbidden in [
|
||||
"std::net",
|
||||
"tokio::net",
|
||||
"TcpStream",
|
||||
"TcpListener",
|
||||
"UdpSocket",
|
||||
"UnixStream",
|
||||
"ToSocketAddrs",
|
||||
"reqwest",
|
||||
"hyper",
|
||||
"tonic",
|
||||
] {
|
||||
assert!(
|
||||
!code.contains(forbidden),
|
||||
"offline enrollment must not reach the network, but the implementation names {forbidden}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Executable
+419
@@ -0,0 +1,419 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail when committed tests silently fall out of their execution wiring."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import unittest
|
||||
from unittest import mock
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def words(value: str) -> set[str]:
|
||||
return {item.strip() for item in value.split(",") if item.strip()}
|
||||
|
||||
|
||||
def rust_code_only(source: str) -> str:
|
||||
"""Blank Rust comments and literals while preserving byte positions."""
|
||||
def quoted_end(quote_index: int, delimiter: str) -> int:
|
||||
end = quote_index + 1
|
||||
while end < len(source):
|
||||
if source[end] == "\\":
|
||||
end += 2
|
||||
elif source[end] == delimiter:
|
||||
return end + 1
|
||||
else:
|
||||
end += 1
|
||||
return end
|
||||
|
||||
code = list(source)
|
||||
index = 0
|
||||
while index < len(source):
|
||||
if source.startswith("//", index):
|
||||
end = source.find("\n", index)
|
||||
end = len(source) if end < 0 else end
|
||||
elif source.startswith("/*", index):
|
||||
depth = 1
|
||||
end = index + 2
|
||||
while end < len(source) and depth:
|
||||
if source.startswith("/*", end):
|
||||
depth += 1
|
||||
end += 2
|
||||
elif source.startswith("*/", end):
|
||||
depth -= 1
|
||||
end += 2
|
||||
else:
|
||||
end += 1
|
||||
else:
|
||||
raw = re.match(r'(?:br|cr|r)(?P<hashes>#{0,})"', source[index:])
|
||||
if raw:
|
||||
marker = '"' + raw.group("hashes")
|
||||
end = source.find(marker, index + raw.end())
|
||||
end = len(source) if end < 0 else end + len(marker)
|
||||
elif source[index] == '"' or source.startswith(('b"', 'c"'), index):
|
||||
start_quote = index if source[index] == '"' else index + 1
|
||||
end = quoted_end(start_quote, '"')
|
||||
elif source[index] == "'" and index + 2 < len(source) and (
|
||||
source[index + 1] == "\\" or source[index + 2] == "'"
|
||||
):
|
||||
end = quoted_end(index, "'")
|
||||
elif source.startswith("b'", index):
|
||||
end = quoted_end(index + 1, "'")
|
||||
else:
|
||||
index += 1
|
||||
continue
|
||||
|
||||
for offset in range(index, end):
|
||||
if code[offset] != "\n":
|
||||
code[offset] = " "
|
||||
index = end
|
||||
return "".join(code)
|
||||
|
||||
|
||||
def declared(parent: Path, module: str) -> bool:
|
||||
source = parent.read_text()
|
||||
code = rust_code_only(source)
|
||||
pattern = re.compile(rf"^\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+{re.escape(module)}\s*;", re.MULTILINE)
|
||||
allowed_preambles = {
|
||||
"#[cfg(test)]": "#[cfg(test)]",
|
||||
"#[cfg(all(test,target_os=))]": '#[cfg(all(test,target_os="linux"))]',
|
||||
}
|
||||
for match in pattern.finditer(code):
|
||||
prefix = code[: match.start()]
|
||||
depths = {"(": 0, "[": 0, "{": 0}
|
||||
pairs = {")": "(", "]": "[", "}": "{"}
|
||||
for char in prefix:
|
||||
if char in depths:
|
||||
depths[char] += 1
|
||||
elif char in pairs:
|
||||
depths[pairs[char]] -= 1
|
||||
if any(depths.values()):
|
||||
continue
|
||||
|
||||
boundary = max(prefix.rfind(";"), prefix.rfind("{"), prefix.rfind("}"))
|
||||
code_preamble = re.sub(r"\s+", "", prefix[boundary + 1 :])
|
||||
if not code_preamble:
|
||||
return True
|
||||
if code_preamble in allowed_preambles:
|
||||
attr_start = prefix.rfind("#[cfg", boundary + 1)
|
||||
if attr_start >= 0 and re.sub(r"\s+", "", source[attr_start : match.start()]) == allowed_preambles[code_preamble]:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def module_source(src: Path, directory: Path) -> Path | None:
|
||||
if not directory.parts:
|
||||
return src / "lib.rs"
|
||||
|
||||
mod_file = src / directory / "mod.rs"
|
||||
if mod_file.is_file():
|
||||
return mod_file
|
||||
|
||||
sibling = src.joinpath(*directory.parts[:-1], f"{directory.name}.rs")
|
||||
return sibling if sibling.is_file() else None
|
||||
|
||||
|
||||
def check_e2e_modules(root: Path) -> list[str]:
|
||||
src = root / "crates/e2e_test/src"
|
||||
errors: list[str] = []
|
||||
for test_file in sorted(src.rglob("*_test.rs")):
|
||||
relative = test_file.relative_to(root).as_posix()
|
||||
directory = test_file.relative_to(src).parent
|
||||
parent = module_source(src, directory)
|
||||
if parent is None:
|
||||
errors.append(f"{relative}: no canonical parent module")
|
||||
continue
|
||||
|
||||
if not declared(parent, test_file.stem):
|
||||
errors.append(f"{relative}: not declared by {parent.relative_to(root).as_posix()}")
|
||||
while directory.parts:
|
||||
module = directory.name
|
||||
directory = directory.parent
|
||||
parent = module_source(src, directory)
|
||||
if parent is None:
|
||||
errors.append(f"{relative}: module {module} has no canonical parent")
|
||||
break
|
||||
if not declared(parent, module):
|
||||
errors.append(f"{relative}: module {module} not declared by {parent.relative_to(root).as_posix()}")
|
||||
return errors
|
||||
|
||||
|
||||
def check_fuzz_targets(root: Path) -> list[str]:
|
||||
manifest = tomllib.loads((root / "fuzz/Cargo.toml").read_text())
|
||||
expected = {item["name"] for item in manifest.get("bin", []) if "name" in item}
|
||||
errors: list[str] = []
|
||||
if not expected:
|
||||
return ["fuzz/Cargo.toml: no [[bin]] fuzz targets found"]
|
||||
|
||||
runner = (root / "scripts/fuzz/run.sh").read_text()
|
||||
match = re.search(r'^targets="([^"]+)"', runner, re.MULTILINE)
|
||||
runner_targets = set(match.group(1).split()) if match else set()
|
||||
if runner_targets != expected:
|
||||
errors.append(f"scripts/fuzz/run.sh targets {sorted(runner_targets)} != manifest {sorted(expected)}")
|
||||
|
||||
workflow = (root / ".github/workflows/fuzz.yml").read_text()
|
||||
matrices = [words(value) for value in re.findall(r"^\s*target:\s*\[([^]]+)]", workflow, re.MULTILINE)]
|
||||
if len(matrices) != 2:
|
||||
errors.append(f".github/workflows/fuzz.yml: expected smoke and nightly target matrices, found {len(matrices)}")
|
||||
for index, matrix in enumerate(matrices, start=1):
|
||||
if matrix != expected:
|
||||
errors.append(f".github/workflows/fuzz.yml matrix {index} {sorted(matrix)} != manifest {sorted(expected)}")
|
||||
|
||||
runtime_targets = re.findall(r"^\s*FUZZ_TARGET:\s*(\S.*?)\s*$", workflow, re.MULTILINE)
|
||||
if runtime_targets != ["${{ matrix.target }}", "${{ matrix.target }}"]:
|
||||
errors.append(".github/workflows/fuzz.yml: smoke and nightly jobs must pass matrix.target to FUZZ_TARGET")
|
||||
|
||||
dependency_paths = {
|
||||
f"{path.removeprefix('../')}/**"
|
||||
for dependency in manifest.get("dependencies", {}).values()
|
||||
if isinstance(dependency, dict)
|
||||
and isinstance(path := dependency.get("path"), str)
|
||||
and path.startswith("../crates/")
|
||||
}
|
||||
missing_paths = sorted(path for path in dependency_paths if f'"{path}"' not in workflow)
|
||||
if missing_paths:
|
||||
errors.append(f".github/workflows/fuzz.yml missing direct dependency paths: {', '.join(missing_paths)}")
|
||||
|
||||
staged_matches = re.findall(r"^\s*for target in ([^;]+); do", workflow, re.MULTILINE)
|
||||
staged = set(staged_matches[0].split()) if staged_matches else set()
|
||||
if staged != expected:
|
||||
errors.append(f".github/workflows/fuzz.yml staged binaries {sorted(staged)} != manifest {sorted(expected)}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def check_runner_selection(root: Path) -> list[str]:
|
||||
runner = (root / "scripts/run_e2e_tests.sh").read_text()
|
||||
errors: list[str] = []
|
||||
if "--include-ignored" not in runner:
|
||||
errors.append("scripts/run_e2e_tests.sh: runner must include default and ignored tests")
|
||||
if "--test-threads=1" not in runner:
|
||||
errors.append("scripts/run_e2e_tests.sh: runner must serialize fixed-port protocol tests")
|
||||
if re.search(r"(?<!include-)--ignored\b", runner):
|
||||
errors.append("scripts/run_e2e_tests.sh: bare --ignored silently excludes default tests")
|
||||
if "--exact" in runner:
|
||||
errors.append("scripts/run_e2e_tests.sh: --test is documented as a pattern and must not force exact matching")
|
||||
if 'eval "$test_cmd"' in runner:
|
||||
errors.append("scripts/run_e2e_tests.sh: command construction must not use eval")
|
||||
return errors
|
||||
|
||||
|
||||
def profile_selection(root: Path, profile: str) -> str:
|
||||
if not re.fullmatch(r"e2e-[a-z0-9-]+", profile):
|
||||
raise ValueError(f"invalid e2e profile name: {profile}")
|
||||
path = root / f".config/{profile}-selection.txt"
|
||||
lines = [line for line in path.read_text().splitlines() if line.strip()]
|
||||
values = dict(line.split("=", 1) for line in lines if "=" in line)
|
||||
if len(values) != len(lines) or any(not re.fullmatch(r"sha256(?:-[a-z0-9]+)?", key) for key in values):
|
||||
raise ValueError(f"{path.relative_to(root).as_posix()}: invalid sha256 entry")
|
||||
key = f"sha256-{sys.platform}"
|
||||
digest = values.get(key, values.get("sha256", ""))
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", digest):
|
||||
raise ValueError(f"{path.relative_to(root).as_posix()}: missing sha256 for {sys.platform}")
|
||||
return digest
|
||||
|
||||
|
||||
def check_profile_definitions(root: Path) -> list[str]:
|
||||
config = tomllib.loads((root / ".config/nextest.toml").read_text())
|
||||
profiles = {
|
||||
profile
|
||||
for profile in config.get("profile", {})
|
||||
if profile.startswith("e2e-")
|
||||
}
|
||||
selection_profiles = {
|
||||
path.name.removesuffix("-selection.txt") for path in (root / ".config").glob("e2e-*-selection.txt")
|
||||
}
|
||||
errors: list[str] = []
|
||||
for profile in sorted(profiles | selection_profiles):
|
||||
if profile not in profiles:
|
||||
errors.append(f".config/nextest.toml: missing profile.{profile}")
|
||||
if profile not in selection_profiles:
|
||||
errors.append(f".config/{profile}-selection.txt: missing expected profile selection")
|
||||
continue
|
||||
try:
|
||||
profile_selection(root, profile)
|
||||
except (FileNotFoundError, ValueError) as error:
|
||||
errors.append(str(error))
|
||||
return errors
|
||||
|
||||
|
||||
def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]:
|
||||
try:
|
||||
expected_digest = profile_selection(root, profile)
|
||||
data = json.loads(listing.read_text())
|
||||
selected = sorted(
|
||||
f"{suite_id}::{test_name}"
|
||||
for suite_id, suite in data["rust-suites"].items()
|
||||
for test_name, testcase in suite["testcases"].items()
|
||||
if testcase.get("filter-match", {}).get("status") == "matches"
|
||||
)
|
||||
digest = hashlib.sha256(("\n".join(selected) + "\n").encode()).hexdigest()
|
||||
except (FileNotFoundError, KeyError, TypeError, ValueError, json.JSONDecodeError) as error:
|
||||
return [f"cannot read {profile} nextest listing: {error}"]
|
||||
if digest != expected_digest:
|
||||
return [
|
||||
f"{profile} selection changed: count={len(selected)} sha256={digest}; "
|
||||
f"expected sha256={expected_digest}"
|
||||
]
|
||||
print(f"{profile} selection OK: {len(selected)} tests, sha256={digest}")
|
||||
return []
|
||||
|
||||
|
||||
def validate(root: Path) -> list[str]:
|
||||
errors: list[str] = []
|
||||
errors.extend(check_e2e_modules(root))
|
||||
errors.extend(check_fuzz_targets(root))
|
||||
errors.extend(check_runner_selection(root))
|
||||
errors.extend(check_profile_definitions(root))
|
||||
return errors
|
||||
|
||||
|
||||
class SelfTests(unittest.TestCase):
|
||||
def test_e2e_requires_registration(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
src = root / "crates/e2e_test/src"
|
||||
src.mkdir(parents=True)
|
||||
(src / "lib.rs").write_text("")
|
||||
test_file = src / "boundary_test.rs"
|
||||
test_file.write_text("#[test]\nfn boundary() {}\n")
|
||||
self.assertEqual(len(check_e2e_modules(root)), 1)
|
||||
(src / "lib.rs").write_text("mod boundary_test;\n")
|
||||
self.assertEqual(check_e2e_modules(root), [])
|
||||
|
||||
nested = src / "protocols"
|
||||
nested.mkdir()
|
||||
(nested / "mod.rs").write_text("mod fixed_port_test;\n")
|
||||
(nested / "fixed_port_test.rs").write_text("#[test]\nfn fixed_port() {}\n")
|
||||
self.assertEqual(len(check_e2e_modules(root)), 1)
|
||||
(src / "lib.rs").write_text("mod boundary_test;\nmod protocols;\n")
|
||||
self.assertEqual(check_e2e_modules(root), [])
|
||||
|
||||
(src / "lib.rs").write_text("#[cfg(any())]\nmod boundary_test;\nmod protocols;\n")
|
||||
self.assertEqual(len(check_e2e_modules(root)), 1)
|
||||
|
||||
(src / "lib.rs").write_text(
|
||||
"#[cfg(any())]\n/// hidden module\nmod boundary_test;\n#[cfg_attr(test, cfg(any()))]\nmod protocols;\n"
|
||||
)
|
||||
self.assertEqual(len(check_e2e_modules(root)), 2)
|
||||
|
||||
(src / "lib.rs").write_text(
|
||||
'const PHANTOM: &str = r#"{\nmod boundary_test;\n"#;\ndiscard! { mod protocols; }\n'
|
||||
)
|
||||
self.assertEqual(len(check_e2e_modules(root)), 2)
|
||||
|
||||
(src / "lib.rs").write_text(
|
||||
'#[cfg(all(test, target_os = r"windows" /* target_os = "linux" */))]\n'
|
||||
"mod boundary_test;\nmod protocols;\n"
|
||||
)
|
||||
self.assertEqual(len(check_e2e_modules(root)), 1)
|
||||
|
||||
(src / "lib.rs").write_text(
|
||||
'#[cfg(all(test, target_os = r"windows"))] // #[cfg(all(test, target_os = "linux"))]\n'
|
||||
"mod boundary_test;\nmod protocols;\n"
|
||||
)
|
||||
self.assertEqual(len(check_e2e_modules(root)), 1)
|
||||
|
||||
def test_fuzz_runtime_uses_matrix_target(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "fuzz").mkdir()
|
||||
(root / "scripts/fuzz").mkdir(parents=True)
|
||||
(root / ".github/workflows").mkdir(parents=True)
|
||||
(root / "fuzz/Cargo.toml").write_text(
|
||||
'dep = { path = "../crates/dep" }\n[[bin]]\nname = "one"\n'
|
||||
)
|
||||
(root / "scripts/fuzz/run.sh").write_text('targets="one"\n')
|
||||
(root / ".github/workflows/fuzz.yml").write_text(
|
||||
'paths:\n - "crates/dep/**"\n'
|
||||
"target: [one]\nFUZZ_TARGET: fixed\n"
|
||||
"target: [one]\nFUZZ_TARGET: ${{ matrix.target }}\n"
|
||||
"for target in one; do\n"
|
||||
" fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/one\n"
|
||||
)
|
||||
self.assertEqual(len(check_fuzz_targets(root)), 1)
|
||||
|
||||
def test_profile_listing_enforces_selection(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / ".config").mkdir()
|
||||
digest = hashlib.sha256(b"suite::two\n").hexdigest()
|
||||
(root / ".config/e2e-smoke-selection.txt").write_text(f"sha256={digest}\n")
|
||||
listing = root / "listing.json"
|
||||
listing.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"rust-suites": {
|
||||
"suite": {
|
||||
"testcases": {
|
||||
"one": {"filter-match": {"status": "matches"}},
|
||||
"two": {"filter-match": {"status": "mismatch"}},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
self.assertEqual(len(check_profile_listing(root, "e2e-smoke", listing)), 1)
|
||||
|
||||
def test_profile_listing_binds_platform_digest(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / ".config").mkdir()
|
||||
darwin_digest = hashlib.sha256(b"suite::darwin\n").hexdigest()
|
||||
linux_digest = hashlib.sha256(b"suite::linux\n").hexdigest()
|
||||
(root / ".config/e2e-full-selection.txt").write_text(
|
||||
f"sha256-darwin={darwin_digest}\nsha256-linux={linux_digest}\n"
|
||||
)
|
||||
listing = root / "listing.json"
|
||||
listing.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"rust-suites": {
|
||||
"suite": {
|
||||
"testcases": {"darwin": {"filter-match": {"status": "matches"}}}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
with mock.patch.object(sys, "platform", "linux"):
|
||||
self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1)
|
||||
|
||||
def main() -> int:
|
||||
if sys.argv[1:] == ["--self-test"]:
|
||||
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
|
||||
return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
|
||||
if len(sys.argv) == 4 and sys.argv[1] == "--check-profile":
|
||||
errors = check_profile_listing(ROOT, sys.argv[2], Path(sys.argv[3]))
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
if sys.argv[1:]:
|
||||
print(
|
||||
"usage: check_test_wiring.py [--self-test | --check-profile PROFILE LISTING]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
errors = validate(ROOT)
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
print("OK: e2e modules, runner selection, fuzz matrices, and profile guards are wired")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+14
-11
@@ -20,6 +20,8 @@ DATA_DIR="$TARGET_DIR/rustfs_test_data"
|
||||
RUSTFS_PID=""
|
||||
TEST_FILTER=""
|
||||
TEST_TYPE="all"
|
||||
RUSTFS_BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-ftps,webdav,sftp}"
|
||||
export RUSTFS_BUILD_FEATURES
|
||||
|
||||
# Function to print colored output
|
||||
print_info() {
|
||||
@@ -92,7 +94,7 @@ build_rustfs() {
|
||||
print_info "Building RustFS..."
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
if ! cargo build --bin rustfs; then
|
||||
if ! cargo build --bin rustfs --features "$RUSTFS_BUILD_FEATURES"; then
|
||||
print_error "Failed to build RustFS"
|
||||
exit 1
|
||||
fi
|
||||
@@ -219,27 +221,28 @@ start_rustfs() {
|
||||
run_tests() {
|
||||
print_info "Running e2e tests..."
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
local test_cmd="cargo test --package e2e_test --lib"
|
||||
|
||||
|
||||
local test_cmd=(cargo test --package e2e_test --lib)
|
||||
|
||||
case "$TEST_TYPE" in
|
||||
"specific")
|
||||
test_cmd="$test_cmd -- $TEST_FILTER --exact --show-output --ignored"
|
||||
test_cmd+=(-- "$TEST_FILTER")
|
||||
print_info "Running specific test: $TEST_FILTER"
|
||||
;;
|
||||
"file")
|
||||
test_cmd="$test_cmd -- $TEST_FILTER --show-output --ignored"
|
||||
test_cmd+=(-- "$TEST_FILTER")
|
||||
print_info "Running tests in file/module: $TEST_FILTER"
|
||||
;;
|
||||
"all")
|
||||
test_cmd="$test_cmd -- --show-output --ignored"
|
||||
test_cmd+=(--)
|
||||
print_info "Running all e2e tests"
|
||||
;;
|
||||
esac
|
||||
|
||||
print_info "Test command: $test_cmd"
|
||||
|
||||
if eval "$test_cmd"; then
|
||||
test_cmd+=(--show-output --include-ignored --test-threads=1)
|
||||
|
||||
print_info "Test command: ${test_cmd[*]}"
|
||||
|
||||
if "${test_cmd[@]}"; then
|
||||
print_success "All tests passed!"
|
||||
return 0
|
||||
else
|
||||
|
||||
@@ -253,6 +253,9 @@ Test results are saved in the `artifacts/s3tests-${TEST_MODE}/` directory (defau
|
||||
|
||||
- `junit.xml`: Test results in JUnit format (compatible with CI/CD systems)
|
||||
- `pytest.log`: Detailed pytest logs with full test output
|
||||
- `all-collected-nodeids.txt`: Exact node IDs in the pinned upstream suite
|
||||
- `selected-nodeids.txt`: Exact node IDs expected in this run
|
||||
- `unsharded-selected-nodeids.txt`: Exact node IDs before deterministic sharding
|
||||
- `compat-report.md`: Classification report generated by `report_compat.py` —
|
||||
regressions against `implemented_tests.txt`, promotion candidates (tests
|
||||
that pass but are still listed as unimplemented/excluded), and tests missing
|
||||
@@ -449,9 +452,11 @@ RustFS. Two GitHub Actions workflows delegate to it:
|
||||
- **Full sweep** (`.github/workflows/e2e-s3tests.yml`): weekly scheduled (and
|
||||
manually dispatchable) run of the ENTIRE upstream suite (`TEST_SCOPE=all`)
|
||||
against a Docker deployment — single node or a real 4-node distributed
|
||||
cluster behind HAProxy. The sweep fails only on regressions in the
|
||||
implemented whitelist; everything else is reported by `report_compat.py`
|
||||
as promotion candidates or unclassified tests.
|
||||
cluster behind HAProxy. Regressions, unclassified tests, incomplete
|
||||
execution, and infrastructure errors fail the sweep; classified unsupported
|
||||
behavior remains informational. Scheduled topology runs are split into four
|
||||
deterministic exact-node-ID shards, and every case has a five-minute timeout,
|
||||
so one stalled case cannot erase the entire sweep's evidence.
|
||||
|
||||
Keeping both workflows on this script means local runs, the PR gate, and the
|
||||
scheduled sweep always execute tests the same way (same pinned s3-tests
|
||||
@@ -466,8 +471,9 @@ pass/fail table in the job summary.
|
||||
## Companion Tools
|
||||
|
||||
- `report_compat.py` — diffs a junit.xml result against the classification
|
||||
lists; run automatically at the end of `run.sh`, and used by the weekly
|
||||
sweep to gate on whitelist regressions only (`--fail-on-regression`).
|
||||
lists and the exact pytest collection; run before execution to reject stale
|
||||
or missing classifications, then after execution to detect regressions and
|
||||
incomplete parameterized cases.
|
||||
- `api_coverage.py` — quantifies S3 API surface coverage by comparing the
|
||||
s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods
|
||||
RustFS overrides in `impl S3 for FS`:
|
||||
|
||||
@@ -307,3 +307,10 @@ test_object_acl_write
|
||||
test_object_acl_writeacp
|
||||
test_put_bucket_acl_grant_group_read
|
||||
test_object_raw_get_bucket_acl
|
||||
|
||||
# Require upstream cloud-storage or IAM account services
|
||||
test_bucket_logging_requester_assumed_role
|
||||
test_lifecycle_cloud_transition_target_by_bucket
|
||||
test_lifecycle_cloud_transition_target_by_bucket_multiple_buckets
|
||||
test_list_object_versions_restore_status
|
||||
test_list_objects_restore_status
|
||||
|
||||
@@ -521,9 +521,13 @@ test_atomic_dual_conditional_write_1mb
|
||||
test_atomic_write_bucket_gone
|
||||
test_bucket_acl_canned_private_to_private
|
||||
test_bucket_concurrent_set_canned_acl
|
||||
test_bucket_create_delete
|
||||
test_bucket_policy
|
||||
test_bucket_policy_acl
|
||||
test_bucket_policy_put_obj_acl
|
||||
test_bucketv2_policy_acl
|
||||
test_copy_enc
|
||||
test_copy_part_enc
|
||||
test_copy_object_ifmatch_failed
|
||||
test_copy_object_ifnonematch_good
|
||||
test_cors_presigned_put_object_tenant_with_acl
|
||||
|
||||
@@ -21,14 +21,17 @@ Classifies every executed test into:
|
||||
- unclassified passes: passed but not present in any list (new upstream tests)
|
||||
- unclassified failures: failed and not present in any list (new upstream tests)
|
||||
|
||||
Writes a markdown report and prints a summary to stdout. Exit code is 0 unless
|
||||
--fail-on-regression is given and at least one regression was found.
|
||||
Writes a markdown report and prints a summary to stdout. Optional gates reject
|
||||
regressions, unclassified tests, stale classifications, and incomplete node-ID
|
||||
execution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
@@ -45,33 +48,47 @@ LIST_FILES = {
|
||||
}
|
||||
|
||||
|
||||
def load_list(path: pathlib.Path) -> set[str]:
|
||||
names: set[str] = set()
|
||||
def load_entries(path: pathlib.Path) -> list[str]:
|
||||
names: list[str] = []
|
||||
if not path.is_file():
|
||||
return names
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
names.add(line)
|
||||
names.append(line)
|
||||
return names
|
||||
|
||||
|
||||
def classification_errors(entries: dict[str, list[str]]) -> list[str]:
|
||||
errors: list[str] = []
|
||||
lists = {key: set(names) for key, names in entries.items()}
|
||||
for key, names in entries.items():
|
||||
duplicates = sorted(name for name, count in Counter(names).items() if count > 1)
|
||||
if duplicates:
|
||||
errors.append(f"{LIST_FILES[key]} has duplicates: {', '.join(duplicates)}")
|
||||
keys = tuple(lists)
|
||||
for index, left in enumerate(keys):
|
||||
for right in keys[index + 1 :]:
|
||||
overlap = sorted(lists[left] & lists[right])
|
||||
if overlap:
|
||||
errors.append(f"{left}/{right} classifications overlap: {', '.join(overlap)}")
|
||||
return errors
|
||||
|
||||
|
||||
def base_name(testcase_name: str) -> str:
|
||||
"""Strip pytest parametrization (test_foo[param]) to match list entries."""
|
||||
return testcase_name.split("[", 1)[0]
|
||||
|
||||
|
||||
def parse_junit(path: pathlib.Path) -> dict[str, str]:
|
||||
"""Return {test name: status} with status in passed/failed/error/skipped.
|
||||
|
||||
Parametrized cases collapse onto their base name; any failing variant marks
|
||||
the whole test failed.
|
||||
"""
|
||||
def parse_junit(path: pathlib.Path) -> tuple[dict[str, str], list[str], list[tuple[str, str, str, str]]]:
|
||||
"""Return exact statuses, pytest-timeout cases, and failure summaries."""
|
||||
results: dict[str, str] = {}
|
||||
timed_out: list[str] = []
|
||||
failures: list[tuple[str, str, str, str]] = []
|
||||
severity = {"skipped": 0, "passed": 1, "failed": 2, "error": 2}
|
||||
root = ET.parse(path).getroot()
|
||||
for case in root.iter("testcase"):
|
||||
name = base_name(case.get("name", ""))
|
||||
name = case.get("name", "")
|
||||
if not name:
|
||||
continue
|
||||
if case.find("failure") is not None:
|
||||
@@ -85,7 +102,35 @@ def parse_junit(path: pathlib.Path) -> dict[str, str]:
|
||||
prev = results.get(name)
|
||||
if prev is None or severity[status] > severity[prev]:
|
||||
results[name] = status
|
||||
return results
|
||||
node = case.find("failure") if status == "failed" else case.find("error")
|
||||
if node is not None:
|
||||
details = " ".join(filter(None, [node.get("message", ""), node.text or ""]))
|
||||
message = node.get("message") or next(iter((node.text or "").strip().splitlines()), "")
|
||||
failures.append((case.get("classname", ""), name, case.get("time", "0"), message))
|
||||
if re.search(r"\bTimeout\s*(?:>|\()", details, re.IGNORECASE):
|
||||
timed_out.append(name)
|
||||
return results, timed_out, failures
|
||||
|
||||
|
||||
def collapse_results(results: dict[str, str]) -> dict[str, str]:
|
||||
"""Collapse parametrized cases for classification-level reporting."""
|
||||
collapsed: dict[str, str] = {}
|
||||
severity = {"skipped": 0, "passed": 1, "failed": 2, "error": 2}
|
||||
for exact_name, status in results.items():
|
||||
name = base_name(exact_name)
|
||||
previous = collapsed.get(name)
|
||||
if previous is None or severity[status] > severity[previous]:
|
||||
collapsed[name] = status
|
||||
return collapsed
|
||||
|
||||
|
||||
def load_collected_nodeids(path: pathlib.Path) -> set[str]:
|
||||
names: set[str] = set()
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
nodeid = line.strip()
|
||||
if nodeid:
|
||||
names.add(nodeid.rsplit("::", 1)[-1])
|
||||
return names
|
||||
|
||||
|
||||
def render_section(title: str, rows: list[str], hint: str = "") -> list[str]:
|
||||
@@ -102,7 +147,7 @@ def render_section(title: str, rows: list[str], hint: str = "") -> list[str]:
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--junit", required=True, type=pathlib.Path, help="junit.xml produced by pytest")
|
||||
parser.add_argument("--junit", type=pathlib.Path, help="junit.xml produced by pytest")
|
||||
parser.add_argument(
|
||||
"--lists-dir",
|
||||
type=pathlib.Path,
|
||||
@@ -115,14 +160,60 @@ def main() -> int:
|
||||
action="store_true",
|
||||
help="exit non-zero when a test from implemented_tests.txt failed",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fail-on-unclassified",
|
||||
action="store_true",
|
||||
help="exit non-zero when an executed test is absent from every classification",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--collected-nodeids",
|
||||
type=pathlib.Path,
|
||||
help="exact pytest node IDs from the pinned suite's collect-only pass",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check-classifications-only",
|
||||
action="store_true",
|
||||
help="validate classification names against collected node IDs without reading JUnit",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.junit.is_file():
|
||||
entries = {key: load_entries(args.lists_dir / fname) for key, fname in LIST_FILES.items()}
|
||||
lists = {key: set(names) for key, names in entries.items()}
|
||||
invalid_classifications = classification_errors(entries)
|
||||
collected: set[str] = set()
|
||||
if args.collected_nodeids:
|
||||
collected = load_collected_nodeids(args.collected_nodeids)
|
||||
collected_base = {base_name(name) for name in collected}
|
||||
classified = set().union(*lists.values())
|
||||
missing_classifications = sorted(collected_base - classified)
|
||||
stale_classifications = sorted(classified - collected_base)
|
||||
else:
|
||||
missing_classifications = []
|
||||
stale_classifications = []
|
||||
|
||||
if args.check_classifications_only:
|
||||
if not args.collected_nodeids:
|
||||
parser.error("--check-classifications-only requires --collected-nodeids")
|
||||
for error in invalid_classifications:
|
||||
print(f"[INVALID] {error}")
|
||||
for name in missing_classifications:
|
||||
print(f"[UNCLASSIFIED] {name}")
|
||||
for name in stale_classifications:
|
||||
print(f"[STALE] {name}")
|
||||
return 1 if invalid_classifications or missing_classifications or stale_classifications else 0
|
||||
|
||||
if invalid_classifications:
|
||||
for error in invalid_classifications:
|
||||
print(f"[ERROR] {error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if not args.junit or not args.junit.is_file():
|
||||
print(f"[ERROR] junit file not found: {args.junit}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
lists = {key: load_list(args.lists_dir / fname) for key, fname in LIST_FILES.items()}
|
||||
results = parse_junit(args.junit)
|
||||
exact_results, timed_out, failures = parse_junit(args.junit)
|
||||
results = collapse_results(exact_results)
|
||||
missing_results = sorted(collected - exact_results.keys()) if collected else []
|
||||
|
||||
regressions: list[str] = []
|
||||
promotions: dict[str, list[str]] = {"unimplemented": [], "excluded": []}
|
||||
@@ -155,7 +246,9 @@ def main() -> int:
|
||||
lines = [
|
||||
"# S3 compatibility report",
|
||||
"",
|
||||
f"Executed: {len(results)} tests — "
|
||||
f"Executed: {len(exact_results)} exact cases across {len(results)} classified tests.",
|
||||
"",
|
||||
"Classification status — "
|
||||
f"{counts['passed']} passed, {counts['failed']} failed, "
|
||||
f"{counts['error']} errored, {counts['skipped']} skipped.",
|
||||
"",
|
||||
@@ -191,6 +284,16 @@ def main() -> int:
|
||||
unclassified_failed,
|
||||
"Failing and absent from every list — triage into `unimplemented_tests.txt` or `excluded_tests.txt`.",
|
||||
)
|
||||
lines += render_section(
|
||||
"Missing results",
|
||||
missing_results,
|
||||
"Present in the pinned upstream suite but absent from JUnit — the sweep was incomplete.",
|
||||
)
|
||||
lines += render_section(
|
||||
"Timed out",
|
||||
timed_out,
|
||||
"Per-test timeout is an infrastructure failure regardless of compatibility classification.",
|
||||
)
|
||||
|
||||
report = "\n".join(lines)
|
||||
if args.output:
|
||||
@@ -201,13 +304,28 @@ def main() -> int:
|
||||
print(
|
||||
f"[INFO] {len(regressions)} regression(s), "
|
||||
f"{len(promotions['unimplemented']) + len(promotions['excluded']) + len(unclassified_passed)} promotion candidate(s), "
|
||||
f"{len(unclassified_failed)} unclassified failure(s)"
|
||||
f"{len(unclassified_failed)} unclassified failure(s), "
|
||||
f"{len(missing_results)} missing result(s), "
|
||||
f"{len(timed_out)} timeout(s)"
|
||||
)
|
||||
for name in sorted(regressions):
|
||||
print(f"[REGRESSION] {name}")
|
||||
if failures:
|
||||
print("[ERROR] s3-tests failed testcase summary:")
|
||||
for classname, name, duration, message in failures[:20]:
|
||||
nodeid = f"{classname}::{name}" if classname else name
|
||||
print(f"[ERROR] - {nodeid} ({duration}s): {message}")
|
||||
if len(failures) > 20:
|
||||
print(f"[ERROR] - ... {len(failures) - 20} additional failed testcases omitted")
|
||||
|
||||
if args.fail_on_regression and regressions:
|
||||
return 1
|
||||
if args.fail_on_unclassified and (unclassified_passed or unclassified_failed):
|
||||
return 1
|
||||
if args.collected_nodeids and missing_results:
|
||||
return 1
|
||||
if timed_out:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
+96
-62
@@ -58,6 +58,19 @@ if [[ "${TEST_SCOPE}" != "implemented" && "${TEST_SCOPE}" != "all" ]]; then
|
||||
echo "[ERROR] Invalid TEST_SCOPE: ${TEST_SCOPE} (must be \"implemented\" or \"all\")" >&2
|
||||
exit 1
|
||||
fi
|
||||
S3_SHARD_COUNT="${S3_SHARD_COUNT:-1}"
|
||||
S3_SHARD_INDEX="${S3_SHARD_INDEX:-0}"
|
||||
TEST_TIMEOUT="${TEST_TIMEOUT:-300}"
|
||||
if [[ ! "${S3_SHARD_COUNT}" =~ ^[1-9][0-9]*$ ]] \
|
||||
|| [[ ! "${S3_SHARD_INDEX}" =~ ^[0-9]+$ ]] \
|
||||
|| (( S3_SHARD_INDEX >= S3_SHARD_COUNT )); then
|
||||
echo "[ERROR] Invalid S3 shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! "${TEST_TIMEOUT}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "[ERROR] Invalid TEST_TIMEOUT: ${TEST_TIMEOUT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Upstream ceph/s3-tests suite, pinned for reproducible runs.
|
||||
# Bump S3TESTS_REV deliberately: upstream changes can rename tests or change
|
||||
@@ -96,55 +109,6 @@ log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $*"
|
||||
}
|
||||
|
||||
summarize_junit_failures() {
|
||||
local junit_path="$1"
|
||||
|
||||
if [ ! -f "${junit_path}" ]; then
|
||||
log_warn "JUnit report not found: ${junit_path}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 - "${junit_path}" <<'PY'
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
junit_path = sys.argv[1]
|
||||
try:
|
||||
root = ET.parse(junit_path).getroot()
|
||||
except Exception as exc:
|
||||
print(f"[WARN] Failed to parse JUnit report {junit_path}: {exc}")
|
||||
raise SystemExit(0)
|
||||
|
||||
failures = []
|
||||
for case in root.iter("testcase"):
|
||||
failure = case.find("failure")
|
||||
error = case.find("error")
|
||||
node = failure if failure is not None else error
|
||||
if node is None:
|
||||
continue
|
||||
|
||||
classname = case.attrib.get("classname", "")
|
||||
name = case.attrib.get("name", "")
|
||||
duration = case.attrib.get("time", "0")
|
||||
message = node.attrib.get("message") or (node.text or "").strip().splitlines()[0:1]
|
||||
if isinstance(message, list):
|
||||
message = message[0] if message else ""
|
||||
failures.append((classname, name, duration, message))
|
||||
|
||||
if not failures:
|
||||
print("[INFO] No failed testcases found in JUnit report")
|
||||
raise SystemExit(0)
|
||||
|
||||
print("[ERROR] s3-tests failed testcase summary:")
|
||||
for classname, name, duration, message in failures[:20]:
|
||||
nodeid = f"{classname}::{name}" if classname else name
|
||||
print(f"[ERROR] - {nodeid} ({duration}s): {message}")
|
||||
|
||||
if len(failures) > 20:
|
||||
print(f"[ERROR] - ... {len(failures) - 20} additional failed testcases omitted")
|
||||
PY
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Test Classification Files
|
||||
# =============================================================================
|
||||
@@ -322,6 +286,9 @@ Environment Variables:
|
||||
MAXFAIL - Stop after N failures, 0 = never stop (default: 1)
|
||||
XDIST - Enable parallel execution with N workers (default: 0)
|
||||
TEST_SCOPE - "implemented" (whitelist, default) or "all" (entire upstream suite)
|
||||
S3_SHARD_COUNT - Number of deterministic exact-node-ID shards (default: 1)
|
||||
S3_SHARD_INDEX - Zero-based shard index (default: 0)
|
||||
TEST_TIMEOUT - Per-test timeout in seconds (default: 300)
|
||||
S3TESTS_REPO - s3-tests repository URL (default: https://github.com/ceph/s3-tests.git)
|
||||
S3TESTS_REV - Pinned s3-tests commit; bump deliberately and reclassify test lists
|
||||
MARKEXPR - pytest marker expression (default: no marker filtering)
|
||||
@@ -982,9 +949,10 @@ mkdir -p "${ARTIFACTS_DIR}"
|
||||
XDIST_ARGS=""
|
||||
if [ "${XDIST}" != "0" ]; then
|
||||
# Add pytest-xdist to requirements.txt so tox installs it inside its virtualenv
|
||||
echo "pytest-xdist" >> requirements.txt
|
||||
grep -qxF "pytest-xdist" requirements.txt || echo "pytest-xdist" >> requirements.txt
|
||||
XDIST_ARGS="-n ${XDIST} --dist=loadgroup"
|
||||
fi
|
||||
grep -qxF "pytest-timeout" requirements.txt || echo "pytest-timeout" >> requirements.txt
|
||||
|
||||
# Resolve config path (absolute path for tox)
|
||||
if [[ "${S3TESTS_CONF}" = /* ]]; then
|
||||
@@ -1003,12 +971,69 @@ else
|
||||
PYTEST_SELECTION_ARGS=("${S3_TEST_FILE}")
|
||||
fi
|
||||
|
||||
collect_nodeids() {
|
||||
local output_path="$1"
|
||||
shift
|
||||
local collect_log="${output_path%.txt}.log"
|
||||
local collect_rc=0
|
||||
local node_prefix="${S3_TEST_FILE//./\\.}::"
|
||||
|
||||
set +e
|
||||
S3TEST_CONF="${CONF_OUTPUT_PATH}" tox -- -q --collect-only "$@" 2>&1 | tee "${collect_log}"
|
||||
collect_rc=${PIPESTATUS[0]}
|
||||
set -e
|
||||
if [ "${collect_rc}" -ne 0 ]; then
|
||||
log_error "pytest collection failed with exit code ${collect_rc}"
|
||||
return "${collect_rc}"
|
||||
fi
|
||||
grep -E "^${node_prefix}" "${collect_log}" > "${output_path}" || true
|
||||
if [ ! -s "${output_path}" ]; then
|
||||
log_error "pytest collection produced no S3 test node IDs"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
ALL_COLLECTED_NODEIDS="${ARTIFACTS_DIR}/all-collected-nodeids.txt"
|
||||
UNSHARDED_SELECTED_NODEIDS="${ARTIFACTS_DIR}/unsharded-selected-nodeids.txt"
|
||||
SELECTED_NODEIDS="${ARTIFACTS_DIR}/selected-nodeids.txt"
|
||||
collect_nodeids "${ALL_COLLECTED_NODEIDS}" "${S3_TEST_FILE}" -m "not rustfs_never_marker"
|
||||
python3 "${SCRIPT_DIR}/report_compat.py" \
|
||||
--lists-dir "${SCRIPT_DIR}" \
|
||||
--collected-nodeids "${ALL_COLLECTED_NODEIDS}" \
|
||||
--check-classifications-only || {
|
||||
log_error "S3 test classifications do not match pinned revision ${S3TESTS_REV}"
|
||||
exit 1
|
||||
}
|
||||
if [[ "${TEST_SCOPE}" == "all" && -z "${TESTEXPR}" && "${MARKEXPR}" == "not rustfs_never_marker" ]]; then
|
||||
cp "${ALL_COLLECTED_NODEIDS}" "${UNSHARDED_SELECTED_NODEIDS}"
|
||||
else
|
||||
collect_nodeids "${UNSHARDED_SELECTED_NODEIDS}" "${PYTEST_SELECTION_ARGS[@]}" -m "${MARKEXPR}"
|
||||
fi
|
||||
|
||||
if (( S3_SHARD_COUNT > 1 )); then
|
||||
awk -v count="${S3_SHARD_COUNT}" -v shard_index="${S3_SHARD_INDEX}" \
|
||||
'((NR - 1) % count) == shard_index' \
|
||||
"${UNSHARDED_SELECTED_NODEIDS}" > "${SELECTED_NODEIDS}"
|
||||
if [[ ! -s "${SELECTED_NODEIDS}" ]]; then
|
||||
log_error "Shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT} selected no tests"
|
||||
exit 1
|
||||
fi
|
||||
PYTEST_SELECTION_ARGS=()
|
||||
while IFS= read -r nodeid; do
|
||||
PYTEST_SELECTION_ARGS+=("${nodeid}")
|
||||
done < "${SELECTED_NODEIDS}"
|
||||
log_info "Selected shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT}: ${#PYTEST_SELECTION_ARGS[@]} exact cases"
|
||||
else
|
||||
cp "${UNSHARDED_SELECTED_NODEIDS}" "${SELECTED_NODEIDS}"
|
||||
fi
|
||||
|
||||
# Run tests from s3tests/functional
|
||||
set +e
|
||||
S3TEST_CONF="${CONF_OUTPUT_PATH}" \
|
||||
tox -- \
|
||||
-vv -ra --showlocals --tb=long \
|
||||
--maxfail="${MAXFAIL}" \
|
||||
--timeout="${TEST_TIMEOUT}" \
|
||||
--junitxml="${ARTIFACTS_DIR}/junit.xml" \
|
||||
${XDIST_ARGS} \
|
||||
"${PYTEST_SELECTION_ARGS[@]}" \
|
||||
@@ -1033,19 +1058,22 @@ elif [ "${DEPLOY_MODE}" = "existing" ]; then
|
||||
echo "{\"host\": \"${S3_HOST}\", \"port\": ${S3_PORT}, \"mode\": \"existing\"}" > "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/inspect.json" || true
|
||||
fi
|
||||
|
||||
# Step 11: Classification report (informational, never fails the run)
|
||||
# Step 11: Classification report and gate
|
||||
REPORT_SCRIPT="${SCRIPT_DIR}/report_compat.py"
|
||||
if [ -f "${REPORT_SCRIPT}" ] && [ -f "${ARTIFACTS_DIR}/junit.xml" ]; then
|
||||
python3 "${REPORT_SCRIPT}" \
|
||||
--junit "${ARTIFACTS_DIR}/junit.xml" \
|
||||
--lists-dir "${SCRIPT_DIR}" \
|
||||
--output "${ARTIFACTS_DIR}/compat-report.md" \
|
||||
|| log_warn "Compatibility report generation failed"
|
||||
fi
|
||||
|
||||
if [ ${TEST_EXIT_CODE} -ne 0 ]; then
|
||||
summarize_junit_failures "${ARTIFACTS_DIR}/junit.xml"
|
||||
REPORT_ARGS=(
|
||||
--junit "${ARTIFACTS_DIR}/junit.xml"
|
||||
--lists-dir "${SCRIPT_DIR}"
|
||||
--collected-nodeids "${SELECTED_NODEIDS}"
|
||||
--output "${ARTIFACTS_DIR}/compat-report.md"
|
||||
--fail-on-regression
|
||||
)
|
||||
if [[ "${TEST_SCOPE}" == "all" ]]; then
|
||||
REPORT_ARGS+=(--fail-on-unclassified)
|
||||
fi
|
||||
set +e
|
||||
python3 "${REPORT_SCRIPT}" "${REPORT_ARGS[@]}"
|
||||
REPORT_EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
# Summary
|
||||
if [ ${TEST_EXIT_CODE} -eq 0 ]; then
|
||||
@@ -1059,4 +1087,10 @@ else
|
||||
log_info "Check RustFS logs: ${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log"
|
||||
fi
|
||||
|
||||
exit ${TEST_EXIT_CODE}
|
||||
if [[ "${TEST_EXIT_CODE}" -ne 0 && "${TEST_EXIT_CODE}" -ne 1 ]]; then
|
||||
exit "${TEST_EXIT_CODE}"
|
||||
fi
|
||||
if [[ "${TEST_SCOPE}" == "implemented" && "${TEST_EXIT_CODE}" -ne 0 ]]; then
|
||||
exit "${TEST_EXIT_CODE}"
|
||||
fi
|
||||
exit "${REPORT_EXIT_CODE}"
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression tests for the S3 compatibility report."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPORT_PATH = Path(__file__).with_name("report_compat.py")
|
||||
SPEC = importlib.util.spec_from_file_location("report_compat", REPORT_PATH)
|
||||
assert SPEC and SPEC.loader
|
||||
REPORT = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(REPORT)
|
||||
|
||||
|
||||
class ReportCompatTests(unittest.TestCase):
|
||||
def test_upstream_names_expose_incomplete_junit(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
directory = Path(tmp)
|
||||
collected = directory / "collected.txt"
|
||||
collected.write_text("s3tests/functional/test_s3.py::test_one[a]\ns3tests/functional/test_s3.py::test_one[b]\n")
|
||||
junit = directory / "junit.xml"
|
||||
junit.write_text('<testsuite><testcase name="test_one[a]" /></testsuite>')
|
||||
|
||||
expected = REPORT.load_collected_nodeids(collected)
|
||||
results, _, _ = REPORT.parse_junit(junit)
|
||||
|
||||
self.assertEqual(expected - results.keys(), {"test_one[b]"})
|
||||
|
||||
def test_cli_fails_an_incomplete_sweep(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
directory = Path(tmp)
|
||||
collected = directory / "collected.txt"
|
||||
collected.write_text("s3tests/functional/test_s3.py::test_one\ns3tests/functional/test_s3.py::test_two\n")
|
||||
junit = directory / "junit.xml"
|
||||
junit.write_text('<testsuite><testcase name="test_one" /></testsuite>')
|
||||
for filename in REPORT.LIST_FILES.values():
|
||||
(directory / filename).write_text("")
|
||||
(directory / "implemented_tests.txt").write_text("test_one\n")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(REPORT_PATH),
|
||||
"--junit",
|
||||
str(junit),
|
||||
"--lists-dir",
|
||||
str(directory),
|
||||
"--collected-nodeids",
|
||||
str(collected),
|
||||
"--fail-on-regression",
|
||||
"--fail-on-unclassified",
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("1 missing result(s)", result.stdout)
|
||||
|
||||
def test_preflight_rejects_missing_and_stale_classifications(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
directory = Path(tmp)
|
||||
collected = directory / "collected.txt"
|
||||
collected.write_text("s3tests/functional/test_s3.py::test_known[a]\ntest_new\n")
|
||||
for filename in REPORT.LIST_FILES.values():
|
||||
(directory / filename).write_text("")
|
||||
(directory / "implemented_tests.txt").write_text("test_known\ntest_stale\ntest_stale\n")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(REPORT_PATH),
|
||||
"--lists-dir",
|
||||
str(directory),
|
||||
"--collected-nodeids",
|
||||
str(collected),
|
||||
"--check-classifications-only",
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("[UNCLASSIFIED] test_new", result.stdout)
|
||||
self.assertIn("[STALE] test_stale", result.stdout)
|
||||
self.assertIn("[INVALID] implemented_tests.txt has duplicates: test_stale", result.stdout)
|
||||
|
||||
def test_timeout_fails_even_when_test_is_excluded(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
directory = Path(tmp)
|
||||
junit = directory / "junit.xml"
|
||||
junit.write_text(
|
||||
'<testsuite><testcase name="test_slow"><failure message="Failed: Timeout (>300.0s)" /></testcase></testsuite>'
|
||||
)
|
||||
for filename in REPORT.LIST_FILES.values():
|
||||
(directory / filename).write_text("")
|
||||
(directory / "excluded_tests.txt").write_text("test_slow\n")
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(REPORT_PATH), "--junit", str(junit), "--lists-dir", str(directory)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("1 timeout(s)", result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -11,8 +11,12 @@
|
||||
|
||||
# Failed tests
|
||||
test_bucket_create_delete_bucket_ownership
|
||||
test_bucket_logging_request_id
|
||||
test_create_bucket_no_ownership_controls
|
||||
test_bucket_logging_owner
|
||||
test_head_object_404_with_policy_prefix
|
||||
test_multipart_reupload_checksum_and_etag
|
||||
test_multipart_upload_complete_without_create
|
||||
test_object_copy_not_owned_bucket
|
||||
test_bucket_policy_multipart
|
||||
test_post_object_upload_checksum
|
||||
|
||||
Reference in New Issue
Block a user