fix(ecstore): keep the single-block bound on every inline candidate (#7954)

The fault-tolerance suite was the only one without a dashboard upload
step - its reports never reached functional-reports/fault-tolerance/
without manual uploads. Add the standard upload step (continue-on-error,
same shape as the other ten suites), placed before the artifact upload.

Co-authored-by: Hauser <housemecn@gmail.com>

.agents/skills/security-advisory-lessons/references/advisory-patterns.md

@@ -48,7 +48,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
 
 ### S3 object actions, copy, multipart, and upload policy validation
 
-- `GHSA-g8w9-qw9q-fghr`: a valid presigned `PutObject` accepted extra `x-amz-tagging`, website redirect, and storage-class headers omitted from `SignedHeaders`. Lesson: a presigned URL is a bounded capability; reject `x-amz-*` headers that are not cryptographically bound by the signature so unsigned metadata cannot change authorization, lifecycle, redirect, cost, or durability semantics.
+- `GHSA-g8w9-qw9q-fghr` and `GHSA-xm99-m3gq-83g8`: a valid presigned `PutObject` accepted extra `x-amz-*` headers omitted from `SignedHeaders`; `x-amz-copy-source` could turn that upload capability into a cross-bucket read performed as the signer. Lesson: a presigned URL is a bounded capability; inspect all received security-sensitive headers and reject unsigned ones before selecting the S3 operation or reaching storage.
 - `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial.
 - `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies.
 - `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`.
@@ -120,7 +120,7 @@ Use these targeted searches when a diff touches security-sensitive code:
 ```bash
 rg -n "validate_admin_request|check_permissions|AdminAction::|deny_only|is_allowed" rustfs crates
 rg -n "authorize_operation|FtpsDriver|SftpDriver|RETR|MKD|SIZE|MDTM|CreateBucket|GetObject|HeadObject" crates/protocols rustfs
-rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|presign|SignedHeaders|content-length-range|starts-with" rustfs crates
+rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|presign|SignedHeaders|x-amz-copy-source|collect_signed_headers|content-length-range|starts-with" rustfs crates
 rg -n "ListBucketVersions|GetObjectVersion|versionId|VersionId|ExistingObjectTag|ForAllValues|ForAnyValue|POLICY_PLUGIN|opa" rustfs crates
 rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonicalize|\\.\\.|x-forwarded-for|x-real-ip|SourceIp" rustfs crates
 rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
@@ -137,7 +137,7 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
 - Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases, and assert denied paths do not reach the storage backend.
 - IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy.
 - Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases.
-- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, and storage-class headers; require rejection before storage access, and verify explicitly signed equivalents still work.
+- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, storage-class, and cross-bucket copy-source headers; require rejection before source or destination storage access, and verify explicitly signed equivalents still work.
 - Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases.
 - Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode.
 - Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks.

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2
-sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
+sha256-linux=21ab66fd172018682ce825a53f26aa8f286d57f3f687d8616d1850e56474dcd8
+sha256-darwin=21ab66fd172018682ce825a53f26aa8f286d57f3f687d8616d1850e56474dcd8

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=f0c78fdb93471575d9a64c5c46eae6c806bdd0bc10a6e33d7fb574aabd8db5a3
-sha256-linux=03ed7016cab672de9320e31375a0358eceacb4408b0e79cf063614fa7c878b87
+sha256-darwin=9e9687ded961aa4e619a15def6e1ee0a5153be05423d009ee7a58126624c83e7
+sha256-linux=364666693c2bf258997643c9d87e07e7b7501e58175f190d201d53c81946eb20

.config/e2e-nightly-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=364f2329a7b72eb9f1608dbe1a3af37af4095354014f3cbe23ca448492d89961
-sha256-linux=60983f1ebe7068cf660d473c5f76c76a650410ccc99d71934ddca7fd67607987
+sha256-darwin=83a7dcaffd5a789517ae9f02a224f66a9713937885cff96fca2ad7e216f197ae
+sha256-linux=626c10f8c964507ff987b6c86069e9019dc6d2ae7fb02db9be5df5aa8cc5145b

.config/e2e-repl-nightly-selection.txt

@@ -1 +1 @@
-sha256=0e338d305260229e17ccfb2adc48a6212dbdfea36a9ebfb5a4e0d38658e6cc45
+sha256=e4ee3898b840a8374cb02c64342cc15f8d7ecd1becaf4b4a2a4657290f9e7ff1

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=6d18f9cce820c51d5589de944e8cc185f73eeca0ea9a9916651943e3759169d0
+sha256=548a58d74f3cb3b5a1ab9dcd4d3f8625b9ddebe847a2e14de1d08c3b2e6292eb

.config/functional-script-revision.txt

@@ -0,0 +1 @@
+27e9584a2d776db9416d0edfd040d524d48b5d4d

.config/make/tests.mak

@@ -32,6 +32,16 @@ script-tests: ## Run shell script tests
 	./scripts/test_hotpath_warp_ab_gate.sh
 	./scripts/test_hotpath_warp_abba.sh
 	./scripts/test_scanner_validation_harness.sh
+	./scripts/test_scanner_heal_checkpoint_crash_evidence.sh
+	./scripts/test_scanner_heal_authority_evidence.sh
+	./scripts/test_scanner_heal_scoped_ack_evidence.sh
+	./scripts/test_scanner_heal_legacy_rollback_evidence.sh
+	./scripts/test_scanner_heal_g14_multiset_evidence.sh
+	./scripts/test_scanner_heal_scheduler_pressure_evidence.sh
+	./scripts/test_scanner_heal_status_outcome_evidence.sh
+	./scripts/test_scanner_heal_maintenance_evidence.sh
+	./scripts/test_scanner_heal_w13_mrf_evidence.sh
+	./scripts/test_scanner_heal_w16_recovery_evidence.sh
 	./scripts/test_exact_1mib_handoff_abba.sh
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
@@ -39,11 +49,15 @@ script-tests: ## Run shell script tests
 	./scripts/test_python_bin.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test

.config/nextest.toml

@@ -76,6 +76,12 @@ setup = 'ecstore-base-stack'
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'
 
+# The pool metadata placement regression composes the full erasure-heal
+# startup future and overflows the default Linux test-thread stack.
+[[profile.default.scripts]]
+filter = 'package(rustfs-heal) & test(=replacement_pool_metadata_follows_real_two_set_placement)'
+setup = 'ecstore-large-stack'
+
 [[profile.default.overrides]]
 filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(concurrent_config_writes_from_separate_nodes_do_not_lose_writes) | test(/^store::bucket::tests::bucket_delete_(mark_delete|purge_removes|default_s3_delete)/))'
 test-group = 'ecstore-serial-flaky'
@@ -126,6 +132,12 @@ test-group = 'ecstore-serial-flaky'
 filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
 test-group = 'embedded-test-ports'
 
+# The real object probe includes bucket creation and cleanup in its short
+# measurement budget. Keep competing storage fixtures outside that budget.
+[[profile.default.overrides]]
+filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
+threads-required = "num-test-threads"
+
 # Serialize the durable manual-transition checkpoint test across nextest's
 # process boundary; it mutates bucket lifecycle metadata and is not quarantined.
 [[profile.default.overrides]]
@@ -152,9 +164,18 @@ test-group = 'ecstore-serial-flaky'
 # deterministic commit barriers. Keep the whole init decommission family in one
 # nextest group; serial_test alone cannot isolate separate test processes.
 [[profile.default.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
+filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_.*|suspended_.*decommission.*)$/)'
 test-group = 'ecstore-serial-flaky'
 
+# The tier overwrite cleanup-owner matrix restarts a real 4-disk store and
+# drives free-version recovery through lifecycle expiry workers. Reserve the
+# full test budget so competing real-store fixtures cannot starve the walk's
+# filesystem progress deadlines. Keep the production deadlines and no retries.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners)'
+test-group = 'ecstore-serial-flaky'
+threads-required = "num-test-threads"
+
 # Serialize the bucket-incarnation / lifecycle-fence tests. They drive
 # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
 # OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -230,6 +251,12 @@ setup = 'ecstore-base-stack'
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'
 
+# The pool metadata placement regression composes the full erasure-heal
+# startup future and overflows the default Linux test-thread stack.
+[[profile.ci.scripts]]
+filter = 'package(rustfs-heal) & test(=replacement_pool_metadata_follows_real_two_set_placement)'
+setup = 'ecstore-large-stack'
+
 # ===========================================================================
 # QUARANTINE — flaky tests granted retries = 2 under the ci profile ONLY.
 #
@@ -306,6 +333,10 @@ test-group = 'ecstore-serial-flaky'
 filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
 test-group = 'embedded-test-ports'
 
+[[profile.ci.overrides]]
+filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
+threads-required = "num-test-threads"
+
 # Serialize the durable manual-transition checkpoint test under the ci profile
 # too. No retries: failures stay visible.
 [[profile.ci.overrides]]
@@ -323,9 +354,14 @@ filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_reg
 test-group = 'ecstore-serial-flaky'
 
 [[profile.ci.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
+filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_.*|suspended_.*decommission.*)$/)'
 test-group = 'ecstore-serial-flaky'
 
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners)'
+test-group = 'ecstore-serial-flaky'
+threads-required = "num-test-threads"
+
 # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
 # too (see the matching default-profile override near the top). No retries.
 [[profile.ci.overrides]]
@@ -539,6 +575,13 @@ path = "junit.xml"
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'
 
+# The EC8+4 multi-set heal proof deliberately uploads a larger workload so the
+# background root-heal pass can be interrupted after targeting the replacement
+# drive's erasure slots. Keep the extended budget scoped to this proof case.
+[[profile.e2e-nightly.overrides]]
+filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart)'
+slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
+
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
@@ -638,8 +681,8 @@ path = "junit.xml"
 # Each e2e test spawns its own single-node rustfs server on a random port with
 # an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
 # parallel-safe — the same property e2e-smoke relies on. The exceptions are the
-# 4-disk reliability / degraded-read fault-injection tests and the fixed-port
-# Vault tests, both serialized below.
+# 4-disk reliability / degraded-read fault-injection tests, multi-node heal
+# clusters, and the fixed-port Vault tests, all serialized below.
 [profile.e2e-full]
 default-filter = """
     package(e2e_test)
@@ -668,6 +711,12 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
 
+# Heal interruption cases start whole multi-node clusters. Bound their overlap
+# just as the nightly lane does, without changing membership or retry policy.
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(/^heal_erasure_disk_rebuild_test::/)'
+test-group = 'e2e-cluster-nightly'
+
 [[profile.e2e-full.overrides]]
 filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
 test-group = 'e2e-vault'

.config/scanner-heal-required-tests.json

@@ -1,5 +1,5 @@
 {
-  "schema": 1,
+  "schema": 2,
   "cases": {
     "background-target-restart": {
       "gate": "G14",
@@ -13,6 +13,7 @@
       "min_objects": 9,
       "max_objects": 65,
       "topology": {"nodes": 4, "drives_per_node": 1},
+      "erasure": {"data_blocks": 2, "parity_blocks": 2},
       "scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
     },
     "background-target-crash": {
@@ -27,30 +28,422 @@
       "min_objects": 9,
       "max_objects": 65,
       "topology": {"nodes": 4, "drives_per_node": 1},
+      "erasure": {"data_blocks": 2, "parity_blocks": 2},
       "scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
+    },
+    "ec84-target-drive-restart": {
+      "gate": "G14",
+      "task": "W20/W21",
+      "lane": "e2e-distributed",
+      "suite": "e2e_test",
+      "name": "distributed::heal_test::three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart",
+      "oracle": "ec84-target-drive-restart.json",
+      "evidence": "process-restart",
+      "unclean_shutdown_marker": false,
+      "min_objects": 5,
+      "max_objects": 5,
+      "topology": {"nodes": 3, "drives_per_node": 4},
+      "erasure": {"data_blocks": 8, "parity_blocks": 4},
+      "erasure_set_drive_count": 12,
+      "scope": "3-node x 4-drive single-set EC8+4, graceful target restart, preformatted replacement drive, exact unversioned S3 bodies and physical target shards; not mixed-version, multi-pool or long-window ABBA."
+    },
+    "background-target-restart-ec8-4": {
+      "gate": "G14",
+      "task": "W21",
+      "lane": "e2e-nightly",
+      "suite": "e2e_test",
+      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart",
+      "oracle": "background-target-restart-ec8-4.json",
+      "evidence": "process-restart",
+      "unclean_shutdown_marker": false,
+      "min_objects": 9,
+      "max_objects": 65,
+      "topology": {"nodes": 3, "drives_per_node": 4},
+      "erasure": {"data_blocks": 8, "parity_blocks": 4},
+      "erasure_set_drive_count": 12,
+      "scope": "Target process restart during partial background rebuild on a single 3x4 EC8+4 set; exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
+    },
+    "background-target-crash-ec8-4": {
+      "gate": "G14",
+      "task": "W21",
+      "lane": "e2e-nightly",
+      "suite": "e2e_test",
+      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash",
+      "oracle": "background-target-crash-ec8-4.json",
+      "evidence": "process-crash-restart",
+      "unclean_shutdown_marker": true,
+      "min_objects": 9,
+      "max_objects": 65,
+      "topology": {"nodes": 3, "drives_per_node": 4},
+      "erasure": {"data_blocks": 8, "parity_blocks": 4},
+      "erasure_set_drive_count": 12,
+      "scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
+    },
+    "background-target-restart-ec8-4-multi-set": {
+      "gate": "G14",
+      "task": "W21",
+      "lane": "e2e-nightly",
+      "suite": "e2e_test",
+      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart",
+      "oracle": "background-target-restart-ec8-4-multi-set.json",
+      "evidence": "process-restart",
+      "unclean_shutdown_marker": false,
+      "min_objects": 9,
+      "max_objects": 65,
+      "topology": {"nodes": 3, "drives_per_node": 8},
+      "erasure": {"data_blocks": 8, "parity_blocks": 4},
+      "erasure_set_drive_count": 12,
+      "sets": 2,
+      "pools": 1,
+      "scope": "Target process restart during partial background rebuild on a 3x8 EC8+4 layout with two erasure sets in one pool; exact unversioned S3 bodies and replacement-drive shards; not power loss or multi-pool."
+    },
+    "background-target-crash-ec8-4-multi-pool": {
+      "gate": "G14",
+      "task": "W21",
+      "lane": "e2e-nightly",
+      "suite": "e2e_test",
+      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash",
+      "oracle": "background-target-crash-ec8-4-multi-pool.json",
+      "evidence": "process-crash-restart",
+      "unclean_shutdown_marker": true,
+      "min_objects": 9,
+      "max_objects": 65,
+      "topology": {"nodes": 3, "drives_per_node": 12},
+      "erasure": {"data_blocks": 8, "parity_blocks": 4},
+      "erasure_set_drive_count": 12,
+      "sets": 3,
+      "pools": 3,
+      "outage_target_manifest_required": false,
+      "scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with down-window outage PUT refusal recorded and a deferred post-rejoin outage object verified through S3 when the full target pool was offline."
     }
   },
-  "release_pending": {
-    "G01": "W02/W04 complete root and quota authority coverage",
-    "G02": "W03 bounded checkpoint progress and independent version inventory",
-    "G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
-    "G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
-    "G05": "W06/W07 per-object outcomes and bounded terminal retention",
-    "G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
-    "G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
-    "G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
-    "G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
-    "G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
-    "G11": "W04/W19/W24 maintenance and complete producer coverage",
-    "G12": "W02/W15/W16 both quota paths during reset and settlement",
-    "G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
-    "G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
-    "P1": "W20 measured cold-walk share and foreground latency/throughput",
-    "P2": "W20/W24 measured post-stop convergence and cold segment reuse",
-    "P3": "W20 measured two-hour pressure/heal capacity and recovery window",
-    "P4": "W20 measured MRF scale and replay cost with retained responsibility",
-    "R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
-    "R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
-    "R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
-  }
+  "release_lanes": {
+    "single-set-restart": {
+      "status": "implemented",
+      "cases": ["background-target-restart", "background-target-crash"],
+      "covers": ["four-node one-drive topology", "unversioned objects", "target restart/crash"]
+    },
+    "authority-coverage": {
+      "status": "pending",
+      "gates": ["G01", "G12"],
+      "requires": ["root authority coverage", "quota authority coverage"]
+    },
+    "checkpoint-and-crash": {
+      "status": "pending",
+      "gates": ["G02", "G04", "R-E"],
+      "requires": ["bounded checkpoint progress", "boundary crash matrix", "fixed-budget restart evidence"]
+    },
+    "status-and-outcome": {
+      "status": "pending",
+      "gates": ["G05", "G06", "R-D"],
+      "requires": ["per-object outcomes", "legacy status clients", "manager/event/ledger disposition"]
+    },
+    "mrf-responsibility": {
+      "status": "pending",
+      "gates": ["G07", "G08", "P4"],
+      "requires": ["durable MRF responsibility", "disk-full and replica-loss matrix", "MRF replay cost"]
+    },
+    "mixed-version-rollback": {
+      "status": "pending",
+      "gates": ["G03", "G09", "R-L"],
+      "requires": ["mixed-version peers", "rollback payloads", "crash-safe source retirement"]
+    },
+    "scheduler-pressure": {
+      "status": "pending",
+      "gates": ["G10", "P1", "P2", "P3"],
+      "requires": ["bounded scheduling", "foreground latency and throughput", "two-hour pressure evidence"]
+    },
+    "maintenance-producers": {
+      "status": "pending",
+      "gates": ["G11", "G13"],
+      "requires": ["complete producer coverage", "quorum-minus-one and remount matrix"]
+    },
+    "ec8-4-multiset": {
+      "status": "pending",
+      "gates": ["G14"],
+      "requires": ["3x4 EC8+4 topology", "multi-set coverage", "multi-pool coverage"]
+    }
+  },
+  "release_requirements": [
+    {
+      "gate": "G01",
+      "task": "W02/W04",
+      "lane": "authority-coverage",
+      "status": "pending",
+      "description": "Complete root and quota authority coverage",
+      "requires": ["root authority evidence", "quota authority evidence"],
+      "evidence_fields": [
+        "root_authority_evidence",
+        "quota_authority_evidence"
+      ]
+    },
+    {
+      "gate": "G02",
+      "task": "W03",
+      "lane": "checkpoint-and-crash",
+      "status": "pending",
+      "description": "Bounded checkpoint progress and independent version inventory",
+      "requires": ["bounded checkpoint oracle", "independent version inventory"],
+      "evidence_fields": [
+        "bounded_checkpoint_oracle",
+        "independent_version_inventory"
+      ]
+    },
+    {
+      "gate": "G03",
+      "task": "W17/W18",
+      "lane": "mixed-version-rollback",
+      "status": "pending",
+      "description": "Exact scoped ACK with durable publication and mixed peers",
+      "requires": ["durable scoped ACK publication", "mixed-peer evidence"],
+      "evidence_fields": [
+        "durable_root_publication_proof",
+        "scoped_ack_request_identity",
+        "participating_peer_capability_snapshot",
+        "mixed_peer_ack_fallback_oracle"
+      ]
+    },
+    {
+      "gate": "G04",
+      "task": "W03/W15/W16",
+      "lane": "checkpoint-and-crash",
+      "status": "pending",
+      "description": "Crash at every cache, root, floor and intent boundary",
+      "requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"],
+      "evidence_fields": [
+        "cache_boundary_crash_evidence",
+        "root_floor_intent_crash_evidence"
+      ]
+    },
+    {
+      "gate": "G05",
+      "task": "W06/W07",
+      "lane": "status-and-outcome",
+      "status": "pending",
+      "description": "Per-object outcomes and bounded terminal retention",
+      "requires": ["per-object outcome oracle", "terminal retention bounds"],
+      "evidence_fields": [
+        "per_object_outcome_oracle",
+        "terminal_retention_bounds"
+      ]
+    },
+    {
+      "gate": "G06",
+      "task": "W06/W08/W23",
+      "lane": "status-and-outcome",
+      "status": "pending",
+      "description": "Concurrent status, legacy clients and truncation",
+      "requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"],
+      "evidence_fields": [
+        "concurrent_status_evidence",
+        "legacy_client_compatibility",
+        "truncation_behavior"
+      ]
+    },
+    {
+      "gate": "G07",
+      "task": "W12/W13/W14",
+      "lane": "mrf-responsibility",
+      "status": "pending",
+      "description": "Durable MRF responsibility at every commit boundary",
+      "requires": ["MRF responsibility oracle", "commit-boundary crash matrix"],
+      "evidence_fields": [
+        "mrf_responsibility_oracle",
+        "commit_boundary_crash_matrix"
+      ]
+    },
+    {
+      "gate": "G08",
+      "task": "W12/W13/W14",
+      "lane": "mrf-responsibility",
+      "status": "pending",
+      "description": "MRF capacity, disk-full and replica-loss matrix",
+      "requires": ["MRF capacity evidence", "disk-full matrix", "replica-loss matrix"],
+      "evidence_fields": [
+        "mrf_capacity_evidence",
+        "disk_full_matrix",
+        "replica_loss_matrix"
+      ]
+    },
+    {
+      "gate": "G09",
+      "task": "W13/W18/W23",
+      "lane": "mixed-version-rollback",
+      "status": "pending",
+      "description": "Actual mixed-version reader/writer and rollback payloads",
+      "requires": ["mixed-version reader evidence", "mixed-version writer evidence", "rollback payload evidence"],
+      "evidence_fields": [
+        "mixed_version_reader_evidence",
+        "mixed_version_writer_evidence",
+        "rollback_payload_evidence"
+      ]
+    },
+    {
+      "gate": "G10",
+      "task": "W05/W09/W10/W11",
+      "lane": "scheduler-pressure",
+      "status": "pending",
+      "description": "Bounded scheduling and pressure recovery",
+      "requires": ["scheduler bound evidence", "pressure recovery evidence"],
+      "evidence_fields": [
+        "scheduler_bound_evidence",
+        "pressure_recovery_evidence"
+      ]
+    },
+    {
+      "gate": "G11",
+      "task": "W04/W19/W24",
+      "lane": "maintenance-producers",
+      "status": "pending",
+      "description": "Maintenance and complete producer coverage",
+      "requires": [
+        "maintenance producer matrix",
+        "complete producer inventory",
+        "segment activation preflight"
+      ],
+      "evidence_fields": [
+        "maintenance_producer_matrix",
+        "complete_producer_inventory",
+        "segment_activation_preflight"
+      ]
+    },
+    {
+      "gate": "G12",
+      "task": "W02/W15/W16",
+      "lane": "authority-coverage",
+      "status": "pending",
+      "description": "Both quota paths during reset and settlement",
+      "requires": ["reset quota-path evidence", "settlement quota-path evidence"],
+      "evidence_fields": [
+        "reset_quota_path_evidence",
+        "settlement_quota_path_evidence"
+      ]
+    },
+    {
+      "gate": "G13",
+      "task": "W07/W14",
+      "lane": "maintenance-producers",
+      "status": "pending",
+      "description": "Quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
+      "requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"],
+      "evidence_fields": [
+        "quorum_minus_one_matrix",
+        "unknown_disk_remount_matrix",
+        "object_lock_dry_run_grace_evidence"
+      ]
+    },
+    {
+      "gate": "G14",
+      "task": "W20/W21",
+      "lane": "ec8-4-multiset",
+      "status": "pending",
+      "description": "Same-window field evidence with 3x4 EC8+4 and multi-set/pool coverage",
+      "requires": [
+        "same-window field evidence",
+        "3x4 EC8+4 evidence",
+        "multi-set evidence",
+        "multi-pool evidence",
+        "distributed segment invalidation evidence"
+      ],
+      "evidence_fields": [
+        "same_window_field_evidence",
+        "ec8_4_evidence",
+        "multi_set_evidence",
+        "multi_pool_evidence",
+        "distributed_segment_invalidation_evidence"
+      ]
+    },
+    {
+      "gate": "P1",
+      "task": "W20",
+      "lane": "scheduler-pressure",
+      "status": "pending",
+      "description": "Measured cold-walk share and foreground latency/throughput",
+      "requires": ["cold-walk share measurement", "foreground latency/throughput measurement"],
+      "evidence_fields": [
+        "cold_walk_share_measurement",
+        "foreground_latency_throughput_measurement",
+        "profile_evidence"
+      ]
+    },
+    {
+      "gate": "P2",
+      "task": "W20/W24",
+      "lane": "scheduler-pressure",
+      "status": "pending",
+      "description": "Measured post-stop convergence and cold segment reuse",
+      "requires": ["post-stop convergence measurement", "cold segment reuse measurement"],
+      "evidence_fields": [
+        "post_stop_convergence_measurement",
+        "cold_segment_reuse_measurement"
+      ]
+    },
+    {
+      "gate": "P3",
+      "task": "W20",
+      "lane": "scheduler-pressure",
+      "status": "pending",
+      "description": "Measured two-hour pressure/heal capacity and recovery window",
+      "requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"],
+      "evidence_fields": [
+        "two_hour_pressure_measurement",
+        "heal_capacity_measurement",
+        "recovery_window_measurement"
+      ]
+    },
+    {
+      "gate": "P4",
+      "task": "W20",
+      "lane": "mrf-responsibility",
+      "status": "pending",
+      "description": "Measured MRF scale and replay cost with retained responsibility",
+      "requires": ["MRF scale measurement", "MRF replay-cost measurement", "retained responsibility evidence", "cleanup/GC soak evidence"],
+      "evidence_fields": [
+        "mrf_scale_measurement",
+        "mrf_replay_cost_measurement",
+        "retained_responsibility_evidence",
+        "mrf_cleanup_gc_soak_evidence"
+      ]
+    },
+    {
+      "gate": "R-E",
+      "task": "W03/W05",
+      "lane": "checkpoint-and-crash",
+      "status": "pending",
+      "description": "Fixed-budget real process restart through enumeration and classification",
+      "requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"],
+      "evidence_fields": [
+        "fixed_budget_restart_evidence",
+        "enumeration_evidence",
+        "classification_evidence"
+      ]
+    },
+    {
+      "gate": "R-D",
+      "task": "W07/W14",
+      "lane": "status-and-outcome",
+      "status": "pending",
+      "description": "Manager-to-event-to-ledger exact disposition, including grace",
+      "requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"],
+      "evidence_fields": [
+        "manager_disposition_evidence",
+        "event_disposition_evidence",
+        "ledger_disposition_evidence",
+        "grace_handling"
+      ]
+    },
+    {
+      "gate": "R-L",
+      "task": "W13/W14",
+      "lane": "mixed-version-rollback",
+      "status": "pending",
+      "description": "Legacy source conflicts, migration gaps and crash-safe source retirement",
+      "requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"],
+      "evidence_fields": [
+        "legacy_source_conflict_evidence",
+        "migration_gap_evidence",
+        "crash_safe_source_retirement_evidence"
+      ]
+    }
+  ]
 }

.config/security-smoke-floor.txt

@@ -9,4 +9,4 @@
 # if the selected count drops below this number, so a rename or removal that
 # thins the security smoke gate must update this file in the same PR.
 # Adding tests does not require a bump, but bumping keeps the guard tight.
-26
+28

.docker/compose/docker-compose.cluster.local-build.profiling-amd64.yml

@@ -43,6 +43,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
@@ -89,6 +90,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
@@ -135,6 +137,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
@@ -181,6 +184,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
       # should show richer nested spans during request-path verification.
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}

.docker/compose/docker-compose.cluster.local-build.yml

@@ -26,6 +26,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
@@ -72,6 +73,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
@@ -118,6 +120,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
@@ -164,6 +167,7 @@ services:
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
       - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
       - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}

.docker/compose/docker-compose.observability.yaml

@@ -201,6 +201,7 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
@@ -220,6 +221,7 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
@@ -239,6 +241,7 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:
@@ -258,6 +261,7 @@ services:
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_LOGGER_LEVEL=debug
     platform: linux/amd64
     ports:

.docker/observability/README.md

@@ -47,6 +47,10 @@ Three pre-built Grafana dashboards are included for monitoring RustFS GET perfor
 | **GET Resource Impact** | `grafana-get-resource-impact.json` | Monitors resource usage: concurrent requests, IO queue utilization, disk permit wait, RSS trend |
 | **Object Data Cache** | `grafana-object-data-cache.json` | Monitors the GET body cache (`rustfs_object_data_cache_*`): hit ratio, lookup/plan/fill outcomes, fill duration quantiles, hit vs fill throughput, entries/weighted bytes, inflight fills, memory-pressure skips, invalidations, and size-class breakdowns |
 
+### Storage Metrics
+
+Storage panels require `prometheus-rules/rustfs-storage.yml` and the cluster resource attribute. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for ownership, observer selection, freshness, and rolling upgrades.
+
 ### Prometheus Alert Rules
 
 The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-configured alerting rules:

.docker/observability/docker-compose-example-for-rustfs.yml

@@ -32,6 +32,7 @@ services:
       - RUSTFS_SECRET_KEY=rustfsadmin
       - RUSTFS_OBS_LOGGER_LEVEL=info
       - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
       - RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040
     volumes:
       - rustfs-data:/data/rustfs

.docker/observability/grafana/dashboards/rustfs.json

@@ -223,14 +223,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_buckets_total{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_buckets_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Total Buckets",
-      "type": "stat"
+      "type": "stat",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -289,14 +290,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_objects_total{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_objects_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Total Objects",
-      "type": "stat"
+      "type": "stat",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -427,7 +429,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "Used",
           "range": true,
           "refId": "A"
@@ -438,7 +440,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "hide": false,
           "legendFormat": "Total",
           "range": true,
@@ -450,7 +452,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "hide": false,
           "instant": false,
           "legendFormat": "Percent",
@@ -459,7 +461,8 @@
         }
       ],
       "title": "Capacity",
-      "type": "stat"
+      "type": "stat",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -525,7 +528,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_stale_drives{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_stale_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "Stale Drives",
           "range": true,
           "refId": "A"
@@ -536,14 +539,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rustfs_cluster_capacity_missing_drives{job=~\"$job\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_missing_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "Missing Drives",
           "range": true,
           "refId": "B"
         }
       ],
       "title": "Capacity Observation",
-      "type": "stat"
+      "type": "stat",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -1989,8 +1993,8 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"})",
-          "legendFormat": "{{drive}} (bytes)",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
+          "legendFormat": "{{server}} | {{drive}} (bytes)",
           "range": true,
           "refId": "A"
         },
@@ -2000,16 +2004,17 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(rustfs_system_drive_total_bytes{job=~\"$job\", drive=~\"$drive\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_total_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
           "hide": false,
           "instant": false,
-          "legendFormat": "{{drive}} (percent)",
+          "legendFormat": "{{server}} | {{drive}} (percent)",
           "range": true,
           "refId": "B"
         }
       ],
       "title": "System Drive Usage",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -2097,14 +2102,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "max by (drive) (rustfs_system_drive_capacity_observation_age_seconds{job=~\"$job\", drive=~\"$drive\"})",
-          "legendFormat": "{{drive}}",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_age_seconds\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
+          "legendFormat": "{{server}} | {{drive}}",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Drive Capacity Observation Age",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -2190,8 +2196,8 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"stale\"})",
-          "legendFormat": "{{drive}} stale",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"stale\"}",
+          "legendFormat": "{{server}} | {{drive}} stale",
           "range": true,
           "refId": "A"
         },
@@ -2201,14 +2207,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"missing\"})",
-          "legendFormat": "{{drive}} missing",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"missing\"}",
+          "legendFormat": "{{server}} | {{drive}} missing",
           "range": true,
           "refId": "B"
         }
       ],
       "title": "Drive Capacity Observation State",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -4551,7 +4558,7 @@
                   "index": 0,
                   "text": "INACTIVE"
                 },
-                "to": 1e-9
+                "to": 1e-09
               },
               "type": "range"
             }
@@ -6563,7 +6570,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_cluster_health_drives_online_count{job=~\"$job\"}",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_online_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "online - {{job}}",
           "range": true,
           "refId": "A"
@@ -6574,7 +6581,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_cluster_health_drives_offline_count{job=~\"$job\"}",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_offline_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "offline - {{job}}",
           "range": true,
           "refId": "B"
@@ -6585,14 +6592,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_cluster_health_drives_count{job=~\"$job\"}",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
           "legendFormat": "total - {{job}}",
           "range": true,
           "refId": "C"
         }
       ],
       "title": "Cluster Drive Health Counts",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "collapsed": false,
@@ -8554,14 +8562,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "{__name__=~\"rustfs_system_drive_.*\",job=~\"$job\",drive=~\"$drive\"}",
-          "legendFormat": "{{__name__}} | {{drive}}",
+          "expr": "rustfs:storage:current{source_metric=~\"rustfs_system_drive_.*\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
+          "legendFormat": "{{server}} | {{source_metric}} | {{drive}}",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "System Drive (All)",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -8942,14 +8951,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "{__name__=~\"rustfs_cluster_erasure_set_.*\",job=~\"$job\"}",
-          "legendFormat": "{{__name__}}",
+          "expr": "rustfs:storage:current{source_metric=~\"rustfs_cluster_erasure_set_.*\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
+          "legendFormat": "{{source_metric}}",
           "range": true,
           "refId": "A"
         }
       ],
       "title": "Cluster Erasure Set (All)",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -11716,7 +11726,7 @@
           "editorMode": "code",
           "range": true,
           "refId": "A",
-          "expr": "max by (server, drive, pool_index, set_index, drive_index, state) (rustfs_system_drive_runtime_state{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_runtime_state\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}",
           "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{state}}"
         },
         {
@@ -11727,12 +11737,13 @@
           "editorMode": "code",
           "range": true,
           "refId": "B",
-          "expr": "max by (server, drive, pool_index, set_index, drive_index) (rustfs_system_drive_offline_duration_seconds{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
+          "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_offline_duration_seconds\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}",
           "legendFormat": "{{server}} | {{drive}} | offline seconds"
         }
       ],
-      "title": "Drive Runtime State and Offline Duration",
-      "type": "timeseries"
+      "title": "Observed Cluster Drive State and Offline Duration",
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -11825,12 +11836,13 @@
           "editorMode": "code",
           "range": true,
           "refId": "A",
-          "expr": "sum by (server, drive, pool_index, set_index, drive_index, api) (rate(rustfs_system_drive_api_calls_total{job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]))",
+          "expr": "rate(rustfs_system_drive_api_calls_total{collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]) and ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}",
           "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{api}}"
         }
       ],
       "title": "Drive API Calls by Operation",
-      "type": "timeseries"
+      "type": "timeseries",
+      "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
     },
     {
       "datasource": {
@@ -12376,6 +12388,50 @@
         "sort": 1,
         "type": "query"
       },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)",
+        "includeAll": false,
+        "label": "Storage cluster",
+        "multi": false,
+        "name": "storage_cluster",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)",
+          "refId": "PrometheusVariableQueryEditor-storage_cluster"
+        },
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})",
+        "includeAll": false,
+        "label": "Cluster observer",
+        "multi": false,
+        "name": "storage_observer",
+        "options": [],
+        "query": {
+          "qryType": 3,
+          "query": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})",
+          "refId": "PrometheusVariableQueryEditor-storage_observer"
+        },
+        "refresh": 2,
+        "regex": "/observer=\"([^\"]+)\"/",
+        "sort": 1,
+        "type": "query"
+      },
       {
         "allValue": ".*",
         "current": {
@@ -12386,7 +12442,7 @@
           "type": "prometheus",
           "uid": "${datasource}"
         },
-        "definition": "label_values(rustfs_system_drive_api_calls_total,api)",
+        "definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)",
         "includeAll": true,
         "label": "Drive API",
         "multi": true,
@@ -12394,7 +12450,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(rustfs_system_drive_api_calls_total,api)",
+          "query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)",
           "refId": "PrometheusVariableQueryEditor-drive_api"
         },
         "refresh": 2,
@@ -12455,7 +12511,7 @@
           "text": "All",
           "value": "$__all"
         },
-        "definition": "label_values(rustfs_system_drive_used_bytes,drive)",
+        "definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)",
         "includeAll": true,
         "label": "Drive",
         "multi": true,
@@ -12463,7 +12519,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(rustfs_system_drive_used_bytes,drive)",
+          "query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 2,

.docker/observability/prometheus-rules/rustfs-storage.yml

@@ -0,0 +1,575 @@
+# Preserve OTLP timestamps in the Collector. Apply timestamp() directly to
+# each raw selector, before label rewriting; functions such as label_replace()
+# replace the evaluation timestamp and would make a cached value look new.
+# Compare publication times at Prometheus millisecond precision so points
+# published and exported within the same millisecond are not withheld.
+groups:
+  - name: rustfs-storage-snapshots
+    interval: 15s
+    rules:
+      - record: rustfs:storage_snapshot:fresh
+        expr: |
+          rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}
+          and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) <= rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"})
+          and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) >= 0)
+          and (rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"} > 0)
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_buckets_total
+        expr: |
+          rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_free_bytes
+        expr: |
+          rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_missing_drives
+        expr: |
+          rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_raw_total_bytes
+        expr: |
+          rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_stale_drives
+        expr: |
+          rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_usable_total_bytes
+        expr: |
+          rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_capacity_used_bytes
+        expr: |
+          rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_capacity_observation_age_seconds
+        expr: |
+          rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_capacity_observation_state
+        expr: |
+          rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_free_bytes
+        expr: |
+          rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_offline_duration_seconds
+        expr: |
+          rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_present
+        expr: |
+          rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_runtime_state
+        expr: |
+          rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_total_bytes
+        expr: |
+          rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_drive_used_bytes
+        expr: |
+          rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_data_shards
+        expr: |
+          rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_healing_drives_count
+        expr: |
+          rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_health
+        expr: |
+          rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_online_drives_count
+        expr: |
+          rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_overall_health
+        expr: |
+          rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_overall_write_quorum
+        expr: |
+          rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_parity
+        expr: |
+          rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_read_health
+        expr: |
+          rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_read_quorum
+        expr: |
+          rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_read_tolerance
+        expr: |
+          rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_size
+        expr: |
+          rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_write_health
+        expr: |
+          rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_write_quorum
+        expr: |
+          rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_erasure_set_write_tolerance
+        expr: |
+          rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_health_drives_count
+        expr: |
+          rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_health_drives_offline_count
+        expr: |
+          rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_health_drives_online_count
+        expr: |
+          rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_cluster_objects_total
+        expr: |
+          rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"}
+          and (timestamp(rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_node_disk_free_bytes
+        expr: |
+          rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_node_disk_total_bytes
+        expr: |
+          rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_node_disk_used_bytes
+        expr: |
+          rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_api_calls_total
+        expr: |
+          rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_api_latency_by_api_micros
+        expr: |
+          rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_api_latency_micros
+        expr: |
+          rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_availability_errors_total
+        expr: |
+          rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_capacity_observation_age_seconds
+        expr: |
+          rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_capacity_observation_state
+        expr: |
+          rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_count
+        expr: |
+          rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_deletes_total
+        expr: |
+          rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_free_bytes
+        expr: |
+          rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_free_inodes
+        expr: |
+          rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_healing
+        expr: |
+          rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_health
+        expr: |
+          rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_info
+        expr: |
+          rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_io_errors_total
+        expr: |
+          rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_offline_count
+        expr: |
+          rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_offline_duration_seconds
+        expr: |
+          rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_online_count
+        expr: |
+          rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_perc_util
+        expr: |
+          rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_present
+        expr: |
+          rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_reads_await
+        expr: |
+          rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_reads_kb_per_sec
+        expr: |
+          rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_reads_per_sec
+        expr: |
+          rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_runtime_state
+        expr: |
+          rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_scanning
+        expr: |
+          rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_timeout_errors_total
+        expr: |
+          rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_total_bytes
+        expr: |
+          rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_total_inodes
+        expr: |
+          rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_used_bytes
+        expr: |
+          rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_used_inodes
+        expr: |
+          rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_waiting_io
+        expr: |
+          rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_writes_await
+        expr: |
+          rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_writes_kb_per_sec
+        expr: |
+          rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_writes_per_sec
+        expr: |
+          rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
+      - record: rustfs:storage:current
+        labels:
+          source_metric: rustfs_system_drive_writes_total
+        expr: |
+          rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"}
+          and (timestamp(rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"})
+            >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
+            (floor(rustfs:storage_snapshot:fresh * 1000) / 1000))

.docker/observability/tests/storage-rules.test.yml

@@ -0,0 +1,189 @@
+rule_files:
+- ../prometheus-rules/rustfs-storage.yml
+evaluation_interval: 15s
+tests:
+- name: owners, pools, clusters and observer views remain distinct in mixed versions
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"}
+    values: 100x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"}
+    values: 100x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"}
+    values: 100x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"}
+    values: 100x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"}
+    values: 1000x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"}
+    values: 1000x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"}
+    values: 1000x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"}
+    values: 1000x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 400x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 300x8
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="old",server="n0",drive="/data",job="rustfs",instance="collector"}
+    values: 999x8
+  promql_expr_test:
+  - expr: sum by (rustfs_cluster_id) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",collection_scope="local"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{rustfs_cluster_id="a"}'
+      value: 400
+    - labels: '{rustfs_cluster_id="b"}'
+      value: 4000
+  - expr: count by (rustfs_cluster_id,pool_index) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{rustfs_cluster_id="a",pool_index="0"}'
+      value: 2
+    - labels: '{rustfs_cluster_id="a",pool_index="1"}'
+      value: 2
+    - labels: '{rustfs_cluster_id="b",pool_index="0"}'
+      value: 2
+    - labels: '{rustfs_cluster_id="b",pool_index="1"}'
+      value: 2
+  - expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n0"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{}'
+      value: 400
+  - expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n1"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{}'
+      value: 300
+- name: removed values and identities cannot rejoin new snapshots; stalled sources expire
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0 60 _ _ _ _ _ _ _
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="old"}
+    values: 1 1 _ _ _ _ _ _ _
+  - series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="new"}
+    values: _ _ 1 1 1 1 1 1 1
+  - series: rustfs_system_drive_waiting_io{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data"}
+    values: 7 7 _ _ _ _ _ _ _
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data"}
+    values: 100 100 _ _ _ _ _ _ _
+  promql_expr_test:
+  - expr: rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="old"}
+    eval_time: 2m
+    exp_samples: []
+  - expr: rustfs:storage:current{source_metric="rustfs_system_drive_waiting_io"}
+    eval_time: 2m
+    exp_samples: []
+  - expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="new"})
+    eval_time: 2m
+    exp_samples:
+    - labels: '{}'
+      value: 1
+  - expr: rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"}
+    eval_time: 5m
+    exp_samples: []
+  - expr: rustfs:storage_snapshot:fresh{observer="n1"}
+    eval_time: 5m
+    exp_samples: []
+- name: counters reset independently; filter versions before rate and sum rates across owners
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="n0-disk",api="read_all"}
+    values: 0 60 120 30 90 150 210
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0+60x8
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x8
+  - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",disk_id="n1-disk",api="read_all"}
+    values: 0+120x6
+  - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",server="n0",drive="/data",observer="old"}
+    values: 0+999x6
+  promql_expr_test:
+  - expr: sum(resets(rustfs_system_drive_api_calls_total{collection_scope="local"}[5m]))
+    eval_time: 5m
+    exp_samples:
+    - labels: '{}'
+      value: 1
+  - expr: sum(rate(rustfs_system_drive_api_calls_total{collection_scope="local"}[2m]) and ignoring(source_metric) rustfs:storage:current{source_metric="rustfs_system_drive_api_calls_total"})
+    eval_time: 5m
+    exp_samples:
+    - labels: '{}'
+      value: 3
+- name: completed slow collection has no validity budget
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: 0+60x3
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"}
+    values: '0x3'
+  promql_expr_test:
+  - expr: rustfs:storage_snapshot:fresh{observer="slow"}
+    eval_time: 2m
+    exp_samples: []
+- name: submillisecond publication and millisecond OTLP samples share a cutoff
+  interval: 1m
+  input_series:
+  - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"}
+    values: 0.0009+60x3
+  - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"}
+    values: 180x3
+  - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector",server="submillisecond",drive="/data"}
+    values: 100x3
+  promql_expr_test:
+  - expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",observer="submillisecond"})
+    eval_time: 2m15s
+    exp_samples:
+    - labels: '{}'
+      value: 1

.github/actions/schedule-failure-issue/README.md

@@ -5,20 +5,38 @@ This is the single alerting mechanism for all timed pipelines
 (rustfs/backlog#1149 ci-8, arbitration G3): scheduled workflows must consume
 this action instead of inventing their own notification paths.
 
+Issues are filed in **rustfs/backlog** (override with `target-repository`),
+not the repository the workflow runs in, so scheduled-failure noise stays
+out of the code repository's issue tracker.
+
 ## Behavior
 
 - Issue title: `[scheduled-failure] <workflow name>` — the workflow name is
   the dedupe key.
-- If an **open** issue with that exact title exists, the failure is appended
-  as a comment; otherwise a new issue is created (labeled `infrastructure` by
-  default). Closing the issue resets the cycle: the next failure opens a
-  fresh one.
-- The issue body / comment includes the run URL, run attempt, event, ref and
-  the names of the failed (or timed-out/cancelled) jobs of the current run
-  attempt.
+- If an **open** issue with that exact title exists in the target
+  repository, the failure is appended as a comment; otherwise a new issue is
+  created (labeled `infrastructure` by default; the label must exist in the
+  target repository). Closing the issue resets the cycle: the next failure
+  opens a fresh one.
+- The issue body / comment includes the source repository, run URL, run
+  attempt, event, ref and the names of the failed (or timed-out/cancelled)
+  jobs of the current run attempt.
 - Requires `gh` and `jq` on the runner (both preinstalled on GitHub-hosted
   runners such as `ubuntu-latest`).
 
+## Tokens
+
+Two tokens are involved:
+
+- `github-token` authenticates issue create/comment/search against the
+  target repository. The workflow-scoped `secrets.GITHUB_TOKEN` is scoped to
+  the code repository and **cannot** open issues in rustfs/backlog, so this
+  must be a PAT or GitHub App token with `issues: write` on the target
+  repository, stored as the `BACKLOG_ISSUE_TOKEN` repository secret.
+- `source-token` (defaults to `${{ github.token }}`) reads the failed-job
+  list from the reported run, which lives in the repository running the
+  workflow.
+
 ## Usage
 
 Add a final job to the workflow. `always()` is required — without it the job
@@ -41,7 +59,7 @@ this job only; no other job may gain permissions.
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
 ```
 
 Notes:

.github/actions/schedule-failure-issue/action.yml

@@ -16,18 +16,31 @@ name: "Schedule Failure Issue"
 description: >-
   Open (or update) a tracking issue when a scheduled workflow run fails or
   does not complete normally.
+  Issues are filed in the target repository (default rustfs/backlog), not
+  the repository the workflow runs in.
   Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the result is
+  "[scheduled-failure] <workflow name>" already exists there, the result is
   appended as a comment; otherwise a new issue is created. This is the
   single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
 
 inputs:
   github-token:
     description: >-
-      Token with issues:write on the repository. Pass secrets.GITHUB_TOKEN
-      from a job that declares `permissions: issues: write` (scope the
-      permission to the alert job only, never workflow-wide).
+      Token with issues:write on the target repository. The workflow-scoped
+      secrets.GITHUB_TOKEN is scoped to the code repository and cannot open
+      issues in rustfs/backlog, so pass a PAT / GitHub App token stored as a
+      repository secret (BACKLOG_ISSUE_TOKEN).
     required: true
+  target-repository:
+    description: "Repository the failure-tracking issue is filed in."
+    required: false
+    default: rustfs/backlog
+  source-token:
+    description: >-
+      Token used to read the failed-job list from the reported run. Defaults
+      to the workflow's own token, which can read runs of its own repository.
+    required: false
+    default: ${{ github.token }}
   workflow-name:
     description: "Workflow name used for the issue title (the dedupe key)."
     required: false
@@ -35,8 +48,8 @@ inputs:
   label:
     description: >-
       Label applied when a new issue is created. Must already exist in the
-      repository; if applying it fails, the issue is created without a label.
-      Set to an empty string to skip labeling.
+      target repository; if applying it fails, the issue is created without a
+      label. Set to an empty string to skip labeling.
     required: false
     default: "infrastructure"
   source-run-id:
@@ -71,6 +84,8 @@ runs:
       shell: bash
       env:
         GH_TOKEN: ${{ inputs.github-token }}
+        SOURCE_TOKEN: ${{ inputs.source-token }}
+        TARGET_REPOSITORY: ${{ inputs.target-repository }}
         WORKFLOW_NAME: ${{ inputs.workflow-name }}
         ISSUE_LABEL: ${{ inputs.label }}
         SOURCE_RUN_ID: ${{ inputs.source-run-id }}
@@ -86,8 +101,10 @@ runs:
         run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
 
         # Inspect the reported run attempt. It can be the current in-workflow
-        # failure or a completed run observed by the external watchdog.
-        failed_jobs="$(gh api \
+        # failure or a completed run observed by the external watchdog. The
+        # run lives in the repository running the workflow, so authenticate
+        # with the source token, not the target-repository token.
+        failed_jobs="$(GH_TOKEN="${SOURCE_TOKEN}" gh api \
           "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
           --paginate \
           --jq '.jobs[]
@@ -109,6 +126,7 @@ runs:
         body="$(cat <<EOF
         Run of **${WORKFLOW_NAME}** did not complete successfully.
 
+        - Repository: ${GITHUB_REPOSITORY}
         - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
         - Event: \`${SOURCE_EVENT}\`
         - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
@@ -120,26 +138,26 @@ runs:
         EOF
         )"
 
-        # Dedupe by exact title among OPEN issues (a closed issue means the
-        # earlier breakage was resolved; a new failure gets a fresh issue).
-        # GitHub search strips the [] characters, so search loosely and match
-        # the exact title with jq.
-        existing="$(gh issue list --repo "${GITHUB_REPOSITORY}" --state open --limit 100 \
+        # Dedupe by exact title among OPEN issues in the target repository (a
+        # closed issue means the earlier breakage was resolved; a new failure
+        # gets a fresh issue). GitHub search strips the [] characters, so
+        # search loosely and match the exact title with jq.
+        existing="$(gh issue list --repo "${TARGET_REPOSITORY}" --state open --limit 100 \
           --search "\"scheduled-failure\" in:title" --json number,title \
           | jq -r --arg title "${title}" 'map(select(.title == $title)) | (.[0].number // empty)')"
 
         if [ -n "${existing}" ]; then
-          echo "Appending comment to existing open issue #${existing}"
-          gh issue comment "${existing}" --repo "${GITHUB_REPOSITORY}" --body "${body}"
+          echo "Appending comment to existing open issue rustfs/backlog#${existing}"
+          gh issue comment "${existing}" --repo "${TARGET_REPOSITORY}" --body "${body}"
           exit 0
         fi
 
-        echo "Creating new issue: ${title}"
+        echo "Creating new issue in ${TARGET_REPOSITORY}: ${title}"
         if [ -n "${ISSUE_LABEL}" ]; then
-          if gh issue create --repo "${GITHUB_REPOSITORY}" \
+          if gh issue create --repo "${TARGET_REPOSITORY}" \
                --title "${title}" --body "${body}" --label "${ISSUE_LABEL}"; then
             exit 0
           fi
           echo "::warning::issue creation with label '${ISSUE_LABEL}' failed (missing label?); retrying without a label"
         fi
-        gh issue create --repo "${GITHUB_REPOSITORY}" --title "${title}" --body "${body}"
+        gh issue create --repo "${TARGET_REPOSITORY}" --title "${title}" --body "${body}"

.github/s3tests/s3tests.conf

@@ -55,6 +55,10 @@ secret_key = ${S3_SECRET_KEY}
 
 ## replace with key id obtained when secret is created, or delete if KMS not tested
 #kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
+#kms_keyid2 = fedcba98-7654-3210-fedc-ba9876543210
+
+## Expected service default for SSE-KMS requests without a key id; empty means none
+#kms_default_keyid =
 
 ## Storage classes
 #storage_classes = "LUKEWARM, FROZEN"

.github/workflows/audit.yml

@@ -207,4 +207,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/build.yml

@@ -207,12 +207,12 @@ jobs:
           fi
 
           all='{"include":[
-            {"target_id":"linux-x86_64-musl","os":"sm-standard-2","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":""},
-            {"target_id":"linux-aarch64-musl","os":"sm-standard-2","target":"aarch64-unknown-linux-musl","cross":true,"platform":"linux","rustflags":""},
-            {"target_id":"linux-x86_64-gnu","os":"sm-standard-2","target":"x86_64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":""},
-            {"target_id":"linux-aarch64-gnu","os":"sm-standard-2","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":""},
-            {"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":""},
-            {"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":""}
+            {"target_id":"linux-x86_64-musl","os":"sm-standard-4","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":""},
+            {"target_id":"linux-aarch64-musl","os":"sm-standard-4","target":"aarch64-unknown-linux-musl","cross":true,"platform":"linux","rustflags":"","features":""},
+            {"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":"","features":"pyroscope"},
+            {"target_id":"linux-aarch64-gnu","os":"sm-standard-4","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope"},
+            {"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope"},
+            {"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":""}
           ]}'
 
           if [[ "${selected}" == "all" ]]; then
@@ -406,12 +406,25 @@ jobs:
           # Force rebuild by touching build.rs
           touch rustfs/build.rs
 
-          if [[ "${{ matrix.cross }}" == "true" ]]; then
-            # All cross targets in the matrix are Linux; zigbuild handles them.
-            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bin rustfs
-          else
-            cargo build --release --target ${{ matrix.target }} -p rustfs --bin rustfs
+          BINARY_NAMES=(rustfs)
+          if [[ "${{ matrix.platform }}" == "linux" ]]; then
+            BINARY_NAMES+=(rustfs-cli)
           fi
+          FEATURE_ARGS=()
+          if [[ -n "${{ matrix.features }}" ]]; then
+            FEATURE_ARGS=(--features "${{ matrix.features }}")
+          fi
+
+          # Build one binary at a time so release LTO links cannot exhaust a
+          # self-hosted runner by running concurrently.
+          for binary in "${BINARY_NAMES[@]}"; do
+            if [[ "${{ matrix.cross }}" == "true" ]]; then
+              # All cross targets in the matrix are Linux; zigbuild handles them.
+              cargo zigbuild --release --target ${{ matrix.target }} "${FEATURE_ARGS[@]}" -p rustfs --bin "$binary"
+            else
+              cargo build --release --target ${{ matrix.target }} "${FEATURE_ARGS[@]}" -p rustfs --bin "$binary"
+            fi
+          done
 
       - name: Create release package
         id: package
@@ -498,17 +511,24 @@ jobs:
             BINARY_NAME="rustfs"
           fi
 
-          # Verify the binary exists before packaging
-          if [[ ! -f "$BINARY_NAME" ]]; then
-            echo "❌ Binary $BINARY_NAME not found in $(pwd)"
-            if [[ "${{ matrix.platform }}" == "windows" ]]; then
-              dir
-            else
-              ls -la
-            fi
-            exit 1
+          BINARY_FILES=("$BINARY_NAME")
+          if [[ "${{ matrix.platform }}" == "linux" ]]; then
+            BINARY_FILES+=("rustfs-cli")
           fi
 
+          # Verify the binaries exist before packaging
+          for binary in "${BINARY_FILES[@]}"; do
+            if [[ ! -f "$binary" ]]; then
+              echo "❌ Binary $binary not found in $(pwd)"
+              if [[ "${{ matrix.platform }}" == "windows" ]]; then
+                dir
+              else
+                ls -la
+              fi
+              exit 1
+            fi
+          done
+
           # Universal packaging function
           package_zip() {
             local src=$1
@@ -526,8 +546,12 @@ jobs:
           }
 
           # Create the zip package
-          echo "Start packaging: $BINARY_NAME -> ../../../${PACKAGE_NAME}.zip"
-          package_zip "$BINARY_NAME" "../../../${PACKAGE_NAME}.zip"
+          echo "Start packaging: ${BINARY_FILES[*]} -> ../../../${PACKAGE_NAME}.zip"
+          if [[ "${{ matrix.platform }}" == "linux" ]]; then
+            zip "../../../${PACKAGE_NAME}.zip" "${BINARY_FILES[@]}"
+          else
+            package_zip "$BINARY_NAME" "../../../${PACKAGE_NAME}.zip"
+          fi
 
           cd ../../..
 
@@ -601,6 +625,28 @@ jobs:
           echo "🔧 Build type: ${BUILD_TYPE}"
           echo "📊 Version: ${VERSION}"
 
+      - name: Verify packaged Linux CLI
+        if: matrix.platform == 'linux'
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          package_file="${{ steps.package.outputs.package_file }}"
+          archive_members="$(unzip -Z1 "$package_file" | sort)"
+          expected_members=$'rustfs\nrustfs-cli'
+          if [[ "$archive_members" != "$expected_members" ]]; then
+            echo "Linux package must contain exactly rustfs and rustfs-cli" >&2
+            printf '%s\n' "$archive_members" >&2
+            exit 1
+          fi
+
+          if [[ "${{ matrix.target }}" == "x86_64-unknown-linux-gnu" ]]; then
+            cli_dir="$(mktemp -d)"
+            trap 'rm -rf "$cli_dir"' EXIT
+            unzip -q "$package_file" rustfs-cli -d "$cli_dir"
+            "$cli_dir/rustfs-cli" --help >/dev/null
+          fi
+
       - name: Verify packaged console
         if: matrix.target == 'x86_64-unknown-linux-gnu'
         shell: bash
@@ -1100,4 +1146,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/ci.yml

@@ -259,9 +259,6 @@ jobs:
           } > artifacts/test-and-lint/doctest-diagnostics.txt
           exit "${status}"
 
-      - name: Check offline enrollment E2E root boundary
-        run: ./scripts/check_offline_enrollment_e2e.sh
-
       - name: Upload test reports and diagnostics
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
@@ -296,6 +293,37 @@ jobs:
       - name: Run rebalance/decommission migration proofs
         run: ./scripts/check_migration_gate_count.sh
 
+  # This gate builds into a fresh target directory to isolate the E2E root.
+  # Give its cold build a separate budget from workspace tests and migration proofs.
+  offline-enrollment-e2e:
+    name: Offline Enrollment E2E Root Boundary
+    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    needs: [ quick-checks, classify-changes ]
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+          install-test-tools: 'false'
+
+      - name: Protect Connect test home
+        run: chmod go-w "$(realpath "$HOME")"
+
+      - name: Check offline enrollment E2E root boundary
+        run: ./scripts/check_offline_enrollment_e2e.sh
+
   # Dedicated serial lane for the ILM / lifecycle integration tests. These tests
   # drive the object layer through process-global singletons (the GLOBAL_ENV
   # ECStore, the global tier-config manager, background-expiry workers) and bind
@@ -335,11 +363,7 @@ jobs:
       # re-enabled by backlog#1304 (restore accepts serialize on a short CAS
       # guard; the copy-back no longer holds the #4877 whole-copy-back lock,
       # so the mid-restore ongoing read and fast 409 rejection it asserts are
-      # the implemented contract). The remaining exclusions each hit a
-      # DIFFERENT, independent issue (all tracked under rustfs/backlog#1148;
-      # they keep #[ignore] with a backlog reference):
-      #   - test_noncurrent_{expiry,transition}_still_works_after_immediate_compensation_transition:
-      #     noncurrent transition/expiry after an immediate compensation transition.
+      # the implemented contract).
       - name: Run ignored ILM integration tests serially
         env:
           # Match the measured Test and Lint link budget. The default exposed
@@ -352,7 +376,7 @@ jobs:
           NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \
             cargo nextest run -j1 --run-ignored ignored-only \
             -p rustfs-scanner -p rustfs \
-            -E '(binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))) and not (test(test_noncurrent_expiry_still_works_after_immediate_compensation_transition) or test(test_noncurrent_transition_still_works_after_immediate_compensation_transition))' \
+            -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \
             --status-level all --final-status-level all \
             2>&1 | tee artifacts/ilm-integration/nextest.log
           status=${PIPESTATUS[0]}
@@ -568,7 +592,7 @@ jobs:
                       digest.update(chunk)
               return digest.hexdigest()
 
-          argv = ["cargo", "build", "-p", "rustfs", "--bins", "--features", "e2e-test-hooks"]
+          argv = ["python3", "scripts/e2e_binary.py", "build", "--bins", "--features", "e2e-test-hooks"]
           commit, tree = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^{tree}")
           clean_before = not git("status", "--porcelain", "--untracked-files=normal")
           if not clean_before:
@@ -602,6 +626,7 @@ jobs:
           name: rustfs-debug-binary
           path: |
             target/debug/rustfs
+            target/debug/rustfs.e2e.json
             target/debug/rustfs.e2e-startup-cas-build.json
           if-no-files-found: error
           retention-days: 1
@@ -634,13 +659,15 @@ jobs:
           install-build-packaging-tools: 'false'
 
       - name: Build debug binary with rio-v2
-        run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
+        run: python3 scripts/e2e_binary.py build --bins --features rio-v2,e2e-test-hooks
 
       - name: Upload debug binary
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: rustfs-debug-binary-rio-v2
-          path: target/debug/rustfs
+          path: |
+            target/debug/rustfs
+            target/debug/rustfs.e2e.json
           if-no-files-found: error
           retention-days: 1
 
@@ -789,7 +816,7 @@ jobs:
           NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
         run: |
-          cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
             --status-level all --final-status-level all --failure-output final
 
       - name: Upload e2e smoke diagnostics
@@ -825,7 +852,7 @@ jobs:
           RUSTFS_TEST_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
           RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT}" \
           RUSTFS_TEST_LOG="${RUN_ROOT}/rustfs.log" \
-          ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
 
       - name: Upload test logs
         if: failure()
@@ -953,6 +980,7 @@ jobs:
           if manifest["binary_sha256"] != digest.hexdigest() or manifest["commit"] != commit:
               raise SystemExit("downloaded hooks binary identity mismatch")
           shutil.copy2(manifest_path, target / manifest_path.name)
+          shutil.copy2(source.with_name("rustfs.e2e.json"), target / "rustfs.e2e.json")
           binary.chmod(0o755)
           PYINPUT
 
@@ -969,10 +997,11 @@ jobs:
       # debug binary; each test spawns its own rustfs server on a random port.
       - name: Run e2e full suite
         env:
+          CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
           RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
           RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
           RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
-        run: cargo nextest run --profile e2e-full -p e2e_test
+        run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
 
       - name: Upload junit
         if: always()
@@ -1044,7 +1073,7 @@ jobs:
       - name: Run end-to-end tests
         run: |
           s3s-e2e --version
-          ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
+          python3 scripts/e2e_binary.py run --features rio-v2,e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
 
       - name: Upload test logs
         if: failure()
@@ -1087,7 +1116,7 @@ jobs:
           S3_PORT="${S3_PORT}" \
           DATA_ROOT="${RUN_ROOT}" \
           S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
-          ./scripts/s3-tests/run.sh
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh
 
       - name: Upload s3 test artifacts
         if: always()
@@ -1169,7 +1198,7 @@ jobs:
           S3_PORT="${S3_PORT}" \
           DATA_ROOT="${RUN_ROOT}" \
           S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
-          ./scripts/s3-tests/run.sh
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh
 
       - name: Upload s3 test artifacts
         if: always()
@@ -1188,6 +1217,7 @@ jobs:
       - typos
       - quick-checks
       - test-and-lint
+      - offline-enrollment-e2e
       - test-ilm-integration-serial
       - test-and-lint-rio-v2
       - connect-short-credential-boundary
@@ -1221,6 +1251,7 @@ jobs:
       - typos
       - quick-checks
       - test-and-lint
+      - offline-enrollment-e2e
       - test-ilm-integration-serial
       - test-and-lint-rio-v2
       - test-and-lint-protocols
@@ -1247,4 +1278,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/connect-performance-drive-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect performance.drive service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  performance-drive:
+    name: Verify native Linux x86_64 performance.drive service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PERFORMANCE_DRIVE_EVIDENCE="$GITHUB_WORKSPACE/performance-drive-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=performance-drive
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('performance-drive-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            performance-drive-service-job-evidence.json >performance-drive-service-job-evidence.bound.json
+          mv performance-drive-service-job-evidence.bound.json performance-drive-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('performance-drive-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-performance-drive-evidence-${{ github.run_id }}
+          path: |
+            performance-drive-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-performance-network-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect performance.network service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  performance-network:
+    name: Verify native Linux x86_64 performance.network service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PERFORMANCE_NETWORK_EVIDENCE="$GITHUB_WORKSPACE/performance-network-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=performance-network
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('performance-network-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            performance-network-service-job-evidence.json >performance-network-service-job-evidence.bound.json
+          mv performance-network-service-job-evidence.bound.json performance-network-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('performance-network-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-performance-network-evidence-${{ github.run_id }}
+          path: |
+            performance-network-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-profile-cpu-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect profile.cpu service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  profile-cpu:
+    name: Verify native Linux x86_64 profile.cpu service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PROFILE_EVIDENCE="$GITHUB_WORKSPACE/profile-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=profile
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('profile-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            profile-service-job-evidence.json >profile-service-job-evidence.bound.json
+          mv profile-service-job-evidence.bound.json profile-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('profile-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-profile-cpu-evidence-${{ github.run_id }}
+          path: |
+            profile-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-profile-threads-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect profile.threads service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  profile-threads:
+    name: Verify native Linux x86_64 profile.threads service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PROFILE_THREADS_EVIDENCE="$GITHUB_WORKSPACE/profile-threads-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=profile-threads
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('profile-threads-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            profile-threads-service-job-evidence.json >profile-threads-service-job-evidence.bound.json
+          mv profile-threads-service-job-evidence.bound.json profile-threads-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('profile-threads-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-profile-threads-evidence-${{ github.run_id }}
+          path: |
+            profile-threads-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-top-api-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect top.api service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  top-api:
+    name: Verify native Linux x86_64 top.api service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_TOP_API_EVIDENCE="$GITHUB_WORKSPACE/top-api-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=top-api
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('top-api-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            top-api-service-job-evidence.json >top-api-service-job-evidence.bound.json
+          mv top-api-service-job-evidence.bound.json top-api-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('top-api-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-top-api-evidence-${{ github.run_id }}
+          path: |
+            top-api-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-top-disk-acceptance.yml

@@ -0,0 +1,142 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect top.disk artifact acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Successful main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  top-disk:
+    name: Verify native Linux x86_64 top.disk artifact
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.conclusion' <<<"$run") == success ]]
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+
+      - name: Run top.disk acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+        run: |
+          set -euo pipefail
+          scripts/ci/check_connect_top_disk_artifact.sh \
+            binary/rustfs "$SOURCE_SHA" "$BINARY_SHA256" \
+            top-disk-runtime-evidence.json top-disk.zip
+
+      - name: Bind workflow and artifact provenance
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest}' \
+            top-disk-runtime-evidence.json >top-disk-runtime-evidence.bound.json
+          mv top-disk-runtime-evidence.bound.json top-disk-runtime-evidence.json
+
+      - name: Upload acceptance evidence
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-top-disk-evidence-${{ github.run_id }}
+          path: |
+            top-disk-runtime-evidence.json
+            top-disk.zip
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-top-locks-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect top.locks service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  top-locks:
+    name: Verify native Linux x86_64 top.locks service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_TOP_LOCKS_EVIDENCE="$GITHUB_WORKSPACE/top-locks-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=top-locks
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('top-locks-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            top-locks-service-job-evidence.json >top-locks-service-job-evidence.bound.json
+          mv top-locks-service-job-evidence.bound.json top-locks-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('top-locks-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-top-locks-evidence-${{ github.run_id }}
+          path: |
+            top-locks-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-top-rpc-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect top.rpc service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  top-rpc:
+    name: Verify native Linux x86_64 top.rpc service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_TOP_RPC_EVIDENCE="$GITHUB_WORKSPACE/top-rpc-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=top-rpc
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('top-rpc-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            top-rpc-service-job-evidence.json >top-rpc-service-job-evidence.bound.json
+          mv top-rpc-service-job-evidence.bound.json top-rpc-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('top-rpc-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-top-rpc-evidence-${{ github.run_id }}
+          path: |
+            top-rpc-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/coverage.yml

@@ -140,4 +140,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/e2e-distributed.yml

@@ -151,8 +151,7 @@ jobs:
 
       - name: Build rustfs binary
         run: |
-          cargo build -p rustfs --bins
-          : > target/debug/rustfs.features
+          python3 scripts/e2e_binary.py build --bins
 
       - name: Verify distributed e2e membership
         env:
@@ -168,9 +167,9 @@ jobs:
         run: |
           set -euo pipefail
           if [ -n "${FILTER}" ]; then
-            cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
+            python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
           else
-            cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
+            python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
           fi
 
       - name: Upload distributed e2e diagnostics
@@ -213,4 +212,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/e2e-replication-nightly.yml

@@ -89,14 +89,10 @@ jobs:
       - name: Verify awscurl
         run: test -x "$AWSCURL_PATH"
 
-      # Build the rustfs binary once up front. The e2e tests spawn it as a
-      # child process (crates/e2e_test/src/common.rs) and will build it on
-      # demand otherwise, but a single explicit build avoids several parallel
-      # nextest test processes racing to build it at once.
+      # Build once and carry its source/binary identity into the test invocation.
       - name: Build rustfs binary
         run: |
-          cargo build -p rustfs --bins
-          : > target/debug/rustfs.features
+          python3 scripts/e2e_binary.py build --bins
 
       - name: Verify replication e2e membership
         env:
@@ -108,7 +104,7 @@ jobs:
       - name: Run replication e2e nightly suite
         env:
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
-        run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
+        run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
 
       - name: Upload nextest junit report
         if: always()
@@ -144,8 +140,7 @@ jobs:
 
       - name: Build rustfs binary
         run: |
-          cargo build -p rustfs --bins --features e2e-test-hooks
-          : > target/debug/rustfs.features
+          python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
 
       - name: Verify cluster fault e2e membership
         env:
@@ -156,8 +151,9 @@ jobs:
 
       - name: Run cluster fault e2e nightly suite
         env:
+          CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
-        run: cargo nextest run --profile e2e-nightly -p e2e_test
+        run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
 
       - name: Upload cluster fault diagnostics
         if: always()
@@ -198,6 +194,9 @@ jobs:
           sudo apt-get install -y -qq iproute2
           ss -tn state CLOSE-WAIT >/dev/null
 
+      - name: Build protocol server
+        run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
+
       # The suite owns fixed protocol ports and serializes its internal cases.
       - name: Verify protocol e2e membership
         env:
@@ -210,7 +209,7 @@ jobs:
         env:
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
         run: >-
-          cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
+          python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
 
       - name: Upload protocol diagnostics
         if: always()
@@ -243,4 +242,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/e2e-s3tests.yml

@@ -460,4 +460,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/e2e-upgrade.yml

@@ -125,15 +125,25 @@ jobs:
 
       - name: Build current RustFS binary
         run: |
-          cargo build --locked -p rustfs --bin rustfs
-          : > target/debug/rustfs.features
+          python3 scripts/e2e_binary.py build
 
       - name: Run upgrade compatibility test
+        env:
+          RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
         run: |
-          cargo test --locked -p e2e_test \
+          python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \
             "upgrade_compatibility_test::${{ matrix.test }}" \
             -- --ignored --exact --nocapture
 
+      - name: Upload scanner/heal G09 evidence
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: ${{ matrix.artifact }}-g09-evidence-${{ github.run_number }}
+          path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
+          if-no-files-found: ignore
+          retention-days: 14
+
       - name: Upload server logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6

.github/workflows/functional-chain-health.yml

@@ -0,0 +1,31 @@
+name: Functional chain health
+on:
+  schedule:
+    - cron: '23 * * * *'
+  workflow_dispatch:
+permissions:
+  contents: read
+  actions: read
+concurrency:
+  group: functional-chain-health
+  cancel-in-progress: false
+jobs:
+  collect:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Collect and publish verified chain health
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          NIGHTLY_SOURCE_REF: ${{ vars.NIGHTLY_BRANCH || 'main' }}
+        run: python3 scripts/functional_chain_health.py --source-ref "${NIGHTLY_SOURCE_REF}" --publish --output "${RUNNER_TEMP}/chain-health.json"
+      - name: Retain health observation
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-health-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-health.json
+          if-no-files-found: error

.github/workflows/fuzz.yml

@@ -252,4 +252,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/minio-interop.yml

@@ -143,4 +143,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/mint.yml

@@ -270,4 +270,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/nightly-gnu.yml

@@ -42,18 +42,39 @@ env:
   NIGHTLY_BUILD_REF: ${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
 
 jobs:
+  resolve-source:
+    name: Resolve nightly source
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      source_sha: ${{ steps.source.outputs.sha }}
+      source_ref: ${{ steps.source.outputs.ref }}
+    steps:
+      - name: Checkout selected source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+          ref: ${{ env.NIGHTLY_BUILD_REF }}
+      - name: Record immutable source
+        id: source
+        run: |
+          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          echo "ref=${NIGHTLY_BUILD_REF}" >> "$GITHUB_OUTPUT"
+
   build:
+    needs: resolve-source
     name: Build x86_64 GNU
     runs-on: sm-standard-4
     timeout-minutes: 150
     env:
+      NIGHTLY_BUILD_REF: ${{ needs.resolve-source.outputs.source_ref }}
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: ${{ env.NIGHTLY_BUILD_REF }}
+          ref: ${{ needs.resolve-source.outputs.source_sha }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -334,9 +355,10 @@ jobs:
 
           CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
           jq -n --arg source_sha "${SOURCE_SHA}" \
+            --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${NIGHTLY_BUILD_REF}" \
             --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
             --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
-            '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
+            '{schema: 2, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
             > "${CANDIDATE_FILE}"
           echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
 
@@ -352,69 +374,16 @@ jobs:
       # engineers can download and install the nightly directly. The branch is
       # a single-commit orphan rewritten on every build, which keeps the repo
       # small while the latest files stay reachable at stable raw URLs.
-      - name: Publish packages to auto-testing assets
+      # Publish the deb/rpm pair as assets of the rolling `nightly` release on
+      # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           DEB_FILE: ${{ steps.deb.outputs.deb_file }}
           RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
-        run: |
-          set -euo pipefail
-
-          if [ -z "${ASSETS_TOKEN}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping assets upload"
-            exit 0
-          fi
-          export GH_TOKEN="${ASSETS_TOKEN}"
-
-          for f in "${DEB_FILE}" "${RPM_FILE}"; do
-            [ -f "$f" ] || { echo "missing package: $f"; exit 1; }
-          done
-
-          # Plain git + token URL: the build fleet has no gh CLI.
-          ASSETS_URL="https://x-access-token:${ASSETS_TOKEN}@github.com/rustfs/auto-testing.git"
-          rm -rf assets-work && mkdir assets-work
-          if git clone -q --depth 1 --branch assets "${ASSETS_URL}" assets-work 2>/dev/null; then
-            echo "assets branch cloned"
-          else
-            echo "assets branch does not exist yet; creating an orphan"
-            git -C assets-work init -q -b assets
-          fi
-          cd assets-work
-          git remote add origin "${ASSETS_URL}" 2>/dev/null || \
-            git remote set-url origin "${ASSETS_URL}"
-
-          mkdir -p nightly
-          cp "../${DEB_FILE}" "nightly/${DEB_FILE}"
-          cp "../${RPM_FILE}" "nightly/${RPM_FILE}"
-          cp "nightly/${DEB_FILE}" nightly/rustfs-nightly-latest.deb
-          cp "nightly/${RPM_FILE}" nightly/rustfs-nightly-latest.rpm
-
-          SOURCE_SHA="$(git -C .. rev-parse HEAD 2>/dev/null || echo "${GITHUB_SHA}")"
-          {
-            echo "# Nightly packages"
-            echo ""
-            echo "- Built: ${DEB_DATE} from \`${BUILD_REF}@${SOURCE_SHA:0:12}\`"
-            echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-            echo ""
-            echo '| File | Size | SHA256 |'
-            echo '|---|---|---|'
-            for f in "nightly/${DEB_FILE}" "nightly/${RPM_FILE}" nightly/rustfs-nightly-latest.deb nightly/rustfs-nightly-latest.rpm; do
-              printf '| %s | %s | %s |\n' "$f" "$(du -h "$f" | cut -f1)" "$(sha256sum "$f" | cut -d' ' -f1)"
-            done
-            echo ""
-            echo "Download: replace /blob/ with /raw/ in any file URL, e.g."
-            echo "\`https://raw.githubusercontent.com/rustfs/auto-testing/assets/nightly/rustfs-nightly-latest.deb\`"
-          } > BUILD-INFO.md
-
-          git add -A
-          git -c user.name="rustfs-nightly-bot" -c user.email="support@rustfs.com" \
-            commit -q -m "nightly ${DEB_DATE} (${BUILD_REF}@${SOURCE_SHA:0:12})" \
-            --allow-empty
-          git push --force origin assets
-          echo "✅ Published ${DEB_FILE} and ${RPM_FILE} to rustfs/auto-testing@assets"
-
+        run: bash scripts/release/publish_nightly_assets.sh
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
@@ -431,6 +400,7 @@ jobs:
   # self-hosted fleet is heterogeneous — a docker-dependent workflow has been
   # burned by it before (see the banner in e2e-s3tests.yml, rustfs/backlog#1149).
   kms-vault-lane:
+    needs: resolve-source
     name: KMS live Vault lane
     runs-on: ubuntu-latest
     timeout-minutes: 90
@@ -452,7 +422,7 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: ${{ env.NIGHTLY_BUILD_REF }}
+          ref: ${{ needs.resolve-source.outputs.source_sha }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -530,6 +500,7 @@ jobs:
   # flake cannot mask the main lane's verdict, and vice versa. The script
   # provisions and tears down its own Docker cluster.
   kms-vault-ha-failover:
+    needs: resolve-source
     name: KMS Vault HA failover lane
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -541,7 +512,7 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: ${{ env.NIGHTLY_BUILD_REF }}
+          ref: ${{ needs.resolve-source.outputs.source_sha }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -572,4 +543,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/oidc-keycloak.yml

@@ -107,4 +107,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/on-demand-migration-interop.yml

@@ -132,7 +132,7 @@ jobs:
             s3api create-bucket --bucket "${RUSTFS_ODM_INTEROP_BUCKET}"
 
       - name: Build the RustFS binary under test
-        run: cargo build --locked -p rustfs --bins
+        run: python3 scripts/e2e_binary.py build --bins
 
       # The lane selects tests by module, so a rename would quietly shrink it.
       # The committed digest in .config/e2e-odm-interop-selection.txt fails
@@ -143,7 +143,7 @@ jobs:
           python3 ./scripts/check_test_wiring.py --check-profile e2e-odm-interop "${NEXTEST_LISTING}"
 
       - name: Run the interop cases against MinIO
-        run: cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
+        run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
 
       - name: Build the MinIO interop report
         if: always()
@@ -251,7 +251,7 @@ jobs:
 
       - name: Build the RustFS binary under test
         if: steps.credentials.outputs.present == 'true'
-        run: cargo build --locked -p rustfs --bins
+        run: python3 scripts/e2e_binary.py build --bins
 
       # A filterset that matches nothing is valid, so the count is asserted
       # rather than inferred from a green run.
@@ -272,7 +272,7 @@ jobs:
       - name: Run the three-case minimum
         if: steps.credentials.outputs.present == 'true'
         run: |
-          cargo nextest run --profile e2e-odm-interop -p e2e_test \
+          python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test \
             -E "${CLOUD_CASE_FILTER}" --no-tests=fail
 
       - name: Build the ${{ matrix.provider }} interop report
@@ -312,4 +312,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/performance-ab.yml

@@ -378,4 +378,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/runner-hygiene.yml

@@ -78,4 +78,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/rustfs-fault-tolerance-test.yml

@@ -0,0 +1,365 @@
+# RustFS Fault-Tolerance (degradation) Test
+#
+# Scenario suite for the 2026-09 degradation report: verifies read/write
+# behavior under drive and node loss against the erasure-coding contract and
+# snapshots health-endpoint responses at every tier.
+#
+#   A  single-node 4 drives (SNMD): hide 1/2/3 drives, restore
+#   B  multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
+#   C  multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
+#      stop 2 nodes, restore
+#   C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
+#      satisfy the EC read quorum while the lock majority is broken (the
+#      reported divergence window: reads 503 with lock_quorum_unavailable)
+#
+# Expectations come from product source (default_parity_count, erasure set
+# sizing). By default a "reads refused although the read quorum is met"
+# observation is reported as known-divergence without failing the suite; the
+# strict input turns those into failures once the product behavior changes.
+
+name: RustFS Fault-Tolerance Test
+
+on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      package_url:
+        description: 'Direct .deb URL. Required unless the nightly default is wanted.'
+        required: false
+        type: string
+      strict:
+        description: 'Fail the suite when reads are refused despite a met read quorum'
+        type: boolean
+        default: false
+      cleanup_before:
+        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
+        type: boolean
+        default: true
+      cleanup_after:
+        description: 'Reset the nodes after the test (DESTROYS test data/config)'
+        type: boolean
+        default: true
+  repository_dispatch:
+    # Chain handoff: dispatched when the replication suite finishes, ahead of
+    # the performance suite.
+    types: [rustfs-chain-fault-tolerance]
+
+permissions:
+  contents: read
+
+# The suite stops services and hides drive dirs on the shared fleet; only one
+# functional suite may touch the environment at a time.
+concurrency:
+  group: rustfs-shared-functional-tests-v2
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+
+jobs:
+  fault-tolerance-test:
+    runs-on: smoke-testing
+    timeout-minutes: 60
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-fault-tolerance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
+      # Pin the private suite checkout when the chain supplies a candidate.
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        timeout-minutes: 5
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          aws --version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        timeout-minutes: 5
+        if: ${{ inputs.cleanup_before != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
+
+      - name: Stage verified candidate package
+        id: chain_package
+        if: ${{ inputs.chain_manifest != '' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/prepare_functional_package.py prepare
+
+      - name: Run fault-tolerance scenarios (A, B, C, C2)
+        timeout-minutes: 45
+        id: test
+        # Standalone case failures are reported by the backlog manager.
+        # Chain evidence separately requires complete passing cases.
+        continue-on-error: true
+        run: |
+          ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
+          if [ "${{ inputs.strict }}" = "true" ]; then
+            ARGS+=(--strict)
+          fi
+          ./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        timeout-minutes: 2
+        id: chain_report
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          {
+            echo "# RustFS fault-tolerance test report"
+            echo ""
+            echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+            echo "- Strict mode: ${{ inputs.strict || 'false' }}"
+            echo ""
+            echo "## Per-probe results"
+            echo ""
+            echo '```'
+            grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
+            echo '```'
+            echo ""
+            echo "## Health snapshots"
+            echo ""
+            for f in "${FUNCTIONAL_ARTIFACTS_DIR}"/evidence/*.code; do
+              [ -e "${f}" ] || continue
+              printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")"
+            done
+          } > "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          # Product gate: failing FT cases keep the run green — the report above and
+          # the backlog issue manager carry the signal. Only harness/environment
+          # breakdowns (no case verdicts at all) turn the workflow red.
+          FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
+          FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
+          FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
+          HARNESS_OK=0
+          if [ "${{ steps.test.outcome }}" = "success" ]; then
+            HARNESS_OK=1
+          elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
+          [ "${HARNESS_OK}" = "1" ]
+
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Signal-based lifecycle: dedups against open issues by label
+        # (fault-tolerance + FT case), labels new issues (functional-test,
+        # category, case IDs, env), and closes fixed issues after a fully
+        # green run. Never acts on cancelled runs. Logic lives in
+        # auto-testing/scripts/issue_manager.py, which parses the
+        # FT-CASE verdict lines from the suite log.
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'fault-tolerance'
+          SUITE_LABEL: 'Fault-Tolerance'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
+            exit 0
+          fi
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
+            exit 0
+          fi
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
+
+      - name: Upload functional report to dashboard
+        timeout-minutes: 2
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: ${{ env.REPORT_FILE }}
+          SUITE: fault-tolerance
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+          echo "dashboard updated: ${REPORT_PATH}"
+
+      - name: Upload test logs & evidence
+        timeout-minutes: 2
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        timeout-minutes: 5
+        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
+
+      - name: "Continue functional chain (next: Performance)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff retries, then files an alert issue in rustfs/backlog.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-performance' \
+              -F 'client_payload[from_suite]=fault-tolerance'; then
+              echo "dispatched next suite Performance (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after fault-tolerance (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **fault-tolerance** to **Performance** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS fault-tolerance test failed"
+          echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+          echo "See the uploaded log artifact and FT-CASE lines for details."
+
+      - name: Remove verified candidate package
+        if: ${{ always() && inputs.chain_manifest != '' && steps.chain.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/prepare_functional_package.py cleanup
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite fault-tolerance
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/suite.log"
+          --output "${RUNNER_TEMP}/chain-fault-tolerance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/fault-tolerance.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}/fault-tolerance.json
+          if-no-files-found: error

.github/workflows/rustfs-functional-chain.yml

@@ -12,50 +12,224 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Functional chain driver: runs the ten functional suites in a fixed order
-# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication -> performance). Each suite attempts the next handoff even
-# when its tests fail.
-#
-# Each suite workflow can still be dispatched standalone (workflow_dispatch);
-# only chain-triggered runs forward to the next suite via repository_dispatch,
-# so a standalone run never drags the rest of the chain behind it.
-#
-# Why not workflow_run chaining: GitHub does not guarantee delivery of
-# workflow_run events (they are fire-and-forget), and the head-SHA filter made
-# newly added suites (storage) unable to trigger at all. Explicit
-# repository_dispatch handoffs are verifiable and re-drivable.
-
+# Reusable workflows run from this driver's commit in one Actions run. Each
+# suite still runs after an earlier suite fails; the final job requires all twelve.
 name: RustFS Functional Chain
 
 on:
   workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Successful nightly build run on main
+        type: string
+        required: true
+      build_run_attempt:
+        description: Exact successful build attempt
+        type: string
+        required: true
   workflow_run:
-    # Entry point: start the chain after the nightly build completes. The
-    # build's own conclusion does not gate the chain; each suite reports its
-    # own result to rustfs/backlog and the dashboard.
     workflows: ["Nightly GNU Build"]
     types: [completed]
 
 permissions:
   contents: read
+  actions: read
+
+concurrency:
+  group: rustfs-functional-chain-runs
+  cancel-in-progress: false
 
 jobs:
-  start-chain:
-    name: Start functional chain (upgrade first)
+  prepare:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule') }}
+    outputs:
+      manifest: ${{ steps.candidate.outputs.manifest }}
+      performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}
     steps:
-      - name: Dispatch first suite (upgrade)
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Resolve published candidate
+        id: candidate
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          CHAIN_OUTPUT: ${{ runner.temp }}/chain-candidate.json
+        run: python3 scripts/resolve_functional_candidate.py
+      - name: Retain candidate identity
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-candidate.json
+          if-no-files-found: error
+
+      - name: Check shared functional fleet runner before scheduling suites
+        # The eleven functional suites run on the smoke-testing fleet; a hard
+        # requirement. The performance lane runs on pf-testing, which is a
+        # separate fleet - probe it without failing the whole chain so an
+        # offline performance runner skips only the performance lane.
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: python3 scripts/check_functional_runners.py smoke-testing
+
+      - name: Probe performance fleet runner
+        id: perf_probe
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot start the functional chain" >&2
-            exit 1
+          if python3 scripts/check_functional_runners.py pf-testing; then
+            echo "performance_ready=online" >> "$GITHUB_OUTPUT"
+          else
+            echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
           fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-upgrade' \
-            -F 'client_payload[from_suite]=nightly-build'
+
+  upgrade:
+    needs: [prepare]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-upgrade-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  s3:
+    needs: [prepare, upgrade]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-s3-compat-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  kms:
+    needs: [prepare, s3]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-kms-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  tier:
+    needs: [prepare, kms]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-tier-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  storage:
+    needs: [prepare, tier]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-storage-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  heal:
+    needs: [prepare, storage]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-heal-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  pool:
+    needs: [prepare, heal]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-pool-expand-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  security:
+    needs: [prepare, pool]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-security-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  replication:
+    needs: [prepare, security]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-replication-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  fault-tolerance:
+    needs: [prepare, replication]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-fault-tolerance-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  table:
+    needs: [prepare, fault-tolerance]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-table-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  performance:
+    needs: [prepare, table]
+    if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.performance_ready == 'online' }}
+    uses: ./.github/workflows/rustfs-performance-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  complete-chain:
+    needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, fault-tolerance, table, performance]
+    if: ${{ always() && needs.prepare.result != 'skipped' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Download suite evidence
+        id: download
+        continue-on-error: true
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          pattern: functional-chain-*-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-evidence
+          merge-multiple: true
+      - name: Summarize every suite including failures and missing evidence
+        if: always()
+        env:
+          CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
+          CHAIN_NEEDS: ${{ toJSON(needs) }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py summarize
+          --directory "${RUNNER_TEMP}/chain-evidence"
+          --output "${RUNNER_TEMP}/chain-report.json"
+      - name: Retain chain report regardless of test verdict
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-report-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/chain-report.json
+            ${{ runner.temp }}/chain-report.md
+          if-no-files-found: error
+      - name: Verify every required suite
+        if: always()
+        env:
+          CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
+          CHAIN_NEEDS: ${{ toJSON(needs) }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py aggregate
+          --allow-skipped "${{ needs.prepare.outputs.performance_ready == 'online' && '' || 'performance' }}"
+          --directory "${RUNNER_TEMP}/chain-evidence"
+          --output "${RUNNER_TEMP}/chain-complete.json"
+      - name: Upload complete-chain evidence
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-complete-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-complete.json
+          if-no-files-found: error

.github/workflows/rustfs-heal-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Heal Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       package_url:
@@ -35,7 +41,7 @@ permissions:
 # Only one test at a time: both this and the pool-expansion workflow mutate
 # the same test environment, so they share one concurrency group.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -54,11 +60,16 @@ env:
 jobs:
   heal-test:
     runs-on: smoke-testing
-    timeout-minutes: 480
+    timeout-minutes: 60
     # Standalone manual run, or one link of the nightly functional chain
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
       - name: Initialize functional evidence
         id: evidence
         run: |
@@ -74,25 +85,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -103,6 +110,7 @@ jobs:
           df -h /data | tail -1
 
       - name: Cleanup environment (before)
+        timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
           set -euo pipefail
@@ -122,6 +130,7 @@ jobs:
           done
 
       - name: Install RustFS package & start cluster
+        timeout-minutes: 5
         run: |
           ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
           if [ -n "${{ inputs.package_url }}" ]; then
@@ -132,6 +141,7 @@ jobs:
           ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
+        timeout-minutes: 5
         run: |
           ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
           if [ -n "${{ inputs.package_url }}" ]; then
@@ -142,7 +152,11 @@ jobs:
           ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
+        timeout-minutes: 45
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           ./auto-testing/rustfs_heal_test.sh \
             --steps "3,4,5,6,7" -y \
@@ -152,6 +166,8 @@ jobs:
             --log-file "${LOG_FILE}"
 
       - name: Generate report
+        id: chain_report
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -224,6 +240,20 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing steps keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns turn the workflow red.
+          STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)"
+          STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)"
+          STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 ))
+          STEPS_PASS=$(( STEPS_TOTAL - STEPS_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${STEPS_TOTAL}" -gt 0 ] && [ "${STEPS_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS heal test report"
             echo ""
@@ -235,21 +265,27 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${STEPS_TABLE}" ]; then
               cat "${STEPS_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${STEPS_PASS} passed, ${STEPS_FAIL} failed (failing steps are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${STEPS_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; step failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
@@ -280,67 +316,55 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'heal'
           SUITE_LABEL: 'Heal'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload test logs
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
@@ -353,6 +377,7 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        timeout-minutes: 5
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
           set -euo pipefail
@@ -426,3 +451,24 @@ jobs:
           echo "RustFS heal test failed"
           echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite heal
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
+          --output "${RUNNER_TEMP}/chain-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/heal.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json
+          if-no-files-found: error

.github/workflows/rustfs-kms-test.yml

@@ -1,12 +1,18 @@
 name: RustFS KMS Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -31,7 +37,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -49,8 +55,8 @@ env:
 jobs:
   kms-test:
     runs-on: smoke-testing
-    timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    timeout-minutes: 60
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -71,25 +77,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -99,6 +101,7 @@ jobs:
           docker --version || true
 
       - name: Cleanup environment (before)
+        timeout-minutes: 5
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -127,6 +130,10 @@ jobs:
 
       - name: Run KMS suite
         id: test
+        timeout-minutes: 45
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-kms-test.sh
@@ -156,6 +163,8 @@ jobs:
           ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -175,6 +184,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS KMS test report"
             echo ""
@@ -186,21 +210,27 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
@@ -231,67 +261,59 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'kms'
           SUITE_LABEL: 'KMS'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
@@ -303,6 +325,7 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        timeout-minutes: 5
         if: always()
         run: |
           set -euo pipefail
@@ -375,3 +398,24 @@ jobs:
         run: |
           echo "RustFS KMS suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite kms
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/kms.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-kms-${{ github.run_id }}-${{ github.run_attempt }}/kms.json
+          if-no-files-found: error

.github/workflows/rustfs-performance-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Performance Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       package_url:
@@ -58,7 +64,7 @@ permissions:
 # The default performance nodes overlap the other suites' remote VMs, even
 # though the runner differs. Hold the shared lock through cleanup as well.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -79,7 +85,21 @@ env:
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
 jobs:
+  runner-check:
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Check performance runner before entering its queue
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: python3 scripts/check_functional_runners.py pf-testing
+
   performance-test:
+    needs: runner-check
     runs-on: pf-testing
     timeout-minutes: 900
     env:
@@ -90,8 +110,13 @@ jobs:
       RUSTFS_WARP_CONCURRENCY: ${{ inputs.warp_concurrency || '64' }}
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
       - name: Initialize functional evidence
         id: evidence
         run: |
@@ -108,25 +133,21 @@ jobs:
             printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -192,6 +213,7 @@ jobs:
           } > "${VERSION_FILE}"
 
       - name: Upload report to dashboard (reports/YYYY-MM-DD.md)
+        id: chain_report
         if: ${{ steps.benchmark.conclusion == 'success' }}
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
@@ -237,65 +259,48 @@ jobs:
             echo "created ${REPORT_PATH} in rustfs/dashboard"
           fi
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.benchmark.outcome == 'failure' || steps.benchmark.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Signal-based lifecycle: dedups against open issues by label,
+        # labels new issues (functional-test, category, case IDs, env),
+        # closes fixed issues after a fully green run. Never acts on
+        # cancelled runs. Logic lives in auto-testing/scripts/.
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'performance'
           SUITE_LABEL: 'Performance'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.benchmark.outcome }}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload test logs & results
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -325,3 +330,24 @@ jobs:
           echo "RustFS performance test failed"
           echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.benchmark.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite performance
+          --report "${RUSTFS_RESULT_DIR}/summary.tsv"
+          --output "${RUNNER_TEMP}/chain-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/performance.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-performance-${{ github.run_id }}-${{ github.run_attempt }}/performance.json
+          if-no-files-found: error

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Pool Expansion Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
@@ -52,7 +58,7 @@ permissions:
 # environment (vm000/vm001/vm002), so concurrent runs must not clobber each
 # other.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -63,7 +69,7 @@ env:
   RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
   RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
   RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002' }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
   # Package used by the nightly run (workflow_dispatch inputs are empty for
@@ -76,36 +82,38 @@ jobs:
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
-    timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    timeout-minutes: 60
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     env:
       RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
       RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
       RUSTFS_POOL_WARP_ENDPOINT: http://127.0.0.1:19000
       RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
-      RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
+      RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000 http://rustfs-node4:9000' }}
     steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Initialize pool test artifacts
+        id: evidence
         run: |
           set -euo pipefail
           ARTIFACT_DIR="${RUNNER_TEMP}/rustfs-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
@@ -120,7 +128,19 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
+      - name: Validate pool topology before destructive cleanup
+        id: topology
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES}"
+          read -r -a ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
+          if [ "${#NODES[@]}" -lt 3 ] || [ "${#ENDPOINTS[@]}" -lt "${#NODES[@]}" ]; then
+            echo "Pool tests require at least three nodes and a direct endpoint for every node" >&2
+            exit 1
+          fi
+
       - name: Cleanup environment (before)
+        timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
           set -euo pipefail
@@ -140,6 +160,7 @@ jobs:
           done
 
       - name: Install RustFS package & start cluster
+        timeout-minutes: 5
         run: |
           ARGS=(--steps "1,2,3" -y \
             --admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
@@ -156,6 +177,7 @@ jobs:
           ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
 
       - name: Preflight checks
+        timeout-minutes: 5
         run: |
           ARGS=(--preflight \
             --admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
@@ -172,6 +194,7 @@ jobs:
           ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
 
       - name: Reset dedicated pool proxy
+        timeout-minutes: 2
         run: |
           set -euo pipefail
           RUSTFS_POOL_NGINX_CONFIG_PATH=/etc/nginx/conf.d/rustfs-pool-test.conf \
@@ -181,6 +204,7 @@ jobs:
             ./auto-testing/rustfs_pool_nginx_stage.sh cleanup
 
       - name: Capture pool test baseline
+        timeout-minutes: 2
         run: |
           set -uo pipefail
           BASELINE_FILE="${POOL_ARTIFACT_DIR}/pool-baseline.log"
@@ -227,6 +251,10 @@ jobs:
 
       - name: Run pool expansion & decommission test
         id: pool_test
+        timeout-minutes: 45
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -o pipefail
           STEPS="4,5,6"
@@ -263,6 +291,7 @@ jobs:
             ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
 
       - name: Collect pool test diagnostics
+        timeout-minutes: 2
         if: always()
         run: |
           set -uo pipefail
@@ -363,6 +392,8 @@ jobs:
           fi
 
       - name: Generate report
+        id: chain_report
+        timeout-minutes: 2
         if: always()
         run: |
           set -euo pipefail
@@ -450,6 +481,7 @@ jobs:
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
 
       - name: Validate pool diagnostic completeness
+        timeout-minutes: 2
         if: always()
         run: |
           set -euo pipefail
@@ -471,7 +503,11 @@ jobs:
           }
 
           require_nonempty "${POOL_ARTIFACT_DIR}/pool-test.log"
-          require_nonempty "${POOL_ARTIFACT_DIR}/warp.log"
+          # warp is stopped early (SIGINT/SIGTERM) once the storage threshold
+          # is reached, and it only writes its final report on a clean exit —
+          # an empty warp.log is the expected shape of a healthy run, so only
+          # require that the redirect target exists.
+          require_available "${POOL_ARTIFACT_DIR}/warp.log"
           require_nonempty "${POOL_ARTIFACT_DIR}/pool-report.md"
           require_nonempty "${POOL_ARTIFACT_DIR}/pool-baseline.log"
           require_nonempty "${POOL_ARTIFACT_DIR}/nginx-config-redacted.txt"
@@ -522,9 +558,20 @@ jobs:
             echo "dedicated proxy error log contains an upstream timeout or connection failure" >&2
             failed=1
           fi
+          # A mid-script abort is a product/harness signal: the run step's
+          # continue-on-error would otherwise keep the workflow green, so the
+          # validator must turn it red. Aborts go through die(), which prints
+          # an [ERROR] line; a step that dies never emits its [POOL-STEP]
+          # verdict, so "any FAIL verdict" cannot be the trigger.
+          if grep -q '^\[POOL-STEP\] .* FAIL' "${POOL_ARTIFACT_DIR}/pool-test.log" 2>/dev/null \
+            || grep -q '\[ERROR\]' "${POOL_ARTIFACT_DIR}/pool-test.log" 2>/dev/null; then
+            echo "pool test script reported a failure (step FAIL verdict or die() abort)" >&2
+            failed=1
+          fi
           [ "${failed}" -eq 0 ] || exit 1
 
       - name: Upload functional report to dashboard
+        timeout-minutes: 2
         if: always()
         continue-on-error: true
         env:
@@ -556,66 +603,58 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           SUITE: 'pool'
           SUITE_LABEL: 'Pool expansion'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-report.md'
-          LOG_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.pool_test.outcome }}" \
+            --report-file "${POOL_ARTIFACT_DIR}/pool-report.md" \
+            --log "${POOL_ARTIFACT_DIR}/pool-test.log" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload test logs
+        timeout-minutes: 2
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
@@ -624,6 +663,7 @@ jobs:
           if-no-files-found: warn
 
       - name: Restore dedicated pool proxy
+        timeout-minutes: 2
         if: always()
         run: |
           set -euo pipefail
@@ -634,7 +674,8 @@ jobs:
             ./auto-testing/rustfs_pool_nginx_stage.sh cleanup
 
       - name: Cleanup environment (after)
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        timeout-minutes: 5
+        if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -707,3 +748,24 @@ jobs:
           echo "RustFS pool expansion test failed"
           echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.pool_test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite pool
+          --report "${POOL_ARTIFACT_DIR}/pool-steps.md"
+          --output "${RUNNER_TEMP}/chain-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/pool.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json
+          if-no-files-found: error

.github/workflows/rustfs-replication-test.yml

@@ -15,12 +15,18 @@
 name: RustFS Replication Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -43,7 +49,7 @@ permissions:
 # The replication suite uses the same shared VMs as the other functional
 # tests, so it must serialize with them instead of running in parallel.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -61,8 +67,8 @@ env:
 jobs:
   replication-test:
     runs-on: smoke-testing
-    timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    timeout-minutes: 60
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -83,25 +89,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -112,6 +114,7 @@ jobs:
           df -h /data | tail -1 || true
 
       - name: Cleanup environment (before)
+        timeout-minutes: 5
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -131,6 +134,10 @@ jobs:
 
       - name: Run replication suite
         id: test
+        timeout-minutes: 45
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-replication-test.sh
@@ -153,6 +160,8 @@ jobs:
           ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -182,6 +191,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS replication test report"
             echo ""
@@ -194,21 +218,27 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
@@ -239,67 +269,59 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'replication'
           SUITE_LABEL: 'Replication (bucket + site)'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
@@ -311,6 +333,7 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        timeout-minutes: 5
         if: always()
         run: |
           set -euo pipefail
@@ -329,7 +352,7 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Performance)"
+      - name: "Continue functional chain (next: Fault tolerance)"
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -342,9 +365,9 @@ jobs:
           DISPATCHED=0
           for attempt in 1 2 3; do
             if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-performance' \
+              -f event_type='rustfs-chain-fault-tolerance' \
               -F 'client_payload[from_suite]=replication'; then
-              echo "dispatched next suite Performance (attempt ${attempt})"
+              echo "dispatched next suite Fault tolerance (attempt ${attempt})"
               DISPATCHED=1
               break
             fi
@@ -352,19 +375,19 @@ jobs:
             sleep "${attempt}0"
           done
           if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            echo "ERROR: functional chain stalled: could not dispatch Fault tolerance after 3 attempts" >&2
             TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
             BODY_FILE="$(mktemp)"
             trap 'rm -f "${BODY_FILE}"' EXIT
             {
-              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+              echo "The functional chain could not hand off from **replication** to **Fault tolerance** after 3 attempts."
               echo ""
               echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Expected next event: 'rustfs-chain-fault-tolerance'"
               echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
               echo "- Recovery: re-dispatch manually with"
               FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-fault-tolerance'"
               FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
             } > "${BODY_FILE}"
             gh issue create -R rustfs/backlog --title "${TITLE}" \
@@ -380,3 +403,24 @@ jobs:
           echo "RustFS replication suite failed"
           echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite replication
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/replication.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-replication-${{ github.run_id }}-${{ github.run_attempt }}/replication.json
+          if-no-files-found: error

.github/workflows/rustfs-s3-compat-test.yml

@@ -1,12 +1,18 @@
 name: RustFS S3 Compatibility Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -19,7 +25,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -37,8 +43,8 @@ env:
 jobs:
   s3-compat-test:
     runs-on: smoke-testing
-    timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    timeout-minutes: 60
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -59,25 +65,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -87,6 +89,7 @@ jobs:
           df -h /data | tail -1
 
       - name: Cleanup environment (before)
+        timeout-minutes: 5
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -106,6 +109,10 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
+        timeout-minutes: 45
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -122,6 +129,8 @@ jobs:
           ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -151,6 +160,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
@@ -163,21 +187,27 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
@@ -208,67 +238,59 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 's3'
           SUITE_LABEL: 'S3 compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
@@ -280,6 +302,7 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        timeout-minutes: 5
         if: always()
         run: |
           set -euo pipefail
@@ -352,3 +375,24 @@ jobs:
         run: |
           echo "RustFS S3 compatibility suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite s3
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-s3-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/s3.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-s3-${{ github.run_id }}-${{ github.run_attempt }}/s3.json
+          if-no-files-found: error

.github/workflows/rustfs-security-test.yml

@@ -15,12 +15,18 @@
 name: RustFS Security Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -56,7 +62,7 @@ permissions:
 # The security suite uses the same shared VMs as the other functional tests,
 # so it must serialize with them instead of running in parallel.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -74,8 +80,8 @@ env:
 jobs:
   security-test:
     runs-on: smoke-testing
-    timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    timeout-minutes: 60
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       # Checkout the repository into its own subdirectory. Checking out at
       # the workspace root would wipe the auto-testing clone above (that is
@@ -95,25 +101,21 @@ jobs:
           mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
           printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -125,6 +127,7 @@ jobs:
           df -h /data | tail -1
 
       - name: Cleanup environment (before)
+        timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
         run: |
           set -euo pipefail
@@ -145,6 +148,9 @@ jobs:
 
       - name: Run security suite
         id: test
+        timeout-minutes: 45
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
         continue-on-error: true
         env:
           REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
@@ -176,15 +182,37 @@ jobs:
 
       - name: Generate report
         id: report
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         env:
           TEST_OUTCOME: ${{ steps.test.outcome }}
         run: |
           set -euo pipefail
+          LOG_FILE="${SECURITY_ARTIFACTS_DIR}/suite.log"
           RESULT=failure
           if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns turn the workflow red. The suite log lines carry ANSI
+          # color escapes both before and after the verdict tag
+          # (e.g. "\e[1;31m[FAIL]\e[0m STS-105 ..."), so the pattern allows
+          # any number of escapes on each side of it.
+          ESC=$'\033'
+          VERDICTS_TOTAL="$(grep -cE "^(${ESC}\[[0-9;]*m)*\[(PASS|FAIL|UNSUPPORTED)\](${ESC}\[[0-9;]*m)* [A-Z]" "${LOG_FILE}" 2>/dev/null || true)"
+          VERDICTS_FAIL="$(grep -cE "^(${ESC}\[[0-9;]*m)*\[FAIL\](${ESC}\[[0-9;]*m)* [A-Z]" "${LOG_FILE}" 2>/dev/null || true)"
+          VERDICTS_TOTAL=$(( ${VERDICTS_TOTAL:-0} + 0 )); VERDICTS_FAIL=$(( ${VERDICTS_FAIL:-0} + 0 ))
+          VERDICTS_PASS=$(( VERDICTS_TOTAL - VERDICTS_FAIL ))
+          VERDICTS_SKIPPED="$(grep -cE "^(${ESC}\[[0-9;]*m)*\[SKIP\](${ESC}\[[0-9;]*m)* [A-Z]" "${LOG_FILE}" 2>/dev/null || true)"
+          VERDICTS_SKIPPED=$(( ${VERDICTS_SKIPPED:-0} + 0 ))
+          HARNESS_OK=0
+          if [ "${TEST_OUTCOME}" = "success" ]; then
+            HARNESS_OK=1
+          elif [ "${TEST_OUTCOME}" = "failure" ] && [ "${VERDICTS_TOTAL}" -gt 0 ] && [ "${VERDICTS_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS security test report"
             echo ""
@@ -197,18 +225,22 @@ jobs:
             echo ""
             # The dashboard prioritizes case rows over the step outcome.
             # Keep partial case results in the artifact when the suite fails.
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
               cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
-            elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
-              echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
+              echo ""
+            fi
+            if [ "${VERDICTS_SKIPPED}" -gt 0 ]; then
+              echo "- Product result: ${VERDICTS_PASS} passed, ${VERDICTS_FAIL} failed, ${VERDICTS_SKIPPED} skipped (failing cases are tracked in rustfs/backlog)"
             else
-              echo "The suite did not produce a non-empty report."
+              echo "- Product result: ${VERDICTS_PASS} passed, ${VERDICTS_FAIL} failed (failing cases are tracked in rustfs/backlog)"
             fi
           } > "${SECURITY_ARTIFACTS_DIR}/report.md"
           cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
@@ -240,67 +272,58 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           SUITE: 'security'
           SUITE_LABEL: 'Security'
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
-          LOG_FILE: ''
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report-file "${SECURITY_ARTIFACTS_DIR}/report.md" \
+            --log "${SECURITY_ARTIFACTS_DIR}/suite.log" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
@@ -313,6 +336,7 @@ jobs:
           retention-days: 3
 
       - name: Cleanup environment (after)
+        timeout-minutes: 5
         if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
         run: |
           set -euo pipefail
@@ -355,3 +379,24 @@ jobs:
           echo "RustFS security test failed"
           echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
           echo "See the uploaded report and logs for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite security
+          --report "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
+          --output "${RUNNER_TEMP}/chain-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/security.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-security-${{ github.run_id }}-${{ github.run_attempt }}/security.json
+          if-no-files-found: error

.github/workflows/rustfs-storage-test.yml

@@ -1,12 +1,18 @@
 name: RustFS Storage Engine Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -28,7 +34,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -46,8 +52,8 @@ env:
 jobs:
   storage-test:
     runs-on: smoke-testing
-    timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    timeout-minutes: 60
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -68,25 +74,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -96,6 +98,7 @@ jobs:
           df -h /data | tail -1
 
       - name: Cleanup environment (before)
+        timeout-minutes: 5
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -115,6 +118,10 @@ jobs:
 
       - name: Run storage engine suite
         id: test
+        timeout-minutes: 45
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-storage-test.sh
@@ -137,6 +144,8 @@ jobs:
           ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -166,6 +175,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS storage engine test report"
             echo ""
@@ -178,21 +202,27 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
@@ -223,67 +253,59 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'storage'
           SUITE_LABEL: 'Storage engine'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
@@ -295,6 +317,7 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        timeout-minutes: 5
         if: always()
         run: |
           set -euo pipefail
@@ -367,3 +390,24 @@ jobs:
         run: |
           echo "RustFS storage engine suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite storage
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/storage.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-storage-${{ github.run_id }}-${{ github.run_attempt }}/storage.json
+          if-no-files-found: error

.github/workflows/rustfs-table-test.yml

@@ -0,0 +1,360 @@
+name: RustFS Table Test
+
+on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+      rustfs_version:
+        description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
+        required: false
+        type: string
+      skip_smoke:
+        description: 'Skip the PyIceberg end-to-end case (TBL-101)'
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: rustfs-shared-functional-tests-v2
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
+  RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || 'http://rustfs-node1:9000' }}
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  table-test:
+    name: table-test
+    runs-on: smoke-testing
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-table-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        timeout-minutes: 5
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Show environment
+        run: |
+          set -euo pipefail
+          echo "nodes: ${RUSTFS_NODES}"
+          echo "endpoint: ${RUSTFS_API_ENDPOINT}"
+          aws --version
+          python3 --version
+
+      - name: Cleanup environment (before)
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          set -uo pipefail
+          for node in ${RUSTFS_NODES}; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            ' || echo "pre-cleanup failed on ${node} (continuing)"
+          done
+
+      - name: Stage verified candidate package
+        id: chain_package
+        if: ${{ inputs.chain_manifest != '' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/prepare_functional_package.py prepare
+
+      - name: Run table suite
+        id: test
+        timeout-minutes: 45
+        # Standalone case failures are reported by the backlog manager.
+        # Chain evidence separately requires complete passing cases.
+        continue-on-error: true
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-table-test.sh
+          PACKAGE_URL='${{ steps.chain_package.outputs.package_url || inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          SMOKE_PATH="${GITHUB_WORKSPACE:-$PWD}/scripts/table-catalog/pyiceberg_smoke.py"
+          ARGS=(-y --smoke-script "${SMOKE_PATH}")
+          if [ '${{ inputs.skip_smoke }}' = 'true' ]; then
+            ARGS+=(--skip-smoke)
+          fi
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
+            ARGS+=(--package-url "https://github.com/rustfs/rustfs/releases/download/${RUSTFS_VERSION}/rustfs_${RUSTFS_VERSION}_amd64.deb")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-table-test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+
+      - name: Generate report
+        timeout-minutes: 2
+        id: chain_report
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
+          {
+            echo "# RustFS S3 Tables (Iceberg REST Catalog) test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
+
+      - name: Upload functional report to dashboard
+        timeout-minutes: 2
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: ${{ env.REPORT_FILE }}
+          SUITE: table
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+          echo "dashboard updated: ${REPORT_PATH}"
+
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
+            exit 0
+          fi
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found; skipping backlog issue management"
+            exit 0
+          fi
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          if [ -n "${PACKAGE_URL}" ]; then PACKAGE_SOURCE="${PACKAGE_URL}"; fi
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --outcome "${{ steps.test.outcome }}" \
+            --suite table --category table --suite-label "S3 Tables" \
+            --outcome '${{ steps.test.outcome }}' \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${GITHUB_EVENT_NAME}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --date "$(date -u +%Y-%m-%d)"
+
+      - name: Upload report and logs
+        timeout-minutes: 2
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-table-test-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+          if-no-files-found: error
+          retention-days: 30
+
+      - name: Cleanup environment (after)
+        timeout-minutes: 5
+        if: always()
+        continue-on-error: true
+        run: |
+          set -uo pipefail
+          for node in ${RUSTFS_NODES}; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -f /etc/default/rustfs
+            ' || echo "cleanup failed on ${node} (continuing)"
+          done
+
+      - name: "Continue functional chain (next: Performance)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: Performance"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-performance' \
+            -F 'client_payload[from_suite]=table'
+
+      - name: Notify on failure
+        if: failure()
+        run: echo "RustFS table test workflow failed (harness/environment breakdown); see logs and report artifact."
+
+      - name: Remove verified candidate package
+        if: ${{ always() && inputs.chain_manifest != '' && steps.chain.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/prepare_functional_package.py cleanup
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite table
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-table-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/table.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-table-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-table-${{ github.run_id }}-${{ github.run_attempt }}/table.json
+          if-no-files-found: error

.github/workflows/rustfs-tier-test.yml

@@ -1,12 +1,18 @@
 name: RustFS Tier Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -39,7 +45,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -61,9 +67,14 @@ env:
 jobs:
   tier-test:
     runs-on: smoke-testing
-    timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    timeout-minutes: 60
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
       - name: Initialize run evidence directory
         id: evidence
         run: |
@@ -76,27 +87,24 @@ jobs:
           test -d "${TIER_ARTIFACTS_DIR}"
           test ! -L "${TIER_ARTIFACTS_DIR}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Prepare pinned RustFS CLI
+        timeout-minutes: 5
         id: rc
         run: |
           set -euo pipefail
@@ -209,6 +217,7 @@ jobs:
           df -h /data | tail -1
 
       - name: Cleanup environment (before)
+        timeout-minutes: 5
         run: |
           set -euo pipefail
           sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
@@ -229,6 +238,7 @@ jobs:
           done
 
       - name: Ensure MQTT broker + clients
+        timeout-minutes: 5
         run: |
           set -euo pipefail
           if ! command -v mosquitto_sub >/dev/null 2>&1; then
@@ -258,6 +268,9 @@ jobs:
 
       - name: Run tier suite
         id: test
+        timeout-minutes: 45
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
         continue-on-error: true
         env:
           PACKAGE_URL_INPUT: ${{ inputs.package_url }}
@@ -299,6 +312,8 @@ jobs:
           mv "${TMP_FILE}" "${RESULT_FILE}"
 
       - name: Generate report
+        id: chain_report
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         env:
           PACKAGE_URL_INPUT: ${{ inputs.package_url }}
@@ -343,6 +358,20 @@ jobs:
               echo "Structured report generation failed before producing output (exit ${CASE_GATE_RC})."
             } > "${CASE_TABLE}"
           fi
+          # The tier case table carries a "## Case Summary" bullet list
+          # (Total/PASS/FAIL/...); parse the counts from there. Fall back to
+          # the case-ID table rows if the bullets are absent.
+          CASES_FAIL="$(grep -m1 -E '^- FAIL: [0-9]+' "${CASE_TABLE}" 2>/dev/null | grep -oE '[0-9]+' || true)"
+          if [ -n "${CASES_FAIL}" ]; then
+            CASES_PASS="$(grep -m1 -E '^- PASS: [0-9]+' "${CASE_TABLE}" 2>/dev/null | grep -oE '[0-9]+' || true)"
+            CASES_PASS=$(( ${CASES_PASS:-0} + 0 ))
+          else
+            CASES_TOTAL="$(grep -cE '^\| [A-Za-z0-9_.-]+ \| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+            CASES_FAIL="$(grep -cE '^\| [A-Za-z0-9_.-]+ \| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+            CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+            CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          fi
+          CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
           {
             echo "# RustFS tier test report"
             echo ""
@@ -355,6 +384,8 @@ jobs:
             echo ""
             cat "${CASE_TABLE}"
             echo ""
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
             echo "## Log tail"
             echo '```text'
             tail -n 200 "${LOG_FILE}" || true
@@ -363,6 +394,7 @@ jobs:
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
 
       - name: Upload functional report to dashboard
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
@@ -395,6 +427,7 @@ jobs:
           rm -f "${B64_FILE}"
 
       - name: Verify required tier evidence
+        timeout-minutes: 2
         id: evidence_verify
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
@@ -425,6 +458,7 @@ jobs:
           [ "${failed}" -eq 0 ]
 
       - name: Upload report and logs
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
@@ -433,6 +467,7 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        timeout-minutes: 5
         if: always()
         run: |
           set -euo pipefail
@@ -454,6 +489,7 @@ jobs:
           done
 
       - name: Cleanup pinned RustFS CLI
+        timeout-minutes: 2
         if: always()
         run: |
           set -euo pipefail
@@ -496,7 +532,8 @@ jobs:
           [ "${failed}" -eq 0 ]
 
       - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || steps.gate.outcome == 'failure' || steps.gate.outcome == 'cancelled') }}
+        timeout-minutes: 2
+        if: ${{ always() && (steps.evidence.outcome != 'success' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || (steps.gate.outcome == 'failure' && steps.test.outcome != 'failure') || steps.gate.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -515,10 +552,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -565,6 +602,68 @@ jobs:
           fi
           echo "filed backlog issue for suite ${SUITE}"
 
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Case-level issue management complements the infrastructure failure report:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          TEST_OUTCOME: ${{ steps.test.outcome }}
+          VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
+          GATE_OUTCOME: ${{ steps.gate.outcome }}
+          SUITE: 'tier'
+          SUITE_LABEL: 'Tier'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
+            exit 0
+          fi
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
+            exit 0
+          fi
+          if [ ! -d "${TIER_ARTIFACTS_DIR}" ] || [ -L "${TIER_ARTIFACTS_DIR}" ]; then
+            echo "tier evidence directory is missing or unsafe; skipping backlog issue management" >&2
+            exit 1
+          fi
+          OUTCOME="${TEST_OUTCOME}"
+          if [ "${OUTCOME}" != "cancelled" ] && { [ "${VERIFY_OUTCOME}" != "success" ] || [ "${GATE_OUTCOME}" != "success" ]; }; then
+            OUTCOME=failure
+          fi
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${OUTCOME}" \
+            --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md" \
+            --report-file "${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md" \
+            --log "${TIER_ARTIFACTS_DIR}/rustfs-tier.log" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
+
       - name: "Continue functional chain (next: Storage engine)"
         # Only chain-triggered runs forward to the next suite; standalone
         # workflow_dispatch runs stop after their own cleanup. A failed
@@ -619,3 +718,24 @@ jobs:
         run: |
           echo "RustFS tier suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite tier
+          --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
+          --output "${RUNNER_TEMP}/chain-tier-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/tier.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-tier-${{ github.run_id }}-${{ github.run_attempt }}/tier.json
+          if-no-files-found: error

.github/workflows/rustfs-upgrade-test.yml

@@ -15,6 +15,12 @@
 name: RustFS Upgrade Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       from_version:
@@ -61,7 +67,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -79,8 +85,8 @@ env:
 jobs:
   upgrade-test:
     runs-on: smoke-testing
-    timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    timeout-minutes: 60
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -101,25 +107,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -131,6 +133,7 @@ jobs:
           df -h /data | tail -1
 
       - name: Cleanup environment (before)
+        timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
         run: |
           set -euo pipefail
@@ -150,6 +153,7 @@ jobs:
           done
 
       - name: Ensure docker (Vault container)
+        timeout-minutes: 5
         run: |
           if ! command -v docker >/dev/null 2>&1; then
             sudo apt-get update
@@ -160,6 +164,10 @@ jobs:
 
       - name: Run upgrade compatibility suite
         id: test
+        timeout-minutes: 45
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
@@ -218,6 +226,8 @@ jobs:
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -247,6 +257,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS upgrade compatibility report"
             echo ""
@@ -259,23 +284,31 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${MATRIX_TABLE}" ]; then
               cat "${MATRIX_TABLE}"
               echo ""
+            fi
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
@@ -306,67 +339,66 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        timeout-minutes: 2
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'upgrade'
           SUITE_LABEL: 'Upgrade compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          FROM_URL='${{ inputs.from_url }}'
+          FROM_VERSION='${{ inputs.from_version }}'
+          TO_URL='${{ inputs.to_url }}'
+          TO_VERSION='${{ inputs.to_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${TO_URL}" ]; then
+            PACKAGE_SOURCE="to ${TO_URL}"
+          elif [ -n "${TO_VERSION}" ]; then
+            PACKAGE_SOURCE="to version ${TO_VERSION}"
+          else
+            PACKAGE_SOURCE="to ${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          if [ -n "${FROM_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_SOURCE}; from ${FROM_URL}"
+          elif [ -n "${FROM_VERSION}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_SOURCE}; from version ${FROM_VERSION}"
+          fi
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
+        timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
@@ -380,6 +412,7 @@ jobs:
           retention-days: 3
 
       - name: Cleanup environment (after)
+        timeout-minutes: 5
         if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
         run: |
           set -euo pipefail
@@ -454,3 +487,24 @@ jobs:
           echo "From: ${{ inputs.from_url || inputs.from_version || 'release (default)' }}"
           echo "To: ${{ inputs.to_url || inputs.to_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded report and logs for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite upgrade
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/upgrade.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}/upgrade.json
+          if-no-files-found: error

.github/workflows/schedule-failure-alert-drill.yml

@@ -19,7 +19,9 @@
 # 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz,
 # performance-ab). Dispatch it twice to verify both the issue-creation and
 # the dedupe-comment paths, then close the resulting
-# "[scheduled-failure] Schedule Failure Alert Drill" issue.
+# "[scheduled-failure] Schedule Failure Alert Drill" issue in rustfs/backlog
+# (the default target repository). Requires the BACKLOG_ISSUE_TOKEN secret
+# (a token with issues:write on rustfs/backlog).
 #
 # The run itself is expected to end red (the forced failure); only the
 # alert-on-failure job result matters.
@@ -68,4 +70,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/scheduled-validation-freshness.yml

@@ -54,5 +54,5 @@ jobs:
         if: failure()
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
           details-file: ${{ runner.temp }}/scheduled-validation-freshness.md

.github/workflows/scheduled-validation-watchdog.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Open or update incomplete-run issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
           workflow-name: ${{ github.event.workflow_run.name }}
           source-run-id: ${{ github.event.workflow_run.id }}
           source-run-attempt: ${{ github.event.workflow_run.run_attempt }}

.github/workflows/targets-integration.yml

@@ -189,4 +189,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Security
+
+- **Header-signed SigV4 requests honour only signed headers** (GHSA-xm99-m3gq-83g8): a request authenticated with a SigV4 `Authorization` header that carries an `x-amz-*` request header not listed in its `SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3 and the presigned rule from GHSA-g8w9-qw9q-fghr. Previously anyone holding one header-signed `PutObject` request could add an unsigned `x-amz-copy-source` and turn it into a `CopyObject` that ran with the signer's permissions, copying any object the signer could read into the target. An `Authorization` header whose algorithm token is not `AWS4-HMAC-SHA256` is now rejected instead of being verified as SigV4. The request-envelope headers `x-amz-content-sha256`, `x-amz-decoded-content-length`, `x-amz-trailer` and `x-amz-checksum-algorithm` (the same set the upstream `s3s` fix exempts) and `x-amz-cf-id` (CloudFront) remain tolerated unsigned; AWS SDKs and RustFS's own signers already sign every other `x-amz-*` header. SigV2, JWT and anonymous requests are unchanged.
+
 ### Replication
 
 - Object Lock replication PUTs now carry a required integrity header, fixing target rejection introduced by the plain-payload default ([#7097](https://github.com/rustfs/rustfs/pull/7097)). This changes the default outbound request for locked objects but adds no persisted format.
@@ -21,8 +25,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Presigned URLs honour only signed headers** (GHSA-g8w9-qw9q-fghr): a SigV4 presigned request that carries an `x-amz-*` request header not listed in `X-Amz-SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3. Previously the holder of a presigned `PutObject` URL could add unsigned `x-amz-tagging`, `x-amz-storage-class`, `x-amz-website-redirect-location`, ACL, metadata, Object Lock or SSE headers and have them applied. Presigners that intend a property must set it before signing so the SDK lists the header in `SignedHeaders`; `x-amz-cf-id` (CloudFront) remains tolerated unsigned. Header-signed SigV4 and SigV2 requests are unchanged.
 
 ### Fixed
+- **Vault backends accepted key names that escape the key prefix**: only the Local backend refused a key identifier containing a path separator, so on Vault KV2 `POST /rustfs/admin/v3/kms/keys` with a name such as `bad/name` succeeded and created a nested KV2 path that the key listing then reported as a directory rather than a key, and an identifier containing `..` addressed a record outside the configured key prefix once the HTTP client normalised the URL. Both Vault backends now apply the Local backend's containment rule at the point where the identifier becomes a path or a Transit key name: an empty identifier, one containing `/`, `\\` or NUL, or the dot segments `.` and `..` is refused with `400` (`InvalidKey`) before any request reaches Vault. The AWS backend is unaffected, since it addresses keys by ARN and alias. A Vault deployment that created such a key on an earlier build must re-create it under a plain name; objects encrypted under the old name stay readable only until that key is refused, so migrate them first. Refs rustfs/backlog#2474.
 - **Fresh multi-pool bootstrap with distinct format creators**: a new deployment whose pools have their first endpoint on different nodes (for example two single-node pools) could never publish its initial `pool.bin`: each node held fresh-bootstrap proof only for the pool it formatted, the deployment-wide proof collapsed to none, and every node died with `pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available` after the startup retry budget. The first pool's creator now mints the pending cluster identity on its own pool, every other creator copies that nonce-bound identity onto the pool it formatted first-hand, and the elected writer publishes `pool.bin` once every pool replica carries the same pending identity. Corrupt or disagreeing replicas, pools that merely have a format, expansion pools joining an initialized deployment, and restarts without first-hand proof still fail closed. Non-elected nodes that start before `pool.bin` exists, and the elected writer while it waits for the other creators, no longer latch their pool-metadata write gate for the life of the process. Refs rustfs/backlog#2338, rustfs/backlog#2375.
 - **Lock RPC timeout storms** (#7363): the remote lock client no longer evicts and re-dials the shared internode HTTP/2 channel on every request deadline. A timeout evicts only when the peer has not completed any lock RPC for two deadlines, evictions and transport-failure re-dials are rate limited per peer (`RUSTFS_OBJECT_LOCK_RPC_EVICTION_COOLDOWN_MS`, default 5 s), and a timed-out request is left running instead of being reset (bounded per peer by `RUSTFS_OBJECT_LOCK_RPC_DETACHED_LIMIT`, default 256), so a slow lock endpoint can no longer drive the `RST_STREAM`/`GOAWAY too_many_resets`/reconnect loop. A lock granted after its caller timed out is released immediately, and unlocks that fail the quick retries continue on a deferred 1/2/4/8/16 s schedule before the server lease reclaims them. New `rustfs_remote_lock_*` metrics cover timeouts, evictions, suppressed evictions, detached streams, late completions and late releases per peer. Operator guide at `docs/operations/lock-rpc-storm-protection.md`.
+- **KMS failures on the S3 data path carry an actionable status**: only "key not found" and a backend outage were classified; every other KMS failure — a disabled or pending-deletion key, a denied KMS grant, an encryption-context mismatch, an unsupported algorithm, a credential or timeout failure, a capability the backend does not have — collapsed onto `500 InternalError`. SDKs therefore applied exponential backoff to configuration errors that no retry can fix, and monitoring filed every one of them as a server fault. Unusable-key and request-side failures now return `400`, a denied grant `403`, transient backend failures `503` — including a key store the backend could not read, so an outage stays distinguishable from a missing key all the way to the client — and a missing backend capability `501`. Damaged or unreadable key material still returns `500`, which is what it is.
+- **Bare SSE-KMS writes on a node without a running KMS**: `x-amz-server-side-encryption: aws:kms` without a key id (and no bucket default key) returned `500 InternalError` while KMS was stopped or never configured, because the "no key available" branch exited before the availability classification that the keyed form already received. `PutObject` and `CreateMultipartUpload` now return `503 ServiceUnavailable` while a configured KMS is stopped, `400 InvalidRequest` when KMS was never configured, and `400 InvalidRequest` naming the missing key id when a running KMS has no default key.
+- **Reading an object whose KMS key is gone returned `500`**: `GetObject`, `CopyObject` and `UploadPartCopy` on an SSE-KMS object whose key had been deleted reported `500 InternalError` ("KMS key not found"), while the same condition on `PutObject` already returned `400 KMS.NotFoundException`. The read path carried only four classifications across the storage boundary and folded a missing key, a denied KMS grant and a missing backend capability onto "decryption failed". Those reads now return `400 KMS.NotFoundException`, `403 AccessDenied` and `501 NotImplemented` respectively; `HeadObject` is unaffected because it never unwraps the data key. An envelope the configured backend cannot unwrap (the key was re-created under the same name, or the backend was switched) stays `500` but now says so instead of the generic internal-error text.
+- **KMS key-management routes answered `500` for client-side failures**: `POST /rustfs/admin/v3/kms/keys` and the legacy `create-key` alias reported every backend refusal as `500`, including a blank key name (each backend failed differently, the Local backend by writing a key file with an empty stem) and a name that already exists; `POST /rustfs/admin/v3/kms/generate-data-key` did the same for an unknown or disabled key, although `describe` and `delete` already classified those. A blank or whitespace key name is now refused before it reaches any backend (`400`), a taken name is `409`, an unknown key is `404` (`KMS.NotFoundException`), a disabled key `400`, and a capability the backend lacks `501`; damaged key material stays `500`. The read-only Static backend now reports create, delete and cancel-deletion as missing capabilities (`501`), matching its rotate and enable/disable answers, instead of `400`/`500`.
+- **SSE-S3 responses named the internal wrapping key**: `PutObject`, `CopyObject`, `CreateMultipartUpload` and `GetObject` for an `AES256` object returned `x-amz-server-side-encryption-aws-kms-key-id` carrying the KMS key that wraps the SSE-S3 data key (the service default key, or the literal `default` on a node without KMS), although the header is defined for `aws:kms` objects only and `CompleteMultipartUpload` and `HeadObject` already omitted it. Those responses now advertise a key id only for `aws:kms` objects.
+- **PutBucketEncryption accepted algorithms the server cannot honour**: a default-encryption rule naming an unknown `SSEAlgorithm` (for example `AES128`), a rule without `ApplyServerSideEncryptionByDefault`, an empty rule list, or a `KMSMasterKeyID` on an `AES256` rule was stored as written. `GetBucketEncryption` then reported that configuration while every header-less write was encrypted under the `AES256` fallback, so the bucket's advertised and actual schemes disagreed. Those configurations are now refused with `400` (`MalformedXML` for a malformed rule, `InvalidArgument` for a key id on a non-KMS rule) and nothing is stored.
+- **SSE-C on buckets with default encryption**: a `PutObject` carrying a valid SSE-C header triple on a bucket that has default encryption configured no longer fails with `400 InvalidArgument` ("The SSE-C and managed server-side encryption headers cannot be used together"). PUT and the POST-object/extract path resolved the bucket default with a hard-coded "no explicit SSE-C" flag, so the default was layered onto the request and then tripped the request's own mutual-exclusion check; an SSE-C request now suppresses the bucket default on all three write paths, matching COPY and AWS S3. Every bucket with default encryption previously refused SSE-C single PUTs outright, while `CreateMultipartUpload` on the same bucket succeeded.
+- **Explicit SSE-S3 on SSE-KMS-default buckets**: `x-amz-server-side-encryption: AES256` against a bucket whose default is `aws:kms` no longer fails with `400 InvalidArgument`. The bucket default's KMS key id was inherited independently of the effective algorithm, producing a self-contradictory `AES256` + key-id pair; the key id is now inherited only when the effective algorithm is `aws:kms`. `PutBucketEncryption` fills in a default key id automatically, so this affected nearly every SSE-KMS-default bucket.
+- **Restore of encrypted or compressed multipart objects (silent data corruption)**: restoring a multipart object from a remote tier addressed the tier in *plaintext* coordinates while the copy-back reads the *stored* representation. Every part received a misaligned slice of the remote object whose length still satisfied the range, the hash reader and the completion size check, so the restore reported success and replaced the object's bytes. Restore now accumulates stored part sizes, passes the stored length to the hash reader alongside the plaintext length, and validates against the stored size. Objects restored by an affected release must be re-restored from the tier or recovered from a backup — this release does not detect or repair them retroactively.
+- **Restore no longer drifts the object ETag**: the copy-back digests stored (encrypted or compressed) bytes, so the recomputed MD5 is not the object's public ETag. Single-part and multipart restores now preserve the original object ETag, and each restored part keeps its own recorded part ETag.
+- **ILM archive no longer forwards encryption metadata to the tier**: transition requests carried the object's SSE headers and the RustFS-wrapped data key as request headers. Any S3 target rejected an SSE-C archive outright (`400`, no key supplied), an SSE-KMS archive asked the target to encrypt a second time under a key id it does not own, and the wrapped DEK left the cluster. The archive request now strips every SSE header and encryption marker using the same predicate the replication path uses; the local `xl.meta` keeps all of it, so read-through and restore are unaffected.
+- **KMS reload is no longer a no-op on a node whose KMS failed to start**: `POST /rustfs/admin/v3/kms/reload` short-circuited whenever the persisted configuration matched the in-memory one byte for byte. A node whose KMS failed to start (for example Vault briefly unreachable during a rolling restart) keeps that configuration and sits in `Error`, so the documented recovery call returned "reloaded successfully" while leaving the node down — and did the same on every peer through the reload broadcast. Reload now short-circuits only for a service that is actually running, and otherwise reconfigures, which starts the service.
+- **AWS KMS capability reporting**: the AWS backend no longer advertises `versioning` support through `GET /rustfs/admin/v3/kms/status`. AWS KMS key versions are not enumerable through this backend, as the backend documentation already stated.
 - **Multipart admission queue**: an `UploadPart` waiting for a foreground write permit now waits at most 10 s by default (`RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS`, previously 30 s), so a queued part returns S3 `SlowDown` before the client's socket write timeout drops the connection. Separately, the API listener no longer forces a 4 MiB `SO_RCVBUF` on every accepted socket (kernel autotuning applies; `RUSTFS_HTTP_SOCKET_RECV_BUFFER_BYTES` restores a fixed size), so a queued part no longer lets up to 8 MiB of unread body accumulate in kernel memory per connection, which is what throttled whole nodes under SDK-default multipart concurrency. Fixes #7385.
 - **Helm Ingress**: `customAnnotations` are now merged with class-specific annotations (nginx/traefik) instead of being ignored when `ingress.className` is set.
 - **Per-pool erasure parity**: Erasure parity (STANDARD and reduced-redundancy) is now resolved independently for every pool instead of reusing the first pool's value. A heterogeneous topology — for example a 4-drive pool plus a 2-drive pool created during expansion — previously inherited the first pool's parity and could resolve to zero data shards in the smaller pool, panicking Reed-Solomon construction on write. Automatic parity now resolves per pool (for example `2+2` in the 4-drive pool and `1+1` in the 2-drive pool). Fixes #4801.
@@ -64,6 +82,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Four-layer regression-prevention tests guard against silent feature deletion: compile-time module assertion, module-presence unit test, cross-module `Protocol` enum assertion, end-to-end SSH banner test against the running binary
 
 ### Changed
+- **Encryption and KMS work merged since `1.0.0-rc.5`** (entries were missing from this section):
+  - **Persisted KMS configuration secrets** are sealed field-by-field with `RUSTFS_KMS_CONFIG_SECRET`. **When the variable is unset the secrets are persisted in cleartext and the server only warns** (`persisted KMS configuration carries cleartext secrets`); it never refuses the write. Set it, identically, on every node, and re-save the configuration to seal an existing one.
+  - **New v2 ciphertext frame format** with per-frame index binding and final-frame authentication. Its write switch `RUSTFS_ENCRYPTION_FRAME_V2` is **off by default**: v2 frames are unreadable by nodes without v2 read support, and encrypted ciphertext travels verbatim through transition, decommission and SSE-C replication passthrough, so turn it on only after every node — and every RustFS warm/replication target that receives raw ciphertext — runs a release with v2 read support. Reading v2 objects needs no switch.
+  - **Per-key SSE-KMS authorization** (`RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY`, default `false`). With it on, anonymous callers hold no KMS grants, so **a public bucket serving SSE-KMS objects is an incompatible combination** and those reads return `AccessDenied`.
+  - Envelope context binding as KMS AAD (`ENV_KMS_ENVELOPE_AAD`, off by default; a node that predates the field cannot open bound envelopes).
+  - Vault custom CA and mutual TLS; object-level DEK rewrap plus a batch rekey admin API; a backend-locality runtime signal on `kms/status`.
+  - Single-pass decryption for encrypted GET, and encrypted single-part closed-range seek — the latter is now **on by default** (`RUSTFS_ENCRYPTED_RANGE_SEEK`, default `true`; the switch remains as a kill switch).
+- **Vault static tokens are now tracked and renewed**: with `Token` authentication RustFS hard-coded "this token has no lease", so the renewal task never started and no remaining-TTL gauge was published. `vault token create` grants a 768-hour TTL by default, which turned a healthy-looking cluster into one where every KMS call returned 403 about a month later, with no self-healing short of a restart or reconfigure. RustFS now calls `auth/token/lookup-self` at login and adopts what Vault reports: a non-expiring token behaves exactly as before, an expiring renewable one is renewed at half TTL like the other auth methods, and an expiring non-renewable one logs `vault_static_token_not_renewable` and publishes its remaining TTL. The probe never fails the login: a token whose policy omits `lookup-self` (Vault's `default` policy grants it), or a Vault that is unreachable at that moment, logs `vault_static_token_lookup_failed` and falls back to the previous no-lease behaviour, so no deployment that works today stops working.
+- **SSE-C over a plaintext transport is reported**: AWS S3 and MinIO refuse an SSE-C request that did not arrive over TLS, because the customer key travels in a request header. RustFS accepted them on any transport and still does by default — flipping to a rejection inside a release window would break plaintext staging and test deployments. Each such request now increments `rustfs_ssec_plaintext_requests_total` and logs one `ssec_request_without_tls` warning per process, and `RUSTFS_SSE_C_REQUIRE_TLS=true` opts into the AWS `400` now. The default is expected to flip in a later release; confirm the counter reads zero first. The verdict is per connection: a TLS listener satisfies it, and so does an `https` protocol forwarded by a proxy the trusted-proxy configuration accepts.
+- **Local KMS backend on a distributed deployment says what actually breaks**: the backend keeps key material and its Argon2id salt on each node's own disk, so two nodes derive different keys from the same `master_key` and an object encrypted on one node cannot be decrypted on another — intermittent 500s behind a load balancer. Configuring it while the deployment is distributed now logs `kms_node_local_backend_in_distributed_deployment` and appends that consequence to the `kms/configure` response, instead of only the generic "development only" positioning warning. It remains a warning, not a gate.
+- **SSE-KMS is refused when no KMS is running (breaking)**: a write requesting `x-amz-server-side-encryption: aws:kms` on a node with no KMS service no longer succeeds. Earlier releases wrapped the data key with the node-local `RUSTFS_SSE_S3_MASTER_KEY` while still writing `aws:kms` and the requested key id into the object metadata — metadata that claimed a KMS protection the object never had, under a key that was never consulted. Such a request now returns `400 InvalidRequest` when KMS was never configured and `503` when a configured service is not running; the refusal is evaluated after the per-key authorization gate, so an unauthorized caller still receives `403 AccessDenied`. **Upgrade note:** a deployment that relied on this write succeeding will start receiving 4xx/503. Either configure a KMS, or request `AES256` and keep the documented SSE-S3 local-master-key fallback, which is unchanged. Objects already written this way remain readable.
+- **Legacy ciphertext nonce layouts are now locked per segment**: while decrypting a v1 segment, the reader locks onto whichever of the three historical nonce layouts decoded the segment's first non-zero-index frame and rejects any later frame that needs a different one. Because a frame encrypted at block index zero authenticates under the pre-`1.0.0-alpha.91` reused-part-nonce layout at any position, an attacker able to rewrite the underlying shards could previously replay it and have the forged plaintext returned with `200`. New `RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK` (default `true`) drops that third layout entirely when set to `false`, which closes the residual case of a stream built purely from repeats of frame zero. Turn it off only after migrating pre-alpha.91 encrypted objects (rewrite in place with CopyObject); see [KMS backend security properties](docs/operations/kms-backend-security.md) for what the v1 frame layout does and does not authenticate.
 - **HTTP Server Stack**: Integrated `KeystoneAuthLayer` middleware from `rustfs-keystone` crate into service stack (positioned after ReadinessGateLayer)
 - **Storage-class validation on startup (upgrade note)**: A persisted explicit storage class (`RUSTFS_STORAGE_CLASS_STANDARD` / `RUSTFS_STORAGE_CLASS_RRS`, for example `EC:2`) is now validated against the actual per-pool drive counts at startup and rejected when a pool cannot satisfy it. This is fail-closed and correct, but a cluster that persisted a storage class larger than a small or heterogeneous pool can hold (for example `EC:2` alongside a 2-drive pool), which earlier releases accepted and silently resolved to an invalid layout, will now refuse to start after upgrade. To recover, unset `RUSTFS_STORAGE_CLASS_STANDARD` so the server derives a valid per-pool default automatically, or set it to a value every pool can satisfy.
 - **IAMAuth**: Enhanced `get_secret_key()` to return empty secret for Keystone credentials (bypasses signature validation)

Cargo.toml

@@ -72,8 +72,8 @@ resolver = "3"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
-rust-version = "1.98.0"
-version = "1.0.0-rc.5"
+rust-version = "1.98.1"
+version = "1.0.1"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -90,62 +90,62 @@ redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
-rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
-rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
-rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.5" }
-rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
-rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.5" }
-rustfs-config = { path = "./crates/config", version = "1.0.0-rc.5" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.5" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.5" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.5" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
-rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
-rustfs-license = { path = "crates/license", version = "1.0.0-rc.5" }
-rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
-rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
-rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.5" }
-rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.5" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.5" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.5" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.5" }
-rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.5", default-features = false }
-rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.5" }
-rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.5" }
-rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.5" }
-rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.5" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.5" }
-rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.5" }
-rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.5" }
-rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.5" }
-rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.5" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.5" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.5" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.5" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.5" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.5" }
-rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.5" }
-rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.5" }
-rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.5" }
-rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.5" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.5" }
-rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.5" }
-rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.5" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.5" }
-rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.5" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
+rustfs = { path = "./rustfs", version = "1.0.1" }
+rustfs-heal = { path = "crates/heal", version = "1.0.1" }
+rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.1" }
+rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.1" }
+rustfs-audit = { path = "crates/audit", version = "1.0.1" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.1" }
+rustfs-common = { path = "crates/common", version = "1.0.1" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.1" }
+rustfs-config = { path = "./crates/config", version = "1.0.1" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.1" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.1" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.1" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.1" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.1" }
+rustfs-iam = { path = "crates/iam", version = "1.0.1" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.1" }
+rustfs-license = { path = "crates/license", version = "1.0.1" }
+rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.1" }
+rustfs-kms = { path = "crates/kms", version = "1.0.1" }
+rustfs-lock = { path = "crates/lock", version = "1.0.1" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.1" }
+rustfs-notify = { path = "crates/notify", version = "1.0.1" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.1" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.1" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.1" }
+rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.1", default-features = false }
+rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.1" }
+rustfs-obs = { path = "crates/obs", version = "1.0.1" }
+rustfs-policy = { path = "crates/policy", version = "1.0.1" }
+rustfs-protos = { path = "crates/protos", version = "1.0.1" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.1" }
+rustfs-replication = { path = "crates/replication", version = "1.0.1" }
+rustfs-rio = { path = "crates/rio", version = "1.0.1" }
+rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.1" }
+rustfs-s3-client = { path = "crates/s3-client", version = "1.0.1" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.1" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.1" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.1" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.1" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.1" }
+rustfs-security-governance = { path = "crates/security-governance", version = "1.0.1" }
+rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.1" }
+rustfs-signer = { path = "crates/signer", version = "1.0.1" }
+rustfs-storage-api = { path = "crates/storage-api", version = "1.0.1" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.1" }
+rustfs-targets = { path = "crates/targets", version = "1.0.1" }
+rustfs-test-utils = { path = "crates/test-utils", version = "1.0.1" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.1" }
+rustfs-utils = { path = "crates/utils", version = "1.0.1" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.1" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.44" }
+async-compression = { version = "0.4.47" }
 async-recursion = "1.1.1"
 async-trait = "0.1.92"
 async-nats = { version = "0.50.0", default-features = false }
@@ -156,15 +156,16 @@ futures-lite = "2.6.1"
 futures-util = "0.3.34"
 pollster = "1.0.1"
 pulsar = { default-features = false, version = "6.9.0" }
-lapin = { default-features = false, version = "4.10.0" }
+lapin = { default-features = false, version = "4.11.0" }
 hyper = { version = "1.11.1" }
 hyper-rustls = { default-features = false, version = "0.27.9" }
 hyper-util = { version = "0.1.20" }
 http = "1.5.0"
 http-body = "1.1.0"
 http-body-util = "0.1.5"
+memchr = "2.8.3"
 minlz = "1.2.3"
-reqwest = "0.13.4"
+reqwest = "0.13.5"
 rustfs-kafka-async = { version = "1.3.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.53.1" }
@@ -206,12 +207,12 @@ blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
-jsonwebtoken = { version = "11.0.0" }
+jsonwebtoken = { version = "11.1.0" }
 openidconnect = { default-features = false, version = "4.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
-rustls = { default-features = false, version = "0.23.43" }
+rustls = { default-features = false, version = "0.23.45" }
 rustls-native-certs = "0.8"
 rustls-pki-types = "1.15.1"
 x509-parser = "0.18.1"
@@ -224,7 +225,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.35" }
+jiff = { version = "0.2.37" }
 time = { version = "0.3.55" }
 
 # Database
@@ -235,8 +236,7 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork.
-astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
+rustfs-tokio-tar = { version = "0.7.1" }
 # Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
 tar-codec = "0.0.14"
 tar-framing = "0.0.14"
@@ -245,15 +245,15 @@ atomic_enum = "0.3.0"
 aws-config = { version = "1.12.0" }
 aws-credential-types = { version = "1.3.0" }
 aws-sdk-kms = { default-features = false, version = "1.118.0" }
-aws-sdk-s3 = { default-features = false, version = "1.145.0" }
+aws-sdk-s3 = { default-features = false, version = "1.146.1" }
 aws-sdk-sts = { default-features = false, version = "1.114.0" }
 aws-smithy-async = { version = "1.3.0" }
-aws-smithy-http-client = { default-features = false, version = "1.4.0" }
-aws-smithy-runtime-api = { version = "1.16.0" }
-aws-smithy-types = { version = "1.6.3" }
+aws-smithy-http-client = { default-features = false, version = "1.4.2" }
+aws-smithy-runtime-api = { version = "1.16.2" }
+aws-smithy-types = { version = "1.6.4" }
 base64-simd = "0.8.0"
 brotli = "9.0.0"
-clap = { version = "4.6.6" }
+clap = { version = "4.6.7" }
 const-str = { version = "1.1.0" }
 convert_case = "0.12.0"
 criterion = { version = "0.8" }
@@ -261,8 +261,9 @@ crossbeam-queue = "0.3.14"
 crossbeam-channel = "0.5.17"
 crossbeam-deque = "0.8.8"
 crossbeam-utils = "0.8.23"
-datafusion = { default-features = false, version = "55.0.0" }
+datafusion = { default-features = false, version = "55.1.0" }
 derive_builder = "0.20.2"
+ed25519-dalek = "3.0.0"
 enumset = "1.1.14"
 faster-hex = "0.10.0"
 flate2 = "1.1.10"
@@ -297,13 +298,13 @@ percent-encoding = "2.3.2"
 # Server-side QR rendering for TOTP enrollment, so neither the console nor the
 # CLI needs its own QR encoder. No default features: the image/render backends
 # pull in an image stack this only needs SVG and text output from.
-qrcode-rs = { version = "2.0.0", default-features = false, features = ["std", "svg"] }
+qrcode-rs = { version = "2.1.1", default-features = false, features = ["std", "svg"] }
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
 rand = { version = "0.10.2" }
 ratelimit = "2.0.0"
 rayon = "1.12.0"
-rustfs-erasure-codec = { version = "8.0.2" }
+rustfs-erasure-codec = { version = "9.0.0" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
@@ -312,11 +313,12 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "bdcb6259339c41369f9f1c60e3a42b5ab8da607b", version = "0.15.0", features = ["minio"] }
+s3s = { version = "0.16.0", features = ["minio"] }
+s3s-sigv4 = { version = "0.16.0" }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
-smallvec = { version = "1.16.0" }
+smallvec = { version = "1.16.1" }
 compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.3.0" }
@@ -335,7 +337,7 @@ tracing-subscriber = { version = "0.3.23" }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.26.0" }
+uuid = { version = "1.26.1" }
 vaultrs = { version = "0.8.0" }
 tar = "0.4.46"
 walkdir = "2.5.0"
@@ -349,7 +351,7 @@ zstd = "0.14.0"
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9-tokio-telemetry = "0.5.0"
+dial9 = { version = "0.5.0", default-features = false }
 opentelemetry = { version = "0.32.0" }
 opentelemetry-appender-tracing = { version = "0.32.0" }
 opentelemetry-otlp = { version = "0.32.0" }
@@ -362,18 +364,18 @@ pyroscope = { version = "2.1.1" }
 # FTP and SFTP
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
-suppaftp = { version = "11.0.0" }
+suppaftp = { version = "12.0.0" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.63.2" }
-russh-sftp = "2.4.0"
+russh = { version = "0.63.3" }
+russh-sftp = "3.0.0"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.5.3" }
+rustfs-mimalloc = { version = "0.5.4" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "=0.25.0", default-features = false }
+hotpath = { version = "0.25.1", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 
@@ -387,6 +389,14 @@ ignored = ["hotpath", "rustfs"]
 #   CARGO_PROFILE_DEV_DEBUG=full cargo build
 debug = "line-tables-only"
 
+# To further speed up the process, completely disable debug information for dependencies.
+[profile.dev.package."*"]
+debug = false
+# When you really need to debug, you can enable full debugging information using the `--profile debugging` option.
+[profile.debugging]
+inherits = "dev"
+debug = true
+
 [profile.release]
 opt-level = 3
 lto = "thin"

README.md

@@ -141,7 +141,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.1
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command

README_ZH.md

@@ -46,14 +46,31 @@ RustFS 是一个基于 Rust 构建的高性能分布式对象存储系统。Rust
 - **完全开源**：采用 Apache 2.0 许可证，鼓励社区贡献和商业使用。
 - **简单易用**：设计简洁，易于部署和管理。
 
-| 功能               | 状态    | 功能                    | 状态      |
-| :----------------- | :------ | :---------------------- | :-------- |
-| **S3 核心功能**    | ✅ 可用 | **Bitrot (防数据腐烂)** | ✅ 可用   |
-| **上传 / 下载**    | ✅ 可用 | **单机模式**            | ✅ 可用   |
-| **版本控制**       | ✅ 可用 | **存储桶复制**          | ✅ 可用   |
-| **日志功能**       | ✅ 可用 | **生命周期管理**        | 🚧 测试中 |
-| **事件通知**       | ✅ 可用 | **分布式模式**          | 🚧 测试中 |
-| **K8s Helm Chart** | ✅ 可用 | **OPA (策略引擎)**      | 🚧 测试中 |
+状态说明：✅ 可用 —— 已发布并有 CI 门禁覆盖；🧪 预览 —— 已发布但需显式开关，或兼容性承诺有边界。
+
+| 功能                        | 状态    | 功能                     | 状态    |
+| :-------------------------- | :------ | :----------------------- | :------ |
+| **S3 核心功能**             | ✅ 可用 | **分布式模式**           | ✅ 可用 |
+| **上传 / 下载**             | ✅ 可用 | **单机模式**             | ✅ 可用 |
+| **版本控制**                | ✅ 可用 | **Bitrot (防数据腐烂)**  | ✅ 可用 |
+| **对象锁定 (WORM)**         | ✅ 可用 | **修复与扫描器**         | ✅ 可用 |
+| **服务端加密 (SSE)**        | ✅ 可用 | **存储池扩容 / 下线**    | ✅ 可用 |
+| **RustFS KMS**              | ✅ 可用 | **存储桶复制**           | ✅ 可用 |
+| **生命周期管理 (ILM)**      | ✅ 可用 | **站点复制**             | ✅ 可用 |
+| **ILM 分层 (远端 S3)**      | ✅ 可用 | **存储桶配额**           | ✅ 可用 |
+| **S3 Select**               | ✅ 可用 | **事件通知**             | ✅ 可用 |
+| **S3 Tables (Iceberg REST)**| 🧪 预览 | **审计日志**             | ✅ 可用 |
+| **IAM / 策略**              | ✅ 可用 | **日志与可观测性**       | ✅ 可用 |
+| **OIDC / SSO**              | ✅ 可用 | **Web 控制台**           | ✅ 可用 |
+| **Keystone 认证**           | ✅ 可用 | **K8s Helm Chart**       | ✅ 可用 |
+| **Swift API**               | ✅ 可用 | **FTPS / WebDAV**        | ✅ 可用 |
+| **多租户**                  | ✅ 可用 | **SFTP**                 | ✅ 可用 |
+| **MinIO 磁盘格式兼容**      | 🧪 预览 |                          |         |
+
+说明：
+
+- **服务端加密**：支持 SSE-C、SSE-S3 与 SSE-KMS。SSE-KMS 必须先配置 KMS 服务；未配置 KMS 时请求 `aws:kms` 会被拒绝，不会降级到本地主密钥。
+- **RustFS KMS**：生产环境支持 Vault（KV2 / Transit）与 AWS KMS 后端；`Local` 与 `Static` 后端仅供开发与测试使用，详见 [KMS 后端安全属性](docs/operations/kms-backend-security.md)。
 
 ## RustFS vs MinIO 性能对比
 
@@ -121,7 +138,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # 使用指定版本运行
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.1
 ```
 
 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：

crates/common/Cargo.toml

@@ -41,6 +41,7 @@ uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
 metrics = { workspace = true }
 smallvec = { workspace = true }
 tracing = { workspace = true }
+thiserror.workspace = true
 
 [lib]
 doctest = false

crates/common/src/mrf_channel.rs

@@ -16,12 +16,15 @@
 //!
 //! Producers on error paths (read decode failure, scanner metadata
 //! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the
-//! heal crate through a global bounded channel. Delivery is strictly
+//! heal crate through a global bounded channel. Hint delivery is strictly
 //! non-blocking: `try_send_mrf_intent` never awaits and drops the intent
 //! (counting it) when the channel is full or uninitialized — losing one heal
 //! hint is always preferred over stalling an IO path. Durable replay of
 //! unconsumed intents is the consumer's job (see `rustfs-heal`
-//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
+//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`. Committed partial
+//! writes and committed partial delete-marker purges use a separate bounded
+//! channel and await a checkpoint receipt; ordinary complete writes do not use
+//! it.
 
 use std::collections::HashMap;
 use std::collections::hash_map::RandomState;
@@ -33,7 +36,7 @@ use std::sync::{
     atomic::{AtomicBool, Ordering},
 };
 use std::time::{Duration, Instant};
-use tokio::sync::mpsc;
+use tokio::sync::{mpsc, oneshot};
 use uuid::Uuid;
 
 /// Bounded capacity of the global MRF channel. Backpressure is resolved by
@@ -44,10 +47,10 @@ const MRF_COALESCER_MAX_KEYS: usize = 8192;
 const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
 const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
 const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
+pub const MRF_MAX_DELETE_MARKER_BYTES: usize = 1024 * 1024;
 
 /// Why an intent was produced. Drives the heal priority mapping on the
-/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
-/// PartialWrite -> Normal).
+/// consumer side.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum MrfKind {
     /// Erasure decode failed while serving a read (read path).
@@ -56,6 +59,8 @@ pub enum MrfKind {
     MetadataCorruption,
     /// A write left the object with fewer committed shards than the set size.
     PartialWrite,
+    /// A committed explicit delete-marker purge missed at least one member.
+    DeleteMarkerPurge,
 }
 
 impl MrfKind {
@@ -64,8 +69,60 @@ impl MrfKind {
             MrfKind::DecodeFailure => "decode-failure",
             MrfKind::MetadataCorruption => "metadata-corruption",
             MrfKind::PartialWrite => "partial-write",
+            MrfKind::DeleteMarkerPurge => "delete-marker-purge",
         }
     }
+
+    pub const fn is_durable(self) -> bool {
+        matches!(self, Self::PartialWrite | Self::DeleteMarkerPurge)
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct MrfDeleteMarkerPurgeIdentity {
+    pub bucket_incarnation_id: Uuid,
+    pub marker_incarnation_id: Uuid,
+    pub marker_identity: [u8; 32],
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct MrfDeleteMarkerPurge {
+    pub bucket_incarnation_id: Uuid,
+    pub marker_incarnation_id: Uuid,
+    pub marker_identity: [u8; 32],
+    pub marker: Arc<[u8]>,
+}
+
+impl MrfDeleteMarkerPurge {
+    pub fn new(
+        bucket_incarnation_id: Uuid,
+        marker_incarnation_id: Uuid,
+        marker_identity: [u8; 32],
+        marker: Vec<u8>,
+    ) -> Option<Self> {
+        let payload = Self {
+            bucket_incarnation_id,
+            marker_incarnation_id,
+            marker_identity,
+            marker: Arc::from(marker),
+        };
+        payload.is_valid().then_some(payload)
+    }
+
+    pub const fn identity(&self) -> MrfDeleteMarkerPurgeIdentity {
+        MrfDeleteMarkerPurgeIdentity {
+            bucket_incarnation_id: self.bucket_incarnation_id,
+            marker_incarnation_id: self.marker_incarnation_id,
+            marker_identity: self.marker_identity,
+        }
+    }
+
+    pub fn is_valid(&self) -> bool {
+        !self.bucket_incarnation_id.is_nil()
+            && self.marker_incarnation_id == self.bucket_incarnation_id
+            && !self.marker.is_empty()
+            && self.marker.len() <= MRF_MAX_DELETE_MARKER_BYTES
+    }
 }
 
 /// One repair intent. Kept deliberately small so the in-memory queue and the
@@ -78,6 +135,10 @@ pub struct MrfIntent {
     /// Version the intent targets, as raw UUID bytes.
     pub version_id: Option<[u8; 16]>,
     pub kind: MrfKind,
+    /// Present only for [`MrfKind::DeleteMarkerPurge`]. The encoded marker is a
+    /// destructive-operation precondition, while `marker_identity` is its
+    /// deterministic queue and proof identity.
+    pub delete_marker_purge: Option<MrfDeleteMarkerPurge>,
     /// Stable erasure-set scope when the producer has it. Kept optional so
     /// metadata corruption and legacy producers do not invent a scope.
     pub scope: Option<MrfScope>,
@@ -90,13 +151,14 @@ pub struct MrfIntent {
     pub attempts: u8,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct MrfDurableRepairAnchor {
     pub kind: MrfKind,
     pub bucket: Arc<str>,
     pub object: Arc<str>,
     pub version_id: Option<[u8; 16]>,
     pub scope: Option<MrfScope>,
+    pub delete_marker_purge: Option<MrfDeleteMarkerPurgeIdentity>,
     pub lease: MrfIngressLease,
     pub bucket_incarnation_id: Uuid,
 }
@@ -106,17 +168,22 @@ impl MrfDurableRepairAnchor {
     /// incarnation and the original ingress lease. Legacy replay records lack
     /// both pieces and therefore remain fail-closed.
     pub fn from_intent(intent: &MrfIntent, bucket_incarnation_id: Uuid) -> Option<Self> {
+        let lease = intent.lease?;
+        let (version_id, scope) = canonical_identity(intent.kind, intent.version_id, intent.scope);
+        let delete_marker_purge = canonical_delete_marker_purge_identity(intent)?;
+        let bucket_incarnation_id = delete_marker_purge
+            .map(|identity| identity.bucket_incarnation_id)
+            .unwrap_or(bucket_incarnation_id);
         if bucket_incarnation_id.is_nil() {
             return None;
         }
-        let lease = intent.lease?;
-        let (version_id, scope) = canonical_identity(intent.kind, intent.version_id, intent.scope);
         Some(Self {
             kind: intent.kind,
             bucket: intent.bucket.clone(),
             object: intent.object.clone(),
             version_id,
             scope,
+            delete_marker_purge,
             lease,
             bucket_incarnation_id,
         })
@@ -131,6 +198,7 @@ impl MrfDurableRepairAnchor {
             && self.object == event.object
             && self.version_id == event.version_id
             && self.scope == event.scope
+            && self.delete_marker_purge == event.delete_marker_purge
             && self.lease == lease
             && self.bucket_incarnation_id == event.bucket_incarnation_id
     }
@@ -145,6 +213,38 @@ pub fn consume_verified_mrf_repair_events(anchors: &mut Vec<MrfDurableRepairAnch
     before.saturating_sub(anchors.len())
 }
 
+/// Consume recorded verified repairs for one bucket without draining
+/// unrelated or still-unmatched proofs. If the caller crashes before
+/// persisting the retained anchor set, the proof may be replayed by repair
+/// instead of silently deleting the old responsibility.
+pub fn consume_recorded_verified_mrf_repair_events_for(bucket: &str, anchors: &mut Vec<MrfDurableRepairAnchor>) -> usize {
+    let Some(registry) = MRF_VERIFIED_REPAIR_EVENTS.get() else {
+        return 0;
+    };
+    let Ok(mut events) = registry.lock() else {
+        return 0;
+    };
+    let before = anchors.len();
+    let mut retained = std::collections::VecDeque::with_capacity(events.len());
+    while let Some(event) = events.pop_front() {
+        if event.bucket.as_ref() != bucket {
+            retained.push_back(event);
+            continue;
+        }
+        let mut matched = false;
+        anchors.retain(|anchor| {
+            let proven = anchor.is_proven_by(&event);
+            matched |= proven;
+            !proven
+        });
+        if !matched {
+            retained.push_back(event);
+        }
+    }
+    *events = retained;
+    before.saturating_sub(anchors.len())
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub struct MrfScope {
     pub pool_index: u32,
@@ -169,6 +269,7 @@ pub enum MrfDropReason {
     Uninitialized,
     Full,
     OversizedIdentity,
+    InvalidPayload,
     CoalescerFull,
 }
 
@@ -187,12 +288,133 @@ impl MrfIntent {
     pub fn estimated_bytes(&self) -> usize {
         // Struct + strings + version bytes; buckets and objects are usually
         // far below this bound, so rounding up keeps the budget conservative.
-        64 + self.bucket.len() + self.object.len()
+        64usize
+            .saturating_add(self.bucket.len())
+            .saturating_add(self.object.len())
+            .saturating_add(
+                self.delete_marker_purge
+                    .as_ref()
+                    .map_or(0, |purge| std::mem::size_of::<MrfDeleteMarkerPurge>().saturating_add(purge.marker.len())),
+            )
     }
 }
 
 static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
 
+static GLOBAL_DURABLE_MRF_SENDER: OnceLock<mpsc::Sender<MrfDurableSubmission>> = OnceLock::new();
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
+pub enum MrfDurableAdmissionError {
+    #[error("MRF delivery is disabled")]
+    Disabled,
+    #[error("MRF durable consumer is unavailable")]
+    Unavailable,
+    #[error("MRF responsibility capacity is exhausted")]
+    Full,
+    #[error("MRF responsibility identity is invalid or oversized")]
+    InvalidIdentity,
+    #[error("MRF responsibility checkpoint could not be persisted")]
+    Persistence,
+    #[error("MRF durable admission timed out; responsibility may still be retained")]
+    Timeout,
+}
+
+/// A committed storage responsibility awaiting a durable checkpoint, separate
+/// from the best-effort read/scanner channel. Dropping the reply does not
+/// cancel repair.
+pub struct MrfDurableSubmission {
+    pub intent: MrfIntent,
+    pub response: oneshot::Sender<Result<(), MrfDurableAdmissionError>>,
+}
+
+pub fn init_durable_mrf_channel() -> Result<mpsc::Receiver<MrfDurableSubmission>, &'static str> {
+    let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY);
+    GLOBAL_DURABLE_MRF_SENDER
+        .set(sender)
+        .map_err(|_| "MRF durable channel sender already initialized")?;
+    Ok(receiver)
+}
+
+/// Wait for checkpoint publication on a committed partial-write path. Every
+/// write gets a fresh lease: a previous repair of the same unversioned key
+/// cannot discharge a later overwrite. An error is not durable admission.
+pub async fn persist_partial_write_intent(
+    bucket: &str,
+    object: &str,
+    version_id: Option<Uuid>,
+    scope: MrfScope,
+) -> Result<(), MrfDurableAdmissionError> {
+    let intent = MrfIntent {
+        bucket: Arc::from(bucket),
+        object: Arc::from(object),
+        version_id: canonical_version(version_id),
+        kind: MrfKind::PartialWrite,
+        delete_marker_purge: None,
+        scope: Some(scope),
+        lease: None,
+        enqueued_at_ms: unix_now_ms(),
+        attempts: 0,
+    };
+    persist_durable_intent(intent).await
+}
+
+pub async fn persist_delete_marker_purge_intent(
+    bucket: &str,
+    object: &str,
+    version_id: Uuid,
+    scope: MrfScope,
+    delete_marker_purge: MrfDeleteMarkerPurge,
+) -> Result<(), MrfDurableAdmissionError> {
+    if version_id.is_nil() {
+        return Err(MrfDurableAdmissionError::InvalidIdentity);
+    }
+    let intent = MrfIntent {
+        bucket: Arc::from(bucket),
+        object: Arc::from(object),
+        version_id: canonical_version(Some(version_id)),
+        kind: MrfKind::DeleteMarkerPurge,
+        delete_marker_purge: Some(delete_marker_purge),
+        scope: Some(scope),
+        lease: None,
+        enqueued_at_ms: unix_now_ms(),
+        attempts: 0,
+    };
+    persist_durable_intent_unconditionally(intent).await
+}
+
+async fn persist_durable_intent(intent: MrfIntent) -> Result<(), MrfDurableAdmissionError> {
+    if !mrf_delivery_enabled() {
+        return Err(MrfDurableAdmissionError::Disabled);
+    }
+    persist_durable_intent_unconditionally(intent).await
+}
+
+async fn persist_durable_intent_unconditionally(mut intent: MrfIntent) -> Result<(), MrfDurableAdmissionError> {
+    if intent.bucket.is_empty()
+        || intent.object.is_empty()
+        || intent.bucket.len() > MRF_MAX_IDENTITY_COMPONENT
+        || intent.object.len() > MRF_MAX_IDENTITY_COMPONENT
+        || !valid_intent_payload(&intent)
+    {
+        return Err(MrfDurableAdmissionError::InvalidIdentity);
+    }
+    let sender = GLOBAL_DURABLE_MRF_SENDER.get().ok_or(MrfDurableAdmissionError::Unavailable)?;
+    if try_rearm_mrf_replay_intent(&mut intent) != MrfIngressResult::Enqueued {
+        return Err(MrfDurableAdmissionError::InvalidIdentity);
+    }
+    let (response, receipt) = oneshot::channel();
+    sender
+        .try_send(MrfDurableSubmission { intent, response })
+        .map_err(|err| match err {
+            mpsc::error::TrySendError::Full(_) => MrfDurableAdmissionError::Full,
+            mpsc::error::TrySendError::Closed(_) => MrfDurableAdmissionError::Unavailable,
+        })?;
+    tokio::time::timeout(Duration::from_secs(10), receipt)
+        .await
+        .map_err(|_| MrfDurableAdmissionError::Timeout)?
+        .map_err(|_| MrfDurableAdmissionError::Unavailable)?
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 struct MrfIdentityKey {
     kind: MrfKind,
@@ -246,10 +468,27 @@ fn canonical_identity(
     let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
     match kind {
         MrfKind::MetadataCorruption => (None, None),
-        MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
+        MrfKind::DecodeFailure | MrfKind::PartialWrite | MrfKind::DeleteMarkerPurge => (version_id, scope),
     }
 }
 
+fn canonical_delete_marker_purge_identity(intent: &MrfIntent) -> Option<Option<MrfDeleteMarkerPurgeIdentity>> {
+    match intent.kind {
+        MrfKind::DeleteMarkerPurge => intent
+            .delete_marker_purge
+            .as_ref()
+            .filter(|payload| payload.is_valid())
+            .map(|payload| Some(payload.identity())),
+        _ if intent.delete_marker_purge.is_none() => Some(None),
+        _ => None,
+    }
+}
+
+fn valid_intent_payload(intent: &MrfIntent) -> bool {
+    canonical_delete_marker_purge_identity(intent).is_some()
+        && (intent.kind != MrfKind::DeleteMarkerPurge || (intent.version_id.is_some() && intent.scope.is_some()))
+}
+
 fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
     64usize
         .saturating_add(key.bucket.len())
@@ -399,6 +638,9 @@ pub fn try_send_mrf_intent_typed(
     if !mrf_delivery_enabled() {
         return MrfIngressResult::Dropped(MrfDropReason::Disabled);
     }
+    if kind == MrfKind::DeleteMarkerPurge {
+        return MrfIngressResult::Dropped(MrfDropReason::InvalidPayload);
+    }
     let Some(sender) = GLOBAL_MRF_SENDER.get() else {
         return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
     };
@@ -422,6 +664,7 @@ pub fn try_send_mrf_intent_typed(
         object: key.object.clone(),
         version_id: key.version_id,
         kind,
+        delete_marker_purge: None,
         scope,
         lease: Some(lease),
         enqueued_at_ms: unix_now_ms(),
@@ -449,15 +692,18 @@ pub fn try_send_mrf_intent_typed(
 /// producer coalescer key: the replay queue first deduplicates legacy records,
 /// then manager admission owns task-level deduplication with live producers.
 pub fn try_rearm_mrf_replay_intent(intent: &mut MrfIntent) -> MrfIngressResult {
-    if intent.lease.is_some() {
-        return MrfIngressResult::Enqueued;
-    }
     if intent.bucket.len() > MRF_MAX_IDENTITY_COMPONENT || intent.object.len() > MRF_MAX_IDENTITY_COMPONENT {
         return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
     }
     let (version_id, scope) = canonical_identity(intent.kind, intent.version_id, intent.scope);
     intent.version_id = version_id;
     intent.scope = scope;
+    if !valid_intent_payload(intent) {
+        return MrfIngressResult::Dropped(MrfDropReason::InvalidPayload);
+    }
+    if intent.lease.is_some() {
+        return MrfIngressResult::Enqueued;
+    }
     intent.lease = Some(MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed)));
     MrfIngressResult::Enqueued
 }
@@ -522,6 +768,7 @@ pub struct MrfVerifiedRepairEvent {
     pub object: Arc<str>,
     pub version_id: Option<[u8; 16]>,
     pub scope: Option<MrfScope>,
+    pub delete_marker_purge: Option<MrfDeleteMarkerPurgeIdentity>,
     pub lease: Option<MrfIngressLease>,
     pub bucket_incarnation_id: Uuid,
     pub disposition: MrfVerifiedRepairDisposition,
@@ -623,6 +870,7 @@ mod tests {
             object: Arc::from("object"),
             version_id: Some([0u8; 16]),
             kind: MrfKind::DecodeFailure,
+            delete_marker_purge: None,
             scope: None,
             lease: None,
             enqueued_at_ms: 0,
@@ -687,6 +935,7 @@ mod tests {
             object: Arc::from("object"),
             version_id: Some([0; 16]),
             kind: MrfKind::PartialWrite,
+            delete_marker_purge: None,
             scope: Some(MrfScope {
                 pool_index: 1,
                 set_index: 2,
@@ -726,6 +975,7 @@ mod tests {
             object: Arc::from("object"),
             version_id: Some([0; 16]),
             kind: MrfKind::PartialWrite,
+            delete_marker_purge: None,
             scope: Some(MrfScope {
                 pool_index: 2,
                 set_index: 3,
@@ -760,6 +1010,7 @@ mod tests {
                 pool_index: 4,
                 set_index: 5,
             }),
+            delete_marker_purge: None,
             lease,
             bucket_incarnation_id: incarnation,
         };
@@ -769,6 +1020,7 @@ mod tests {
             object,
             version_id: anchor.version_id,
             scope: anchor.scope,
+            delete_marker_purge: None,
             lease: Some(lease),
             bucket_incarnation_id: incarnation,
             disposition: MrfVerifiedRepairDisposition::Repaired,
@@ -821,6 +1073,73 @@ mod tests {
         assert!(retained.is_empty());
     }
 
+    #[test]
+    fn recorded_verified_repair_consumer_retains_unmatched_proofs() {
+        let bucket = Arc::<str>::from(format!("recorded-proof-{}", Uuid::new_v4()));
+        let other_bucket = Arc::<str>::from(format!("recorded-proof-other-{}", Uuid::new_v4()));
+        let incarnation = Uuid::new_v4();
+        let lease = MrfIngressLease::new(21);
+        let retained_anchor = MrfDurableRepairAnchor {
+            kind: MrfKind::PartialWrite,
+            bucket: bucket.clone(),
+            object: Arc::from("retained"),
+            version_id: Some([7; 16]),
+            scope: Some(MrfScope {
+                pool_index: 1,
+                set_index: 2,
+            }),
+            delete_marker_purge: None,
+            lease,
+            bucket_incarnation_id: incarnation,
+        };
+        let waiting_anchor = MrfDurableRepairAnchor {
+            object: Arc::from("waiting"),
+            lease: MrfIngressLease::new(22),
+            ..retained_anchor.clone()
+        };
+        let matched_event = MrfVerifiedRepairEvent {
+            kind: retained_anchor.kind,
+            bucket: bucket.clone(),
+            object: retained_anchor.object.clone(),
+            version_id: retained_anchor.version_id,
+            scope: retained_anchor.scope,
+            delete_marker_purge: None,
+            lease: Some(retained_anchor.lease),
+            bucket_incarnation_id: retained_anchor.bucket_incarnation_id,
+            disposition: MrfVerifiedRepairDisposition::Repaired,
+        };
+        let same_bucket_unmatched = MrfVerifiedRepairEvent {
+            object: Arc::from("future"),
+            lease: Some(MrfIngressLease::new(23)),
+            ..matched_event.clone()
+        };
+        let other_bucket_event = MrfVerifiedRepairEvent {
+            bucket: other_bucket.clone(),
+            ..matched_event.clone()
+        };
+
+        note_mrf_verified_repair(matched_event);
+        note_mrf_verified_repair(same_bucket_unmatched.clone());
+        note_mrf_verified_repair(other_bucket_event.clone());
+
+        let mut anchors = vec![retained_anchor, waiting_anchor.clone()];
+        assert_eq!(consume_recorded_verified_mrf_repair_events_for(&bucket, &mut anchors), 1);
+        assert_eq!(anchors, vec![waiting_anchor]);
+
+        let remaining_bucket_events = take_mrf_verified_repair_events_for(&bucket);
+        assert_eq!(
+            remaining_bucket_events,
+            vec![same_bucket_unmatched],
+            "same-bucket proofs without a retained anchor must remain available"
+        );
+        let remaining_other_events = take_mrf_verified_repair_events_for(&other_bucket);
+        assert_eq!(
+            remaining_other_events,
+            vec![other_bucket_event],
+            "proofs for other buckets must not be drained by this consumer"
+        );
+    }
+
     #[tokio::test]
     async fn try_send_delivers_and_respects_capacity() {
         let mut receiver = init_mrf_channel().expect("first initialization should succeed");
@@ -898,6 +1217,7 @@ mod tests {
                 pool_index: 2,
                 set_index: 3,
             }),
+            delete_marker_purge: None,
             lease: Some(MrfIngressLease::new(42)),
             bucket_incarnation_id,
             disposition: MrfVerifiedRepairDisposition::Repaired,
@@ -914,3 +1234,70 @@ mod tests {
         assert_eq!(take_mrf_verified_repair_events_for("verified-bucket-b").len(), 1);
     }
 }
+#[tokio::test]
+async fn partial_write_durable_admission_waits_for_persistence_and_renews_lease() {
+    let mut receiver = init_durable_mrf_channel().expect("isolated test should initialize durable channel");
+    let scope = MrfScope {
+        pool_index: 2,
+        set_index: 3,
+    };
+    let first = tokio::spawn(async move { persist_partial_write_intent("bucket", "object", None, scope).await });
+    let submission = receiver.recv().await.expect("first submission should arrive");
+    let first_lease = submission.intent.lease;
+    assert!(!first.is_finished(), "channel delivery alone must not acknowledge durable ownership");
+    submission
+        .response
+        .send(Err(MrfDurableAdmissionError::Persistence))
+        .expect("producer should await receipt");
+    assert_eq!(first.await.expect("producer should complete"), Err(MrfDurableAdmissionError::Persistence));
+    let second = tokio::spawn(async move { persist_partial_write_intent("bucket", "object", None, scope).await });
+    let submission = receiver.recv().await.expect("replacement submission should arrive");
+    assert_ne!(
+        first_lease, submission.intent.lease,
+        "an overwrite must not reuse the previous proof identity"
+    );
+    assert_eq!(submission.intent.scope, Some(scope));
+    submission
+        .response
+        .send(Ok(()))
+        .expect("second producer should await receipt");
+    assert_eq!(second.await.expect("second producer should complete"), Ok(()));
+
+    let bucket_incarnation_id = Uuid::new_v4();
+    let marker_identity = [7; 32];
+    let purge = MrfDeleteMarkerPurge::new(bucket_incarnation_id, bucket_incarnation_id, marker_identity, vec![1, 2, 3])
+        .expect("valid purge payload");
+    let version_id = Uuid::new_v4();
+    let purge_submit =
+        tokio::spawn(async move { persist_delete_marker_purge_intent("bucket", "marker", version_id, scope, purge).await });
+    let submission = receiver.recv().await.expect("purge submission should arrive");
+    assert_eq!(submission.intent.kind, MrfKind::DeleteMarkerPurge);
+    assert_eq!(submission.intent.version_id, Some(*version_id.as_bytes()));
+    assert_eq!(submission.intent.scope, Some(scope));
+    assert_eq!(
+        submission
+            .intent
+            .delete_marker_purge
+            .as_ref()
+            .map(MrfDeleteMarkerPurge::identity),
+        Some(MrfDeleteMarkerPurgeIdentity {
+            bucket_incarnation_id,
+            marker_incarnation_id: bucket_incarnation_id,
+            marker_identity,
+        })
+    );
+    assert!(submission.intent.lease.is_some());
+    submission.response.send(Ok(())).expect("purge producer should await receipt");
+    assert_eq!(purge_submit.await.expect("purge producer should complete"), Ok(()));
+
+    let invalid = MrfDeleteMarkerPurge {
+        bucket_incarnation_id,
+        marker_incarnation_id: Uuid::new_v4(),
+        marker_identity,
+        marker: Arc::from([1u8]),
+    };
+    assert_eq!(
+        persist_delete_marker_purge_intent("bucket", "marker", version_id, scope, invalid).await,
+        Err(MrfDurableAdmissionError::InvalidIdentity)
+    );
+}

crates/common/src/trace_bus.rs

@@ -127,6 +127,44 @@ pub struct TraceEvent {
     pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>,
 }
 
+/// A telemetry operation whose published shape cannot retain request data.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TelemetryTraceOperation {
+    GetObject,
+    PutObject,
+    HeadObject,
+    ListObjects,
+    InternalRpc,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TelemetryTraceStatus {
+    Ok,
+    Error,
+}
+
+/// A pre-classified S3 or internode RPC observation.
+///
+/// This event deliberately has no request path, headers, object identity,
+/// payload, or free-form attributes. Producers must classify the operation
+/// and outcome before publishing it.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TelemetryTraceEvent {
+    pub operation: TelemetryTraceOperation,
+    pub duration: Duration,
+    pub status: TelemetryTraceStatus,
+}
+
+impl TelemetryTraceEvent {
+    pub const fn new(operation: TelemetryTraceOperation, duration: Duration, status: TelemetryTraceStatus) -> Self {
+        Self {
+            operation,
+            duration,
+            status,
+        }
+    }
+}
+
 impl TraceEvent {
     pub fn new(kind: TraceKind, func: TraceFunc) -> Self {
         Self {
@@ -174,15 +212,20 @@ impl TraceEvent {
 pub struct TraceBus {
     sender: broadcast::Sender<Arc<TraceEvent>>,
     subscriber_count: Arc<AtomicUsize>,
+    telemetry_sender: broadcast::Sender<Arc<TelemetryTraceEvent>>,
+    telemetry_subscriber_count: Arc<AtomicUsize>,
 }
 
 impl TraceBus {
     pub fn new(capacity: usize) -> Self {
         let capacity = capacity.max(1);
         let (sender, _receiver) = broadcast::channel(capacity);
+        let (telemetry_sender, _telemetry_receiver) = broadcast::channel(capacity);
         Self {
             sender,
             subscriber_count: Arc::new(AtomicUsize::new(0)),
+            telemetry_sender,
+            telemetry_subscriber_count: Arc::new(AtomicUsize::new(0)),
         }
     }
 
@@ -206,6 +249,27 @@ impl TraceBus {
 
         self.sender.send(Arc::new(build())).is_ok()
     }
+
+    pub fn telemetry_subscriber_count(&self) -> usize {
+        self.telemetry_subscriber_count.load(Ordering::Acquire)
+    }
+
+    pub fn subscribe_telemetry(&self) -> TelemetryTraceSubscription {
+        let receiver = self.telemetry_sender.subscribe();
+        self.telemetry_subscriber_count.fetch_add(1, Ordering::AcqRel);
+        TelemetryTraceSubscription {
+            receiver,
+            subscriber_count: Arc::clone(&self.telemetry_subscriber_count),
+        }
+    }
+
+    pub fn emit_telemetry(&self, build: impl FnOnce() -> TelemetryTraceEvent) -> bool {
+        if self.telemetry_subscriber_count() == 0 {
+            return false;
+        }
+
+        self.telemetry_sender.send(Arc::new(build())).is_ok()
+    }
 }
 
 impl Default for TraceBus {
@@ -236,6 +300,28 @@ impl Drop for TraceSubscription {
     }
 }
 
+#[derive(Debug)]
+pub struct TelemetryTraceSubscription {
+    receiver: broadcast::Receiver<Arc<TelemetryTraceEvent>>,
+    subscriber_count: Arc<AtomicUsize>,
+}
+
+impl TelemetryTraceSubscription {
+    pub async fn recv(&mut self) -> Result<Arc<TelemetryTraceEvent>, broadcast::error::RecvError> {
+        self.receiver.recv().await
+    }
+
+    pub fn try_recv(&mut self) -> Result<Arc<TelemetryTraceEvent>, broadcast::error::TryRecvError> {
+        self.receiver.try_recv()
+    }
+}
+
+impl Drop for TelemetryTraceSubscription {
+    fn drop(&mut self) {
+        self.subscriber_count.fetch_sub(1, Ordering::AcqRel);
+    }
+}
+
 pub fn global_trace_bus() -> &'static TraceBus {
     GLOBAL_TRACE_BUS.get_or_init(TraceBus::default)
 }
@@ -252,6 +338,18 @@ pub fn trace_subscriber_count() -> usize {
     global_trace_bus().subscriber_count()
 }
 
+pub fn subscribe_telemetry_trace_events() -> TelemetryTraceSubscription {
+    global_trace_bus().subscribe_telemetry()
+}
+
+pub fn telemetry_trace_emit(build: impl FnOnce() -> TelemetryTraceEvent) -> bool {
+    global_trace_bus().emit_telemetry(build)
+}
+
+pub fn telemetry_trace_subscriber_count() -> usize {
+    global_trace_bus().telemetry_subscriber_count()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -330,4 +428,30 @@ mod tests {
             .expect_err("receiver should observe lag instead of blocking publishers");
         assert!(matches!(err, broadcast::error::RecvError::Lagged(_)));
     }
+
+    #[tokio::test]
+    async fn telemetry_subscription_exposes_only_classified_fields() {
+        let bus = TraceBus::new(4);
+        let _unrelated_subscription = bus.subscribe();
+        let built = AtomicUsize::new(0);
+        assert!(!bus.emit_telemetry(|| {
+            built.fetch_add(1, Ordering::Relaxed);
+            TelemetryTraceEvent::new(TelemetryTraceOperation::GetObject, Duration::ZERO, TelemetryTraceStatus::Ok)
+        }));
+        assert_eq!(built.load(Ordering::Relaxed), 0);
+
+        let mut subscription = bus.subscribe_telemetry();
+        assert_eq!(bus.telemetry_subscriber_count(), 1);
+
+        assert!(bus.emit_telemetry(|| {
+            TelemetryTraceEvent::new(TelemetryTraceOperation::GetObject, Duration::from_micros(7), TelemetryTraceStatus::Ok)
+        }));
+
+        let event = subscription.recv().await.expect("classified telemetry event");
+        assert_eq!(event.operation, TelemetryTraceOperation::GetObject);
+        assert_eq!(event.duration, Duration::from_micros(7));
+        assert_eq!(event.status, TelemetryTraceStatus::Ok);
+        drop(subscription);
+        assert_eq!(bus.telemetry_subscriber_count(), 0);
+    }
 }

crates/config/README.md

@@ -62,6 +62,37 @@ Current guidance:
 - `RUSTFS_CORS_ALLOWED_ORIGINS` defaults to empty, so the S3 endpoint emits no generic CORS headers unless configured. Set `*` for wildcard origins without credentials, or a comma-separated allow-list for credentialed explicit origins.
 - `RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS` defaults to `*` for the console service.
 
+## Console URL prefix
+
+`RUSTFS_CONSOLE_PREFIX` changes the embedded console URL prefix. The default is
+`/rustfs/console`. For example, `RUSTFS_CONSOLE_PREFIX=/console` serves the UI at
+`http://localhost:9001/console/`. Nested prefixes such as `/management/console`
+are supported; one trailing slash is removed. Restart the server after changing it.
+
+The prefix must be a non-root absolute path of at most 256 bytes, with nonempty
+segments containing only ASCII letters, digits, `-`, `_`, `.`, or `~`. Dot
+segments, encoded characters, and overlaps with reserved admin, RPC, health,
+profiling, browser entry, and icon routes are rejected at startup. `/` is not supported.
+Choose a prefix that does not collide with S3 bucket paths.
+
+The console routes, embedded frontend asset URLs, browser redirects, and OIDC
+console redirects use this prefix. Admin API paths and the identity provider's
+`/rustfs/admin/v3/oidc/callback/...` URL remain unchanged. `RUSTFS_CONSOLE_ADDRESS`
+continues to control only the listening address and port. The server adapts bundled
+console asset references from their build-time base path to the runtime prefix.
+
+OEM builds can set `RUSTFS_CONSOLE_BASE_PATH` when compiling RustFS to embed a
+different default, such as `/nuofans/console`. Build the bundled console with the
+same `NEXT_PUBLIC_BASE_PATH`. An unset or empty build variable retains
+`/rustfs/console`. The build path must satisfy the validation rules above and must
+not have a trailing slash.
+
+At startup, `RUSTFS_CONSOLE_PREFIX` takes precedence over the compiled default.
+Changing `RUSTFS_CONSOLE_BASE_PATH` when starting an existing binary has no effect;
+rebuild both components to change the embedded default. If a runtime prefix is
+configured, asset adaptation uses the compiled base path as its source, including
+when restoring `/rustfs/console` for a custom OEM build.
+
 ## Browser redirect environment variables
 
 - `RUSTFS_BROWSER_REDIRECT_URL` sets the externally reachable browser origin used for OIDC callback, console success redirect, and logout fallback URLs. Configure it to the public scheme and authority without a path, for example `https://console.example.com`. In load-balancer deployments, keep OIDC authorize and callback requests on the same backend node because the in-flight OIDC `state` is local to the RustFS node.

crates/config/src/constants/app.rs

@@ -213,6 +213,11 @@ pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
 /// Environment variable for console server address.
 pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
 
+/// URL path prefix for the embedded console, read once at server startup.
+pub const ENV_RUSTFS_CONSOLE_PREFIX: &str = "RUSTFS_CONSOLE_PREFIX";
+/// Default embedded console URL path prefix.
+pub const DEFAULT_CONSOLE_PREFIX: &str = "/rustfs/console";
+
 /// Public browser entrypoint used to build OIDC callback and console redirects.
 ///
 /// This should be the externally reachable scheme and authority, without a path.

crates/config/src/constants/body_limits.rs

@@ -64,6 +64,14 @@ pub const MAX_HEAL_REQUEST_SIZE: usize = 1024 * 1024; // 1 MB
 /// memory exhaustion from malicious or misconfigured remote services.
 pub const MAX_S3_CLIENT_RESPONSE_SIZE: usize = 10 * 1024 * 1024; // 10 MB
 
+/// Maximum body size accepted by a single `PutObject` or `UploadPart` request (5 GiB).
+/// Used for: the s3s streaming-body limit and the request-header admission check.
+/// Rationale: matches the AWS S3 single-PUT / single-part ceiling. Larger objects
+/// must use multipart upload. The header check rejects an oversize
+/// `Content-Length` before any body byte is read so the client gets
+/// `EntityTooLarge` immediately instead of streaming 5 GiB into a mid-stream failure.
+pub const MAX_SINGLE_PUT_OBJECT_SIZE: u64 = 5 * 1024 * 1024 * 1024; // 5 GiB
+
 /// Maximum size for OIDC provider response bodies (1 MB)
 /// Used for: discovery documents, JWKS documents and token endpoint responses
 /// Rationale: a hostile or compromised identity provider must not be able to exhaust

crates/config/src/constants/heal.rs

@@ -208,9 +208,9 @@ pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256;
 
 /// Environment variable selecting how admin heal starts behave when the
 /// requested path overlaps an already running or queued heal: `merge`
-/// (default, keep today's dedup/merge semantics) or `minio_error` (return a
-/// typed already-running / overlapping-paths rejection like madmin).
+/// (default, merge equivalent starts and reject other overlapping admin
+/// scopes) or `minio_error` (also reject equivalent starts like madmin).
 pub const ENV_HEAL_OVERLAP_POLICY: &str = "RUSTFS_HEAL_OVERLAP_POLICY";
 
-/// Default overlap policy: merge duplicate/overlapping requests.
+/// Default overlap policy: merge equivalent requests, reject overlapping scopes.
 pub const DEFAULT_HEAL_OVERLAP_POLICY: &str = "merge";

crates/config/src/constants/object.rs

@@ -116,6 +116,30 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
 /// Default: bitrot verification is enabled on GetObject reads (do not skip).
 pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
 
+/// Create independent shard commitments for new writes after the fleet is upgraded.
+/// Existing protected objects and multipart uploads retain their protection.
+pub const ENV_SHARD_INTEGRITY_WRITE: &str = "RUSTFS_SHARD_INTEGRITY_WRITE";
+pub const DEFAULT_SHARD_INTEGRITY_WRITE: bool = false;
+
+/// Operator confirmation that every reader, writer and background coordinator
+/// understands independent shard commitments. This is not capability discovery
+/// or a fence against an old binary rejoining the fleet.
+pub const ENV_SHARD_INTEGRITY_FLEET_CONFIRMED: &str = "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED";
+pub const DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED: bool = false;
+
+const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_WRITE);
+const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED);
+
+/// How object writes treat a bucket whose stored versioning configuration
+/// cannot be parsed: `permissive` writes as if unversioned (the historical
+/// behavior, recorded by metrics and an error log) and `strict` refuses the
+/// write with 503. Paths that already refuse an unreadable configuration do
+/// so in both modes. Any other value fails startup.
+pub const ENV_BUCKET_CONFIG_PARSE_MODE: &str = "RUSTFS_BUCKET_CONFIG_PARSE_MODE";
+
+/// Default bucket config parse mode.
+pub const DEFAULT_BUCKET_CONFIG_PARSE_MODE: &str = "permissive";
+
 /// Request writing the complete remote-tier version state into object metadata.
 ///
 /// This remains ineffective until

crates/config/src/constants/runtime.rs

@@ -39,7 +39,7 @@ pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET";
 /// Accepted but not honoured; see [`ENV_RUNTIME_DIAL9_S3_BUCKET`].
 pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
 // Note: there are deliberately no task-dump knobs. dial9 only captures task
-// dumps for futures spawned through `dial9_tokio_telemetry::spawn`, and RustFS
+// dumps for futures spawned through `dial9::spawn`, and RustFS
 // spawns with `tokio::spawn` throughout, so the switch could never do anything.
 // Measured: 0 dumps via tokio::spawn vs 14709 via dial9::spawn on an identical
 // workload. See rustfs/backlog#1157 (D9-16).

crates/data-usage/src/data_usage.rs

@@ -593,6 +593,20 @@ pub struct DataUsageSnapshotIdentity {
     pub scanner_epoch: Option<u64>,
 }
 
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct DataUsageSegmentInvalidationProof {
+    #[serde(default)]
+    pub process_epoch: String,
+    #[serde(default)]
+    pub generation_start: u64,
+    #[serde(default)]
+    pub generation_end: u64,
+    #[serde(default)]
+    pub producer_identity_coverage_complete: bool,
+    #[serde(default)]
+    pub cold_zero_walk_oracle: bool,
+}
+
 #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DataUsageSnapshotSetState {
     pub pool_index: u64,
@@ -607,6 +621,8 @@ pub struct DataUsageSnapshotSetState {
     pub complete: bool,
     #[serde(default)]
     pub tombstone: bool,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub segment_invalidation_proof: Option<DataUsageSegmentInvalidationProof>,
 }
 
 impl DataUsageInfo {
@@ -3073,6 +3089,7 @@ mod tests {
             scan_plan_digest: Some([1; 32]),
             complete: false,
             tombstone: false,
+            segment_invalidation_proof: None,
         }];
         assert!(observed_data_usage_is_newer(&partial, &authoritative));
     }
@@ -3095,6 +3112,7 @@ mod tests {
                     scan_plan_digest: Some([1; 32]),
                     complete: true,
                     tombstone: false,
+                    segment_invalidation_proof: None,
                 },
                 DataUsageSnapshotSetState {
                     pool_index: 1,
@@ -3104,6 +3122,7 @@ mod tests {
                     scan_plan_digest: Some([2; 32]),
                     complete: false,
                     tombstone: false,
+                    segment_invalidation_proof: None,
                 },
             ],
             ..Default::default()
@@ -3113,6 +3132,42 @@ mod tests {
         assert!(partial.is_valid_partial_snapshot());
     }
 
+    #[test]
+    fn set_state_segment_invalidation_proof_is_additive() {
+        #[derive(Deserialize)]
+        struct LegacySetState {
+            pool_index: u64,
+            set_index: u64,
+            complete: bool,
+        }
+
+        let proof = DataUsageSegmentInvalidationProof {
+            process_epoch: "scanner-process".to_string(),
+            generation_start: 3,
+            generation_end: 5,
+            producer_identity_coverage_complete: true,
+            cold_zero_walk_oracle: true,
+        };
+        let state = DataUsageSnapshotSetState {
+            pool_index: 1,
+            set_index: 2,
+            scanner_cycle: Some(9),
+            scanner_epoch: Some(4),
+            scan_plan_digest: Some([7; 32]),
+            complete: true,
+            tombstone: false,
+            segment_invalidation_proof: Some(proof.clone()),
+        };
+        let encoded = rmp_serde::to_vec_named(&state).expect("set state should encode with additive proof");
+        let legacy: LegacySetState = rmp_serde::from_slice(&encoded).expect("legacy readers should ignore proof metadata");
+        assert_eq!(legacy.pool_index, 1);
+        assert_eq!(legacy.set_index, 2);
+        assert!(legacy.complete);
+
+        let decoded: DataUsageSnapshotSetState = rmp_serde::from_slice(&encoded).expect("new readers should restore proof");
+        assert_eq!(decoded.segment_invalidation_proof, Some(proof));
+    }
+
     #[test]
     fn completeness_marker_requires_a_snapshot_timestamp() {
         let untimestamped = DataUsageInfo {

crates/e2e_test/AGENTS.md

@@ -27,4 +27,4 @@ follow.
 
 ## Suggested Validation
 
-- `cargo test --package e2e_test`
+- `python3 scripts/e2e_binary.py build --features e2e-test-hooks`, then `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo test --package e2e_test` (running `cargo test -p e2e_test` directly fails with a missing E2E run receipt; see [`README.md`](README.md#how-to-run))

crates/e2e_test/Cargo.toml

@@ -131,7 +131,7 @@ md-5 = { workspace = true }
 opentelemetry-proto = { workspace = true }
 prost.workspace = true
 sha2 = { workspace = true }
-astral-tokio-tar = { workspace = true }
+rustfs-tokio-tar = { workspace = true }
 s3s = { workspace = true, features = ["minio"] }
 zstd.workspace = true
 time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }

crates/e2e_test/README.md

@@ -1,7 +1,7 @@
 # e2e_test
 
 End-to-end test suite for RustFS. Each test spawns a **real `rustfs` binary**
-(built on demand from the workspace) and drives it over the network with the
+(built and identified before the test invocation) and drives it over the network with the
 AWS SDK (`aws-sdk-s3`), raw HTTP (`reqwest` / `awscurl`), or a protocol client
 (FTPS / WebDAV / SFTP). This is the black-box integration layer: exhaustive
 end-to-end behavior lives here, unit behavior stays in the source crates
@@ -30,34 +30,39 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
+The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
+
 ## How to run
 
-All commands assume repo root. `cargo test` triggers an on-demand build of the
-`rustfs` binary from [`src/common.rs`](src/common.rs) (`rustfs_binary_path`) on
-first use — the first invocation is slow, later ones reuse the binary.
+All commands assume repo root and Python 3.9 or newer on Linux or macOS. Build the server once through the provenance entry point, then run the test command through the same script:
 
 ```bash
-# Whole crate (default = ignored tests skipped)
-cargo nextest run -p e2e_test
+python3 scripts/e2e_binary.py build --features e2e-test-hooks
+
+# Whole crate (ignored tests remain skipped)
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test
 
 # One module
-cargo nextest run -p e2e_test -E 'test(list_objects_v2_pagination_test)'
-
-# PR smoke subset (see "CI smoke subset" below)
-cargo nextest run --profile e2e-smoke -p e2e_test
-
-# ILM serial lane — ignored lifecycle tests, single-threaded (mirrors CI)
-cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
-  -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test -E 'test(list_objects_v2_pagination_test)'
 
+# PR smoke subset
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
 ```
 
-The protocols suite has its own contract (fixed bind ports 9022–9301,
-single-worker execution, feature-gated scheduling) documented in
-[`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
-selects which features the spawned binary is built with; leave it unset to run
-every protocol entry. Use the exact profile command under
-[Troubleshooting](#troubleshooting) for CI-equivalent execution.
+Root-heal interruption scenarios use a test-only commit barrier, so build and run them with `e2e-test-hooks`:
+
+```bash
+python3 scripts/e2e_binary.py build --features e2e-test-hooks
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test -E 'test(heal_erasure_disk_rebuild_test)'
+```
+
+`build` records the source contents, HEAD, resolved Cargo features, profile, toolchain, and binary SHA-256 beside the executable in `rustfs.e2e.json`. `run` validates that identity before and after the command, preserves command failures, and removes its temporary run receipt on completion. The Rust harness checks that receipt before starting each server; it never compiles a server inside a test process. Source or binary changes during a run invalidate the result, even when the test command succeeds. Use an isolated worktree and keep it unchanged until the command finishes.
+
+The additional `--features` arguments must match between `build` and `run`; Cargo defaults remain enabled. The wrapper supplies `RUSTFS_BUILD_FEATURES` from Cargo's resolved feature list, including features enabled by `full`. Protocol helpers require a subset of that list. `CARGO_TARGET_DIR` and `--profile release` are supported. An in-workspace target directory must be Git-ignored; tracked files are always included in the source identity. `build --bins` preserves CI lanes that compile all RustFS binary targets. For a downloaded artifact, copy both the executable and its sidecar, then use `run`; do not generate a new identity for an arbitrary prebuilt binary. `CARGO_BIN_EXE_rustfs` cannot override the verified executable.
+
+Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; concurrent wrappers fail immediately. Use a private target directory and do not run ordinary Cargo builds against it while tests are active: Cargo does not honor this marker. Interrupted runs fail and terminate their command group. After an uncatchable kill, inspect the PID recorded in a leftover marker and remove it only after confirming its owner has stopped. Embedded file symlinks are hashed through their target; embedded directory symlinks are rejected because their contents cannot be enumerated safely by this entry point.
+
+The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
 
 ### `#[ignore]` semantics
 
@@ -123,7 +128,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
 | `wait_for_server_ready` | Poll readiness before issuing requests |
 | `create_s3_client` / `create_test_bucket` / `delete_test_bucket` | aws-sdk-s3 client + bucket lifecycle |
 | `find_available_port` | Random free port (isolation primitive) |
-| `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
+| `rustfs_binary_path` / `_with_features` | Verify this run's binary receipt and required feature subset |
 | `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
 | `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
 | `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
@@ -189,35 +194,33 @@ the wiring source of truth. Committed test-ID digests under
 **Reproduce a CI failure locally** — run the exact profile/lane:
 
 ```bash
-# Smoke (e2e-tests job) — includes the 20 fast replication tests
-cargo nextest run --profile e2e-smoke -p e2e_test
-# Full single-node merge/main lane
-cargo nextest run --profile e2e-full -p e2e_test
-# Cluster fault nightly lane
-cargo nextest run --profile e2e-nightly -p e2e_test
-# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
-# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
-cargo nextest run --profile e2e-distributed -p e2e_test
-# Replication nightly lane; awscurl is required for STS paths
-cargo nextest run --profile e2e-repl-nightly -p e2e_test
-# Fixed-port protocol nightly lane
-RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
-  cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
-# ILM serial lane
+# Smoke, full, and cluster lanes share a server with fault-test hooks.
+python3 scripts/e2e_binary.py build --features e2e-test-hooks
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
+python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
+
+# Distributed 4-node 4-disk lane uses the default server.
+# Upgrade cases require RUSTFS_UPGRADE_SOURCE_BINARY and fail closed without it.
+python3 scripts/e2e_binary.py build
+python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test
+
+# Replication nightly uses the default server; awscurl is required for STS.
+python3 scripts/e2e_binary.py build
+python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
+
+# Protocol nightly owns fixed ports.
+python3 scripts/e2e_binary.py build --features ftps,webdav,sftp
+python3 scripts/e2e_binary.py run --features ftps,webdav,sftp -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
+
+# The ILM serial lane does not use this server harness.
 cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
   -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
-# s3s-e2e black box
-./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs-e2e-data
 ```
 
-**Stale binary.** Tests build the `rustfs` binary once and reuse it. To avoid
-rebuilding while iterating on tests, `common.rs` reuses an existing binary when
-running *inside* the e2e test process even if sources changed
-(`can_reuse_inside_e2e`, [`src/common.rs`](src/common.rs) line 98). Downside: if
-you changed **server** code, force a rebuild with
-`cargo build -p rustfs` (or `touch` a source file outside the reuse window)
-before re-running, or CI's freshly built artifact will diverge from your local
-one.
+The full lane also requires the startup-CAS build manifest generated by the `Build debug binary` step in `.github/workflows/ci.yml`. Preserve that binary and both sidecars as its `Preserve startup CAS binary input` step does, and use the same `RUSTFS_E2E_STARTUP_CAS_*` environment as `Run e2e full suite`. A generic local build alone does not supply that fixture evidence.
+
+**Stale or unverified binary.** Re-run the matching `build` command after changing source or features, then invoke tests through `run`. A missing receipt, copied old executable, or mismatched build identity is a prerequisite failure. Bare Cargo invocations that start a server deliberately fail; unit tests that do not start a server can still run directly.
 
 **Port already in use / orphan processes.** A hard-killed run can leak a
 `rustfs` child holding its port. Find and kill it:
@@ -249,7 +252,8 @@ spawn error. Install the pinned CI version before running their profiles.
 A subset of this crate runs on every PR via the `e2e-tests` job:
 
 ```bash
-cargo nextest run --profile e2e-smoke -p e2e_test
+python3 scripts/e2e_binary.py build --features e2e-test-hooks
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
 ```
 
 The selection lives in `.config/nextest.toml` under `[profile.e2e-smoke]`

crates/e2e_test/build.rs

@@ -51,7 +51,7 @@ fn main() {
         }
     }
     let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
-    let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
+    let dirty = git(&root, &["status", "--porcelain", "--untracked-files=no"]).is_none_or(|status| !status.is_empty());
     let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
     let mut features = std::env::vars()
         .filter_map(|(key, _)| {

crates/e2e_test/src/chaos.rs

@@ -61,6 +61,7 @@ pub(crate) struct VersionShardCensus {
     pub has_xl_meta: bool,
     pub data_dir: Option<String>,
     pub erasure_index: Option<usize>,
+    pub erasure_distribution: Option<Vec<usize>>,
     pub data_blocks: Option<usize>,
     pub parity_blocks: Option<usize>,
     pub expected_part_numbers: BTreeSet<usize>,
@@ -90,6 +91,7 @@ impl VersionShardCensus {
             && manifest.is_complete()
             && self.data_dir == manifest.data_dir
             && self.erasure_index == manifest.erasure_index
+            && self.erasure_distribution == manifest.erasure_distribution
             && self.data_blocks == manifest.data_blocks
             && self.parity_blocks == manifest.parity_blocks
             && self.expected_part_numbers == manifest.expected_part_numbers
@@ -317,6 +319,7 @@ pub(crate) fn census_object_version_on_disk(
             has_xl_meta: false,
             data_dir: None,
             erasure_index: None,
+            erasure_distribution: None,
             data_blocks: None,
             parity_blocks: None,
             expected_part_numbers: BTreeSet::new(),
@@ -334,6 +337,7 @@ pub(crate) fn census_object_version_on_disk(
     };
     let data_dir = file_info.data_dir.map(|id| id.to_string());
     let erasure_index = Some(file_info.erasure.index);
+    let erasure_distribution = Some(file_info.erasure.distribution.clone());
     let inline_data_fingerprint = file_info.data.as_deref().map(shard_fingerprint).transpose()?;
     let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
     let present_part_fingerprints = match std::fs::read_dir(&part_dir) {
@@ -366,6 +370,7 @@ pub(crate) fn census_object_version_on_disk(
         has_xl_meta: true,
         data_dir,
         erasure_index,
+        erasure_distribution,
         data_blocks: Some(file_info.erasure.data_blocks),
         parity_blocks: Some(file_info.erasure.parity_blocks),
         expected_part_numbers,
@@ -374,6 +379,30 @@ pub(crate) fn census_object_version_on_disk(
     })
 }
 
+pub(crate) fn is_cluster_heal_coordination_unavailable(error: &(dyn std::error::Error + Send + Sync)) -> bool {
+    let message = error.to_string();
+    message.contains("500 Internal Server Error") && message.contains("cluster heal coordination unavailable")
+}
+
+/// Wait for the restarted cluster to admit the first root heal request.
+pub(crate) async fn start_root_heal_when_control_ready(
+    heal_url: &str,
+    heal_body: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> ChaosResult<()> {
+    let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(45);
+    loop {
+        match signed_admin_post(heal_url, Some(heal_body), access_key, secret_key).await {
+            Ok(_) => return Ok(()),
+            Err(error) if is_cluster_heal_coordination_unavailable(error.as_ref()) && tokio::time::Instant::now() < deadline => {
+                tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+            }
+            Err(error) => return Err(error),
+        }
+    }
+}
+
 /// `POST` a signed (SigV4, service `s3`) admin request without relying on the
 /// external `awscurl` binary. Mirrors the admin heal calls used by the heal
 /// regression suite.
@@ -421,6 +450,7 @@ mod tests {
             has_xl_meta: true,
             data_dir: Some("data-dir".to_string()),
             erasure_index: Some(3),
+            erasure_distribution: Some(vec![1, 2, 3, 4]),
             data_blocks: Some(2),
             parity_blocks: Some(2),
             expected_part_numbers: BTreeSet::from([1]),

crates/e2e_test/src/cluster_concurrency_test.rs

@@ -268,6 +268,7 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(
     let rule = CorsRule::builder()
         .allowed_methods("GET")
         .allowed_origins("https://example.com")
+        .allowed_headers("*")
         .build()?;
     let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
 
@@ -288,6 +289,60 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(
     assert_eq!(rules[0].allowed_methods(), ["GET"]);
     assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
 
+    let http = reqwest::Client::builder().no_proxy().build()?;
+    let url = format!("http://{}/{}", cluster.nodes[1].address, BUCKET_METADATA_RELOAD_BUCKET);
+    let without_headers = http
+        .request(reqwest::Method::OPTIONS, &url)
+        .header("Origin", "https://example.com")
+        .header("Access-Control-Request-Method", "GET")
+        .send()
+        .await?;
+    assert!(without_headers.status().is_success());
+    assert!(!without_headers.headers().contains_key("access-control-allow-headers"));
+    assert!(
+        without_headers
+            .headers()
+            .get("vary")
+            .and_then(|value| value.to_str().ok())
+            .is_some_and(|value| value.contains("Access-Control-Request-Headers")),
+        "a cached header-free preflight must not suppress a later requested header grant"
+    );
+    let preflight = http
+        .request(reqwest::Method::OPTIONS, &url)
+        .header("Origin", "https://example.com")
+        .header("Access-Control-Request-Method", "GET")
+        .header("Access-Control-Request-Headers", "X-Another-Header, x-could-be-anything")
+        .send()
+        .await?;
+    assert!(preflight.status().is_success(), "peer preflight should succeed: {preflight:?}");
+    assert_eq!(
+        preflight
+            .headers()
+            .get("access-control-allow-headers")
+            .and_then(|value| value.to_str().ok()),
+        Some("x-another-header,x-could-be-anything"),
+        "a wildcard rule must return only the headers requested by this preflight"
+    );
+    assert!(
+        preflight
+            .headers()
+            .get("vary")
+            .and_then(|value| value.to_str().ok())
+            .is_some_and(|value| value.contains("Access-Control-Request-Headers")),
+        "preflight caches must distinguish the requested header list"
+    );
+    let denied = http
+        .request(reqwest::Method::OPTIONS, &url)
+        .header("Origin", "https://disallowed.example.com")
+        .header("Access-Control-Request-Method", "GET")
+        .header("Access-Control-Request-Headers", "x-another-header")
+        .send()
+        .await?;
+    assert!(
+        !denied.headers().contains_key("access-control-allow-headers"),
+        "a rejected origin must not receive the requested header grant"
+    );
+
     writer
         .delete_bucket_cors()
         .bucket(BUCKET_METADATA_RELOAD_BUCKET)

crates/e2e_test/src/common.rs

@@ -31,7 +31,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use s3s::Body;
 use serde_json;
-use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::io::ErrorKind;
 use std::net::SocketAddr;
@@ -44,7 +43,6 @@ use tokio::net::TcpStream;
 use tokio::time::sleep;
 use tracing::{error, info, warn};
 use uuid::Uuid;
-use walkdir::WalkDir;
 
 // Common constants for all E2E tests
 pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
@@ -188,6 +186,15 @@ fn write_next_test_port(port: u16) -> Result<(), Box<dyn std::error::Error + Sen
     Ok(())
 }
 
+fn no_available_test_port_error(config: TestPortAllocatorConfig, attempts: u16, last_error: Option<&std::io::Error>) -> String {
+    let max_inclusive = config.max_exclusive() - 1;
+    let detail = last_error.map(|err| format!("; last bind error: {err}")).unwrap_or_default();
+    format!(
+        "no available E2E test port found in {}..={} after {} attempts{}",
+        config.min, max_inclusive, attempts, detail
+    )
+}
+
 pub(crate) fn capture_command_logs(
     command: &mut Command,
     log_path: Option<&str>,
@@ -428,59 +435,75 @@ fn resolve_rustfs_binary_path(workspace: &Path, configured_target_dir: Option<&P
     path
 }
 
-/// Resolve the RustFS binary relative to the workspace, optionally requesting build features.
+/// Resolve the server verified by `scripts/e2e_binary.py run` for this test invocation.
+/// Requested features are a required subset of the server's resolved Cargo features.
 pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf {
-    if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") {
-        return PathBuf::from(path);
-    }
-    let requested_features = requested_features.and_then(normalize_rustfs_build_features);
-
     let workspace = workspace_root();
     let configured_target_dir = std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from);
-    let binary_path = resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref());
+    let binary_path = std::env::var_os("CARGO_BIN_EXE_rustfs")
+        .map(PathBuf::from)
+        .unwrap_or_else(|| resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref()));
+    let receipt_path = std::env::var_os("RUSTFS_E2E_BINARY_RECEIPT").map(PathBuf::from);
+    receipt_path
+        .ok_or_else(|| std::io::Error::new(ErrorKind::NotFound, "missing E2E run receipt"))
+        .and_then(|receipt| verify_e2e_binary_receipt(&receipt, &workspace, &binary_path, requested_features))
+        .unwrap_or_else(|error| {
+            panic!(
+                "E2E server prerequisite failed: {error}. Build with `python3 scripts/e2e_binary.py build --features <features>` and run tests with `python3 scripts/e2e_binary.py run --features <features> -- cargo nextest run ...`"
+            )
+        })
+}
 
-    let features_match = binary_features_match(&binary_path, requested_features.as_deref());
-    let source_is_newer = workspace_sources_newer_than_binary(&binary_path);
-    let can_reuse_inside_e2e = running_inside_e2e_test_binary() && requested_features.is_none() && features_match;
-    if binary_path.is_file() && features_match && (!source_is_newer || can_reuse_inside_e2e) {
-        if source_is_newer {
-            warn!(
-                "RustFS binary at {:?} appears older than workspace sources; reusing it inside cargo test to avoid nested builds",
-                binary_path
-            );
-        }
-        info!("Using existing RustFS binary at {:?}", binary_path);
-        return binary_path;
+#[derive(serde::Deserialize)]
+#[serde(deny_unknown_fields)]
+struct E2eBinaryReceipt {
+    schema: u32,
+    workspace: PathBuf,
+    binary: PathBuf,
+    size: u64,
+    modified_ns: u128,
+    features: Vec<String>,
+}
+
+fn verify_e2e_binary_receipt(
+    receipt_path: &Path,
+    workspace: &Path,
+    binary_path: &Path,
+    requested_features: Option<&str>,
+) -> std::io::Result<PathBuf> {
+    let receipt: E2eBinaryReceipt = serde_json::from_slice(&stdfs::read(receipt_path)?)?;
+    let binary = binary_path.canonicalize()?;
+    let metadata = binary.metadata()?;
+    let modified_ns = metadata
+        .modified()?
+        .duration_since(std::time::UNIX_EPOCH)
+        .map_err(std::io::Error::other)?
+        .as_nanos();
+    // The runner hashes source and binary before/after the entire suite. Each
+    // nextest process checks only this invocation's path, features, and file stat.
+    if receipt.schema != 1
+        || receipt.workspace != workspace.canonicalize()?
+        || receipt.binary != binary
+        || !metadata.is_file()
+        || receipt.size != metadata.len()
+        || receipt.modified_ns != modified_ns
+    {
+        return Err(std::io::Error::new(
+            ErrorKind::InvalidData,
+            "E2E server differs from this run's verified binary",
+        ));
     }
-
-    info!("Building RustFS binary to ensure it's up to date...");
-    build_rustfs_binary(requested_features.as_deref(), &binary_path);
-
-    info!("Using RustFS binary at {:?}", binary_path);
-    binary_path
-}
-
-fn workspace_sources_newer_than_binary(binary_path: &PathBuf) -> bool {
-    let Ok(binary_meta) = std::fs::metadata(binary_path) else {
-        return true;
-    };
-    let Ok(binary_modified) = binary_meta.modified() else {
-        return true;
-    };
-
-    let workspace = workspace_root();
-    let watch_roots = [
-        workspace.join("Cargo.toml"),
-        workspace.join("Cargo.lock"),
-        workspace.join("rustfs"),
-        workspace.join("crates"),
-    ];
-
-    watch_roots.iter().any(|path| path_is_newer_than(binary_modified, path))
-}
-
-fn running_inside_e2e_test_binary() -> bool {
-    std::env::var("CARGO_PKG_NAME").is_ok_and(|value| value == "e2e_test")
+    if let Some(requested) = requested_features.and_then(normalize_rustfs_build_features)
+        && requested
+            .split(',')
+            .any(|feature| !receipt.features.iter().any(|actual| actual == feature))
+    {
+        return Err(std::io::Error::new(
+            ErrorKind::InvalidInput,
+            "E2E server is missing a requested build feature",
+        ));
+    }
+    Ok(binary)
 }
 
 pub fn requested_rustfs_build_features() -> Option<String> {
@@ -510,96 +533,6 @@ pub fn rustfs_build_feature_enabled(requested_features: Option<&str>, required_f
         .any(|feature| feature.eq_ignore_ascii_case(RUSTFS_FULL_FEATURE) || feature.eq_ignore_ascii_case(required_feature))
 }
 
-fn rustfs_binary_features_stamp_path(binary_path: &Path) -> PathBuf {
-    binary_path.with_extension("features")
-}
-
-fn binary_features_match(binary_path: &Path, requested_features: Option<&str>) -> bool {
-    let stamp_path = rustfs_binary_features_stamp_path(binary_path);
-    let recorded = stdfs::read_to_string(stamp_path)
-        .ok()
-        .and_then(|value| normalize_rustfs_build_features(&value));
-    let requested = requested_features.and_then(normalize_rustfs_build_features);
-
-    match requested.as_deref() {
-        Some(features) => recorded.as_deref() == Some(features),
-        None => recorded.is_none(),
-    }
-}
-
-fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bool {
-    if path.is_file() {
-        return std::fs::metadata(path)
-            .and_then(|meta| meta.modified())
-            .map(|modified| modified > binary_modified)
-            .unwrap_or(false);
-    }
-
-    if !path.is_dir() {
-        return false;
-    }
-
-    WalkDir::new(path)
-        .into_iter()
-        .filter_entry(|entry| {
-            let name = entry.file_name();
-            name != OsStr::new("target") && name != OsStr::new(".git")
-        })
-        .filter_map(Result::ok)
-        .filter(|entry| entry.file_type().is_file())
-        .any(|entry| {
-            std::fs::metadata(entry.path())
-                .and_then(|meta| meta.modified())
-                .map(|modified| modified > binary_modified)
-                .unwrap_or(false)
-        })
-}
-
-/// Build the RustFS binary using cargo
-fn build_rustfs_binary(requested_features: Option<&str>, binary_path: &Path) {
-    let workspace = workspace_root();
-    info!("Building RustFS binary from workspace: {:?}", workspace);
-
-    let _profile = if cfg!(debug_assertions) {
-        info!("Building in debug mode");
-        "dev"
-    } else {
-        info!("Building in release mode");
-        "release"
-    };
-
-    let mut cmd = Command::new("cargo");
-    cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]);
-
-    if let Some(features) = requested_features {
-        cmd.arg("--features").arg(features);
-        info!("Building with features: {}", features);
-    }
-
-    if !cfg!(debug_assertions) {
-        cmd.arg("--release");
-    }
-
-    info!(
-        "Executing: cargo build --bin rustfs {}",
-        if cfg!(debug_assertions) { "" } else { "--release" }
-    );
-
-    let output = cmd.output().expect("Failed to execute cargo build command");
-
-    if !output.status.success() {
-        let stderr = String::from_utf8_lossy(&output.stderr);
-        panic!("Failed to build RustFS binary. Error: {stderr}");
-    }
-
-    let stamp_path = rustfs_binary_features_stamp_path(binary_path);
-    if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) {
-        warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err);
-    }
-
-    info!("✅ RustFS binary built successfully");
-}
-
 fn awscurl_binary_path() -> PathBuf {
     std::env::var_os("AWSCURL_PATH")
         .map(PathBuf::from)
@@ -694,19 +627,23 @@ impl RustFSTestEnvironment {
         let _guard = PortAllocatorGuard::acquire().await?;
         let config = test_port_allocator_config()?;
         let mut next_port = read_next_test_port(config);
+        let mut last_error = None;
 
         for _ in 0..config.range {
             let port = next_port;
             next_port = advance_test_port(next_port, config);
             write_next_test_port(next_port)?;
 
-            if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
-                drop(listener);
-                return Ok(port);
+            match TcpListener::bind(("127.0.0.1", port)) {
+                Ok(listener) => {
+                    drop(listener);
+                    return Ok(port);
+                }
+                Err(err) => last_error = Some(err),
             }
         }
 
-        Err("no available E2E test port found".into())
+        Err(no_available_test_port_error(config, config.range, last_error.as_ref()).into())
     }
 
     /// Kill any existing RustFS processes
@@ -2201,6 +2138,19 @@ mod tests {
         assert!(parse_test_port_allocator_config(Some("not-a-port"), Some("128")).is_err());
     }
 
+    #[test]
+    fn e2e_port_allocator_reports_attempt_window_and_last_bind_error() {
+        let config = TestPortAllocatorConfig { min: 41000, range: 3 };
+        let error = std::io::Error::from(ErrorKind::PermissionDenied);
+
+        let message = no_available_test_port_error(config, config.range, Some(&error));
+
+        assert!(message.contains("41000..=41002"));
+        assert!(message.contains("after 3 attempts"));
+        assert!(message.contains("last bind error"));
+        assert!(message.contains("permission denied"));
+    }
+
     #[test]
     fn resolves_rustfs_binary_in_configured_cargo_target_directory() {
         let workspace = Path::new("workspace");
@@ -2229,16 +2179,66 @@ mod tests {
     }
 
     #[test]
-    fn binary_feature_stamp_matching_uses_normalized_features() {
-        let binary_path = std::env::temp_dir().join(format!("rustfs-feature-stamp-test-{}", Uuid::new_v4()));
-        let stamp_path = rustfs_binary_features_stamp_path(&binary_path);
+    fn explicit_binary_without_run_receipt_is_rejected() {
+        const CHILD_ENV: &str = "RUSTFS_E2E_RECEIPT_TEST_CHILD";
+        if std::env::var_os(CHILD_ENV).is_some() {
+            rustfs_binary_path_with_features(None);
+            return;
+        }
+        let executable = std::env::current_exe().expect("locate isolated test process");
+        let output = Command::new(&executable)
+            .args([
+                "--exact",
+                "common::tests::explicit_binary_without_run_receipt_is_rejected",
+                "--nocapture",
+            ])
+            .env(CHILD_ENV, "1")
+            .env("CARGO_BIN_EXE_rustfs", &executable)
+            .env_remove("RUSTFS_E2E_BINARY_RECEIPT")
+            .output()
+            .expect("run the missing-receipt scenario with isolated environment variables");
+        assert!(!output.status.success(), "an explicit binary must not bypass run verification");
+        assert!(String::from_utf8_lossy(&output.stderr).contains("missing E2E run receipt"));
+    }
 
-        stdfs::write(&stamp_path, " SFTP, ftps ").expect("write feature stamp");
-        assert!(binary_features_match(&binary_path, Some("sftp,ftps")));
-        assert!(binary_features_match(&binary_path, Some(" SFTP, FTPS ")));
-        assert!(!binary_features_match(&binary_path, Some("sftp")));
-
-        stdfs::remove_file(stamp_path).ok();
+    #[test]
+    fn e2e_run_receipt_rejects_replaced_binary_and_missing_features() {
+        let directory = std::env::temp_dir().join(format!("rustfs-e2e-receipt-test-{}", Uuid::new_v4()));
+        stdfs::create_dir(&directory).expect("create receipt fixture");
+        let binary = directory.join("rustfs");
+        let receipt = directory.join("receipt.json");
+        stdfs::write(&binary, "server").expect("write fixture binary");
+        let metadata = binary.metadata().expect("stat fixture binary");
+        let record = serde_json::json!({
+            "schema": 1,
+            "workspace": directory.canonicalize().expect("canonical workspace"),
+            "binary": binary.canonicalize().expect("canonical binary"),
+            "size": metadata.len(),
+            "modified_ns": metadata.modified().expect("modified time").duration_since(std::time::UNIX_EPOCH).expect("positive timestamp").as_nanos(),
+            "features": ["default", "full", "ftps", "webdav", "sftp"]
+        });
+        stdfs::write(&receipt, serde_json::to_vec(&record).expect("serialize receipt")).expect("write receipt");
+        verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("sftp,webdav")).expect("resolved feature subset");
+        verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("full")).expect("full was actually requested");
+        assert_eq!(
+            verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("rio-v2"))
+                .expect_err("full does not enable rio-v2")
+                .kind(),
+            ErrorKind::InvalidInput
+        );
+        let other = directory.join("old-server");
+        stdfs::write(&other, "server").expect("write alternate binary");
+        assert!(verify_e2e_binary_receipt(&receipt, &directory, &other, None).is_err());
+        stdfs::write(&binary, "different server").expect("replace fixture binary");
+        assert!(verify_e2e_binary_receipt(&receipt, &directory, &binary, None).is_err());
+        stdfs::remove_file(&receipt).expect("remove expired receipt");
+        assert_eq!(
+            verify_e2e_binary_receipt(&receipt, &directory, &binary, None)
+                .expect_err("expired receipt")
+                .kind(),
+            ErrorKind::NotFound
+        );
+        stdfs::remove_dir_all(directory).expect("remove receipt fixture");
     }
 
     /// Build a cluster environment struct in-memory (no ports, no processes) so

crates/e2e_test/src/compression_test.rs

@@ -152,9 +152,29 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
     let content_length = head_response.content_length().unwrap_or(0);
     assert_eq!(content_length as usize, original_size, "Content-Length should be original size");
 
+    // A 5 KiB compressed object sits inside the inline budget, so its shard is
+    // embedded in xl.meta rather than written as a part file; the stored size
+    // recorded in the metadata is the compressed length.
     let part_files = find_part_files(&env.temp_dir, COMPRESSION_TEST_BUCKET, object_key)?;
-    assert!(!part_files.is_empty(), "expected on-disk part files for the compressed object");
-    let total_physical_size = part_files_total_size(&part_files)?;
+    assert!(
+        part_files.is_empty(),
+        "small compressed object must be stored inline, found part files {part_files:?}"
+    );
+    let xl_meta_path = PathBuf::from(&env.temp_dir)
+        .join(COMPRESSION_TEST_BUCKET)
+        .join(object_key)
+        .join("xl.meta");
+    let file_info = rustfs_filemeta::FileMeta::load(&fs::read(&xl_meta_path)?)?.into_fileinfo(
+        COMPRESSION_TEST_BUCKET,
+        object_key,
+        "",
+        true,
+        false,
+        true,
+    )?;
+    assert!(file_info.inline_data(), "xl.meta must carry the inline marker for the compressed object");
+    assert!(file_info.is_compressed(), "xl.meta must carry the compression marker");
+    let total_physical_size = u64::try_from(file_info.size)?;
 
     assert!(
         total_physical_size < original_size as u64,

crates/e2e_test/src/copy_object_metadata_test.rs

@@ -18,10 +18,14 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::error::ProvideErrorMetadata;
+    use aws_sdk_s3::presigning::PresigningConfig;
     use aws_sdk_s3::primitives::{ByteStream, DateTime, DateTimeFormat};
     use aws_sdk_s3::types::{
         BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, MetadataDirective, StorageClass, VersioningConfiguration,
     };
+    use flate2::{Compression, read::GzDecoder, write::GzEncoder};
+    use std::io::{Read, Write};
+    use std::time::Duration;
     use tracing::info;
 
     #[tokio::test]
@@ -108,6 +112,24 @@ mod tests {
             copied_head.metadata().and_then(|metadata| metadata.get("stale")),
             Some(&"must-be-removed".to_string())
         );
+        let copied_get = client
+            .get_object()
+            .bucket(bucket)
+            .key(copied_key)
+            .send()
+            .await
+            .expect("GET failed after default copy");
+        assert_eq!(copied_get.content_language(), Some("en-US"));
+        assert_eq!(
+            copied_get
+                .body
+                .collect()
+                .await
+                .expect("Failed to collect copied body")
+                .into_bytes()
+                .as_ref(),
+            content,
+        );
 
         client
             .copy_object()
@@ -241,6 +263,7 @@ mod tests {
             .send()
             .await
             .expect("GET failed after self-copy");
+        assert_eq!(get_resp.content_language(), Some("fr-FR"));
         let body = get_resp
             .body
             .collect()
@@ -286,6 +309,7 @@ mod tests {
             .send()
             .await
             .expect("GET failed after empty metadata replacement");
+        assert_eq!(empty_get_resp.content_language(), None);
         let empty_body = empty_get_resp
             .body
             .collect()
@@ -522,9 +546,143 @@ mod tests {
             multipart_body
         );
 
+        assert_versioned_content_language_round_trip(&client, bucket).await;
         env.stop_server();
     }
 
+    async fn assert_versioned_content_language_round_trip(client: &aws_sdk_s3::Client, bucket: &str) {
+        let http = reqwest::Client::builder()
+            .no_proxy()
+            .no_gzip()
+            .build()
+            .expect("Failed to build HTTP client without gzip decoding");
+        let original = b"Content-Language must follow the requested object version.";
+        for encoding in [None, Some("gzip")] {
+            let key = format!("language-{}.bin", encoding.unwrap_or("plain"));
+            let payload = if encoding.is_some() {
+                let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+                encoder.write_all(original).expect("Failed to encode gzip fixture");
+                encoder.finish().expect("Failed to finish gzip fixture")
+            } else {
+                original.to_vec()
+            };
+            let mut versions = Vec::new();
+            for language in [Some("zh-CN"), None, Some("en-US")] {
+                let put = client
+                    .put_object()
+                    .bucket(bucket)
+                    .key(&key)
+                    .content_type("application/octet-stream")
+                    .set_content_encoding(encoding.map(str::to_string))
+                    .set_content_language(language.map(str::to_string))
+                    .metadata("content-language", "user-language")
+                    .body(ByteStream::from(payload.clone()))
+                    .send()
+                    .await
+                    .expect("Failed to write versioned language fixture");
+                versions.push((put.version_id().expect("PUT must return a version ID").to_string(), language));
+            }
+
+            let current_version = &versions.last().expect("Fixture has three versions").0;
+            let reads = std::iter::once((None, Some("en-US")))
+                .chain(versions.iter().map(|(version, language)| (Some(version.as_str()), *language)));
+            for (version, language) in reads {
+                let expected_version = version.unwrap_or(current_version);
+                let head = client
+                    .head_object()
+                    .bucket(bucket)
+                    .key(&key)
+                    .set_version_id(version.map(str::to_string))
+                    .send()
+                    .await
+                    .expect("HEAD failed for language fixture");
+                assert_eq!(head.content_language(), language);
+                assert_eq!(head.version_id(), Some(expected_version));
+
+                for range in [None, Some("bytes=1-7")] {
+                    let expected_body = if range.is_some() { &payload[1..8] } else { payload.as_slice() };
+                    let content_range = range.map(|_| format!("bytes 1-7/{}", payload.len()));
+                    let request = client
+                        .get_object()
+                        .bucket(bucket)
+                        .key(&key)
+                        .set_version_id(version.map(str::to_string))
+                        .set_range(range.map(str::to_string));
+                    let get = request.clone().send().await.expect("GET failed for language fixture");
+                    assert_eq!(get.content_language(), language);
+                    assert_eq!(get.content_type(), Some("application/octet-stream"));
+                    assert_eq!(get.content_encoding(), encoding);
+                    assert_eq!(
+                        get.content_length(),
+                        Some(i64::try_from(expected_body.len()).expect("Fixture length fits i64"))
+                    );
+                    assert_eq!(get.content_range(), content_range.as_deref());
+                    assert_eq!(get.version_id(), Some(expected_version));
+                    assert_eq!(get.e_tag(), head.e_tag());
+                    assert_eq!(
+                        get.metadata()
+                            .and_then(|metadata| metadata.get("content-language"))
+                            .map(String::as_str),
+                        Some("user-language")
+                    );
+                    assert_eq!(
+                        get.body
+                            .collect()
+                            .await
+                            .expect("Failed to collect GET body")
+                            .into_bytes()
+                            .as_ref(),
+                        expected_body
+                    );
+
+                    for override_language in [None, Some("fr-FR")] {
+                        let presigned = request
+                            .clone()
+                            .set_response_content_language(override_language.map(str::to_string))
+                            .presigned(PresigningConfig::expires_in(Duration::from_secs(300)).expect("Valid signature lifetime"))
+                            .await
+                            .expect("Failed to presign language GET");
+                        let mut raw_request = http.get(presigned.uri());
+                        for (name, value) in presigned.headers() {
+                            raw_request = raw_request.header(name, value);
+                        }
+                        let response = raw_request.send().await.expect("Presigned GET failed");
+                        let status = if range.is_some() {
+                            reqwest::StatusCode::PARTIAL_CONTENT
+                        } else {
+                            reqwest::StatusCode::OK
+                        };
+                        assert_eq!(response.status(), status);
+                        let header = |name: &str| {
+                            response
+                                .headers()
+                                .get(name)
+                                .map(|value| value.to_str().expect("ASCII response header"))
+                        };
+                        assert_eq!(header("content-language"), override_language.or(language));
+                        assert_eq!(header("x-amz-meta-content-language"), Some("user-language"));
+                        assert_eq!(header("content-encoding"), encoding);
+                        assert_eq!(header("content-range"), content_range.as_deref());
+                        assert_eq!(header("x-amz-version-id"), Some(expected_version));
+                        assert_eq!(
+                            response.content_length(),
+                            Some(u64::try_from(expected_body.len()).expect("Fixture length fits u64"))
+                        );
+                        let body = response.bytes().await.expect("Failed to read wire body");
+                        assert_eq!(body.as_ref(), expected_body);
+                        if encoding.is_some() && range.is_none() {
+                            let mut decoded = Vec::new();
+                            GzDecoder::new(body.as_ref())
+                                .read_to_end(&mut decoded)
+                                .expect("GET body must contain actual gzip bytes");
+                            assert_eq!(decoded, original);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     #[tokio::test]
     async fn invalid_replacement_metadata_does_not_mutate_destination() {
         init_logging();

crates/e2e_test/src/delete_authorization_test.rs

@@ -0,0 +1,656 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Standard S3 deletion permissions and the explicit recursive-delete extension.
+
+use crate::common::{
+    AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user,
+    init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
+use aws_sdk_s3::operation::delete_object::{DeleteObjectError, DeleteObjectOutput};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
+use futures::{StreamExt, TryStreamExt, stream};
+use serde_json::{Value, json};
+use std::collections::BTreeSet;
+use std::error::Error;
+use uuid::Uuid;
+
+type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
+type VersionSnapshot = BTreeSet<(String, String, bool)>;
+
+async fn set_policy(env: &RustFSTestEnvironment, name: &str, policy: &Value) -> TestResult {
+    admin_add_canned_policy_via(
+        AdminTransport::Signed,
+        &env.url,
+        &env.access_key,
+        &env.secret_key,
+        name,
+        &policy.to_string(),
+    )
+    .await
+}
+
+async fn policy_user(env: &RustFSTestEnvironment, policy_name: &str, policy: Option<Value>) -> TestResult<Client> {
+    let username = Uuid::new_v4().simple().to_string();
+    let secret = Uuid::new_v4().simple().to_string();
+    admin_create_user(env, &username, &secret).await?;
+    if let Some(policy) = policy {
+        set_policy(env, policy_name, &policy).await?;
+    }
+    admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, policy_name, &username)
+        .await?;
+    Ok(env.create_s3_client_with_credentials(&username, &secret))
+}
+
+async fn versioning(client: &Client, bucket: &str, status: BucketVersioningStatus) -> TestResult {
+    client
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(VersioningConfiguration::builder().status(status).build())
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn put(client: &Client, bucket: &str, key: &str) -> TestResult<String> {
+    let result = client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"delete authorization fixture"))
+        .send()
+        .await?;
+    Ok(result.version_id().unwrap_or("null").to_string())
+}
+
+async fn versions(client: &Client, bucket: &str, prefix: &str) -> TestResult<VersionSnapshot> {
+    let mut result = BTreeSet::new();
+    let mut markers = (None, None);
+    loop {
+        let page = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(prefix)
+            .set_key_marker(markers.0.clone())
+            .set_version_id_marker(markers.1.clone())
+            .send()
+            .await?;
+        for version in page.versions() {
+            result.insert((
+                version.key().ok_or("listed version missing key")?.to_string(),
+                version.version_id().ok_or("listed version missing ID")?.to_string(),
+                false,
+            ));
+        }
+        for marker in page.delete_markers() {
+            result.insert((
+                marker.key().ok_or("listed delete marker missing key")?.to_string(),
+                marker.version_id().ok_or("listed delete marker missing ID")?.to_string(),
+                true,
+            ));
+        }
+        if page.is_truncated() != Some(true) {
+            return Ok(result);
+        }
+        let next = (
+            Some(
+                page.next_key_marker()
+                    .ok_or("truncated versions page missing next key marker")?
+                    .to_string(),
+            ),
+            page.next_version_id_marker().map(str::to_string),
+        );
+        assert_ne!(markers, next, "ListObjectVersions pagination must advance");
+        markers = next;
+    }
+}
+
+async fn force_delete(
+    client: &Client,
+    bucket: &str,
+    prefix: &str,
+) -> Result<DeleteObjectOutput, Box<SdkError<DeleteObjectError>>> {
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(prefix)
+        .customize()
+        .mutate_request(|request| {
+            request.headers_mut().insert("x-rustfs-force-delete", "true");
+        })
+        .send()
+        .await
+        .map_err(Box::new)
+}
+
+async fn replica_force_delete(
+    client: &Client,
+    bucket: &str,
+    prefix: &str,
+) -> Result<DeleteObjectOutput, Box<SdkError<DeleteObjectError>>> {
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(prefix)
+        .customize()
+        .mutate_request(|request| {
+            request.headers_mut().insert("x-rustfs-force-delete", "true");
+            request.headers_mut().insert("x-amz-replication-status", "REPLICA");
+        })
+        .send()
+        .await
+        .map_err(Box::new)
+}
+
+fn assert_denied<T, E>(result: Result<T, SdkError<E>>)
+where
+    T: std::fmt::Debug,
+    E: ProvideErrorMetadata + std::fmt::Debug,
+{
+    let error = result.expect_err("request must be denied by its S3 permission");
+    assert_eq!(
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("AccessDenied"),
+        "expected an S3 authorization denial, got {error:?}"
+    );
+}
+
+fn assert_boxed_denied<T, E>(result: Result<T, Box<SdkError<E>>>)
+where
+    T: std::fmt::Debug,
+    E: ProvideErrorMetadata + std::fmt::Debug,
+{
+    assert_denied(result.map_err(|error| *error));
+}
+
+#[tokio::test]
+async fn sdk_version_deletion_requires_only_delete_object_version() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "delete-version-permissions";
+    root.create_bucket().bucket(bucket).send().await?;
+    put(&root, bucket, "single-null.txt").await?;
+    put(&root, bucket, "batch-null.txt").await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    let old = put(&root, bucket, "single.txt").await?;
+    let current = put(&root, bucket, "single.txt").await?;
+    let batch_version = put(&root, bucket, "batch.txt").await?;
+    let ordinary_version = put(&root, bucket, "ordinary.txt").await?;
+    let user = policy_user(
+        &env,
+        "version-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")}
+        ]})),
+    )
+    .await?;
+
+    user.delete_object()
+        .bucket(bucket)
+        .key("single.txt")
+        .version_id(&old)
+        .send()
+        .await?;
+    user.delete_object()
+        .bucket(bucket)
+        .key("single-null.txt")
+        .version_id("null")
+        .send()
+        .await?;
+    assert_denied(user.delete_object().bucket(bucket).key("ordinary.txt").send().await);
+
+    let batch = user
+        .delete_objects()
+        .bucket(bucket)
+        .delete(
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key("batch.txt")
+                        .version_id(&batch_version)
+                        .build()?,
+                )
+                .objects(ObjectIdentifier::builder().key("batch-null.txt").version_id("null").build()?)
+                .objects(ObjectIdentifier::builder().key("ordinary.txt").build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    assert_eq!(batch.deleted().len(), 2, "both explicit version items must succeed");
+    assert_eq!(batch.errors().len(), 1, "only the unversioned item must be denied");
+    assert_eq!(batch.errors()[0].key(), Some("ordinary.txt"));
+    assert_eq!(batch.errors()[0].code(), Some("AccessDenied"));
+    assert_eq!(
+        versions(&root, bucket, "").await?,
+        BTreeSet::from([
+            ("single.txt".into(), current, false),
+            ("ordinary.txt".into(), ordinary_version, false)
+        ]),
+        "version-only deletion must preserve the current single-object version and denied object"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn sdk_list_bucket_and_list_bucket_versions_permissions_are_independent() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "list-version-permissions";
+    root.create_bucket().bucket(bucket).send().await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    put(&root, bucket, "visible.txt").await?;
+    for (action, name) in [
+        ("s3:ListBucket", "object-lister"),
+        ("s3:ListBucketVersions", "version-lister"),
+    ] {
+        let user = policy_user(
+            &env,
+            name,
+            Some(json!({"Version":"2012-10-17","Statement":[
+                {"Effect":"Allow","Action":action,"Resource":format!("arn:aws:s3:::{bucket}")}
+            ]})),
+        )
+        .await?;
+        if action == "s3:ListBucket" {
+            assert_eq!(user.list_objects_v2().bucket(bucket).send().await?.contents().len(), 1);
+            assert_denied(user.list_object_versions().bucket(bucket).send().await);
+        } else {
+            assert_eq!(user.list_object_versions().bucket(bucket).send().await?.versions().len(), 1);
+            assert_denied(user.list_objects_v2().bucket(bucket).send().await);
+        }
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn console_admin_force_delete_removes_prefix_versions_and_delete_markers() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-console-admin";
+    root.create_bucket().bucket(bucket).send().await?;
+    put(&root, bucket, "folder/null.txt").await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    for key in ["folder/a.txt", "folder/deep/b.txt", "single.txt"] {
+        put(&root, bucket, key).await?;
+        put(&root, bucket, key).await?;
+        root.delete_object().bucket(bucket).key(key).send().await?;
+    }
+    put(&root, bucket, "keep.txt").await?;
+    put(&root, bucket, "folder-sibling/keep.txt").await?;
+    let keep = versions(&root, bucket, "keep.txt").await?;
+    let sibling = versions(&root, bucket, "folder-sibling/").await?;
+    let user = policy_user(&env, "consoleAdmin", None).await?;
+
+    force_delete(&user, bucket, "folder/").await?;
+    assert!(
+        versions(&root, bucket, "folder/").await?.is_empty(),
+        "force prefix deletion must remove null versions and markers"
+    );
+    assert_eq!(versions(&root, bucket, "folder-sibling/").await?, sibling);
+    assert_eq!(
+        versions(&root, bucket, "single.txt").await?.len(),
+        3,
+        "the separate key must survive folder deletion"
+    );
+    force_delete(&user, bucket, "single.txt").await?;
+    assert!(
+        versions(&root, bucket, "single.txt").await?.is_empty(),
+        "explicit force deletion must remove every version of the selected key"
+    );
+    assert_eq!(versions(&root, bucket, "keep.txt").await?, keep);
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_delete_authorizes_only_its_path_scope_without_list_permissions() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-delete-only";
+    root.create_bucket().bucket(bucket).send().await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    put(&root, bucket, "selected.txt").await?;
+    put(&root, bucket, "selected.txt/child.txt").await?;
+    put(&root, bucket, "selected.txt-sibling").await?;
+    let sibling = versions(&root, bucket, "selected.txt-sibling").await?;
+    let user = policy_user(
+        &env,
+        "delete-only",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":[
+                format!("arn:aws:s3:::{bucket}/selected.txt"), format!("arn:aws:s3:::{bucket}/selected.txt/*")
+            ]},
+            {"Effect":"Deny","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/selected.txt-sibling")}
+        ]})),
+    )
+    .await?;
+    assert_denied(user.list_objects_v2().bucket(bucket).send().await);
+    assert_denied(user.list_object_versions().bucket(bucket).send().await);
+    force_delete(&user, bucket, "selected.txt").await?;
+    assert_eq!(
+        versions(&root, bucket, "selected.txt").await?,
+        sibling,
+        "force deletion must remove the selected path and descendants without authorizing or deleting its similarly prefixed sibling"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_directory_delete_cannot_remove_an_unauthorized_colliding_parent() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-directory-collision";
+    root.create_bucket().bucket(bucket).send().await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    let protected_parent_version = put(&root, bucket, "collision.txt").await?;
+    for key in ["collision.txt/child", "collision.txt-sibling"] {
+        put(&root, bucket, key).await?;
+    }
+    put(&root, bucket, "collision.txt").await?;
+    root.delete_object().bucket(bucket).key("collision.txt").send().await?;
+    let user = policy_user(
+        &env,
+        "parent-denier",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/collision.txt"),
+             "Condition":{"StringEquals":{"s3:VersionId":protected_parent_version}}}
+        ]})),
+    )
+    .await?;
+    let mut expected = versions(&root, bucket, "").await?;
+    expected.retain(|(key, _, _)| key != "collision.txt/child");
+    force_delete(&user, bucket, "collision.txt/").await?;
+    assert_eq!(
+        versions(&root, bucket, "").await?,
+        expected,
+        "folder deletion must preserve the denied parent's historical versions and delete marker, plus its sibling"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_unversioned_directory_requires_only_delete_object() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-unversioned-permissions";
+    root.create_bucket().bucket(bucket).send().await?;
+    for key in ["folder/", "folder/child.txt", "outside.txt"] {
+        put(&root, bucket, key).await?;
+    }
+    let outside = versions(&root, bucket, "outside.txt").await?;
+    let user = policy_user(
+        &env,
+        "unversioned-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/*")}
+        ]})),
+    )
+    .await?;
+    force_delete(&user, bucket, "folder/").await?;
+    assert_eq!(
+        versions(&root, bucket, "").await?,
+        outside,
+        "unversioned force deletion, including a synthetic nil directory marker, must use DeleteObject permission"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_delete_denied_child_preserves_every_object_despite_bucket_allow() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-child-denial";
+    root.create_bucket().bucket(bucket).send().await?;
+    for key in ["folder/a-allowed.txt", "folder/z-denied.txt"] {
+        put(&root, bucket, key).await?;
+    }
+    let user = policy_user(
+        &env,
+        "child-denier",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/folder/z-denied.txt")}
+        ]})),
+    )
+    .await?;
+    root.put_bucket_policy()
+        .bucket(bucket)
+        .policy(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Principal":"*","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/*")}
+        ]}).to_string())
+        .send().await?;
+    let before = versions(&root, bucket, "folder/").await?;
+    assert_boxed_denied(force_delete(&user, bucket, "folder/").await);
+    assert_eq!(
+        versions(&root, bucket, "folder/").await?,
+        before,
+        "a denied descendant must prevent every mutation in the force scope"
+    );
+    assert_boxed_denied(replica_force_delete(&user, bucket, "folder/").await);
+    assert_eq!(
+        versions(&root, bucket, "folder/").await?,
+        before,
+        "the REPLICA header must not bypass a descendant's ReplicateDelete denial"
+    );
+
+    root.delete_bucket_policy().bucket(bucket).send().await?;
+    let replica_user = policy_user(
+        &env,
+        "replica-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Allow","Action":"s3:ReplicateDelete","Resource":format!("arn:aws:s3:::{bucket}/*")}
+        ]})),
+    )
+    .await?;
+    replica_force_delete(&replica_user, bucket, "folder/").await?;
+    assert!(
+        versions(&root, bucket, "folder/").await?.is_empty(),
+        "an authorized replica force request must check ReplicateDelete for its descendants"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_delete_denied_historical_version_preserves_versions_and_markers() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-version-denial";
+    root.create_bucket().bucket(bucket).send().await?;
+    put(&root, bucket, "folder/null.txt").await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    let protected_version = put(&root, bucket, "folder/versioned.txt").await?;
+    put(&root, bucket, "folder/versioned.txt").await?;
+    let marker = root.delete_object().bucket(bucket).key("folder/versioned.txt").send().await?;
+    let marker_version = marker
+        .version_id()
+        .ok_or("versioned delete must return a marker version ID")?;
+    put(&root, bucket, "folder/a-allowed.txt").await?;
+    let policy = |version: &str| {
+        json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/folder/*"),
+             "Condition":{"StringEquals":{"s3:VersionId":version}}}
+        ]})
+    };
+    let user = policy_user(&env, "version-denier", Some(policy(&protected_version))).await?;
+    let before = versions(&root, bucket, "folder/").await?;
+    for version in [protected_version.as_str(), "null", marker_version] {
+        set_policy(&env, "version-denier", &policy(version)).await?;
+        assert_boxed_denied(force_delete(&user, bucket, "folder/").await);
+        assert_eq!(
+            versions(&root, bucket, "folder/").await?,
+            before,
+            "denial of a historical, null, or delete-marker version must prevent recursive deletion"
+        );
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn sdk_ordinary_deletion_preserves_versions_and_directory_children() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let user = policy_user(
+        &env,
+        "ordinary-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::*/*"},
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":"arn:aws:s3:::*/*"}
+        ]})),
+    )
+    .await?;
+    for state in ["unversioned", "enabled", "suspended"] {
+        let bucket = format!("ordinary-directory-{state}");
+        root.create_bucket().bucket(&bucket).send().await?;
+        if state != "unversioned" {
+            versioning(&root, &bucket, BucketVersioningStatus::Enabled).await?;
+        }
+        let historical = put(&root, &bucket, "object.txt").await?;
+        if state == "suspended" {
+            versioning(&root, &bucket, BucketVersioningStatus::Suspended).await?;
+            put(&root, &bucket, "object.txt").await?;
+        }
+        put(&root, &bucket, "folder/").await?;
+        put(&root, &bucket, "folder/child.txt").await?;
+        let child_before = versions(&root, &bucket, "folder/child.txt").await?;
+        user.delete_object().bucket(&bucket).key("folder/").send().await?;
+        assert_eq!(
+            versions(&root, &bucket, "folder/").await?,
+            child_before,
+            "ordinary {state} directory-key deletion must remove only its synthetic marker and preserve children"
+        );
+        let deleted = user.delete_object().bucket(&bucket).key("object.txt").send().await?;
+        let object_versions = versions(&root, &bucket, "object.txt").await?;
+        if state == "unversioned" {
+            assert!(object_versions.is_empty());
+        } else {
+            assert_eq!(deleted.delete_marker(), Some(true));
+            assert_eq!(
+                object_versions.len(),
+                2,
+                "ordinary {state} deletion must retain its historical data version"
+            );
+            assert!(object_versions.contains(&("object.txt".into(), historical, false)));
+            if state == "suspended" {
+                assert!(object_versions.contains(&("object.txt".into(), "null".into(), true)));
+            }
+        }
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn sdk_delete_objects_force_header_keeps_explicit_item_scope() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "batch-force-explicit-scope";
+    root.create_bucket().bucket(bucket).send().await?;
+    put(&root, bucket, "folder/").await?;
+    put(&root, bucket, "folder/child.txt").await?;
+    let child = versions(&root, bucket, "folder/child.txt").await?;
+    let user = policy_user(&env, "consoleAdmin", None).await?;
+    let result = user
+        .delete_objects()
+        .bucket(bucket)
+        .delete(
+            Delete::builder()
+                .objects(ObjectIdentifier::builder().key("folder/").build()?)
+                .build()?,
+        )
+        .customize()
+        .mutate_request(|request| {
+            request.headers_mut().insert("x-rustfs-force-delete", "true");
+        })
+        .send()
+        .await?;
+    assert!(result.errors().is_empty());
+    assert_eq!(result.deleted().len(), 1);
+    assert_eq!(
+        versions(&root, bucket, "folder/").await?,
+        child,
+        "batch deletion must remove only the explicit directory marker even with the force header"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_delete_checks_every_version_page_before_mutation() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-delete-pagination";
+    root.create_bucket().bucket(bucket).send().await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    stream::iter(0..1000)
+        .map(|index| {
+            let root = &root;
+            async move { put(root, bucket, &format!("folder/{index:04}.txt")).await.map(|_| ()) }
+        })
+        .buffer_unordered(16)
+        .try_collect::<Vec<_>>()
+        .await?;
+    put(&root, bucket, "folder/z-denied.txt").await?;
+    let allow = json!({"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")});
+    let user = policy_user(
+        &env,
+        "paged-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[allow.clone(),
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/folder/z-denied.txt")}
+        ]})),
+    )
+    .await?;
+    let before = versions(&root, bucket, "folder/").await?;
+    assert_eq!(before.len(), 1001, "the denied key must be beyond one default versions page");
+    assert_boxed_denied(force_delete(&user, bucket, "folder/").await);
+    assert_eq!(
+        versions(&root, bucket, "folder/").await?,
+        before,
+        "a denial on the second page must preserve the first page too"
+    );
+
+    set_policy(&env, "paged-deleter", &json!({"Version":"2012-10-17","Statement":[allow]})).await?;
+    force_delete(&user, bucket, "folder/").await?;
+    assert!(
+        versions(&root, bucket, "folder/").await?.is_empty(),
+        "authorized recursive deletion must cover all pages"
+    );
+    Ok(())
+}

crates/e2e_test/src/distributed/harness.rs

@@ -59,6 +59,9 @@ const POOL_META_V3_ENV: [(&str, &str); 2] = [
 
 #[derive(Clone, Copy, Debug)]
 pub(crate) enum DistLayout {
+    /// 3 nodes × 4 drives, one erasure pool. With `EC:4` this is the
+    /// release-evidence EC8+4 geometry.
+    ThreeByFourEc84,
     /// 4 nodes × 4 drives, one erasure pool spanning every endpoint.
     FourByFour,
     /// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout).
@@ -94,6 +97,7 @@ impl DistCluster {
 
     pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
         let topology = match layout {
+            DistLayout::ThreeByFourEc84 => ClusterTopology::single_pool_multidrive(3, DRIVES_PER_NODE),
             DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE),
             DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT),
             DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]),
@@ -101,7 +105,7 @@ impl DistCluster {
         let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?;
         let pool_storage_roots = match layout {
             DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?),
-            DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
+            DistLayout::ThreeByFourEc84 | DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
         };
         let mut owned_pool_dirs = Vec::new();
         if let Some(roots) = pool_storage_roots.as_deref() {
@@ -958,27 +962,6 @@ pub(crate) async fn wait_for_rebalance_active(
     }
 }
 
-pub(crate) async fn wait_for_rebalance_running_with_progress(
-    cluster: &RustFSTestClusterEnvironment,
-    expected_id: &str,
-    timeout: Duration,
-) -> TestResult {
-    let deadline = Instant::now() + timeout;
-    loop {
-        let status = rebalance_status_json(cluster).await?;
-        if rebalance_running_with_progress(&status, expected_id)? {
-            return Ok(());
-        }
-        if Instant::now() >= deadline {
-            return Err(format!(
-                "rebalance did not become active with non-zero progress within {timeout:?}; last status: {status}"
-            )
-            .into());
-        }
-        sleep(Duration::from_millis(100)).await;
-    }
-}
-
 pub(crate) async fn wait_for_rebalance_complete(
     cluster: &RustFSTestClusterEnvironment,
     expected_id: &str,

crates/e2e_test/src/distributed/heal_test.rs

@@ -0,0 +1,403 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_inventory, get_object_bytes, payload_for, put_object, sha256_hex, unique_bucket,
+    wait_until,
+};
+use crate::chaos::{VersionShardCensus, census_object_version_on_disk, start_root_heal_when_control_ready};
+use crate::common::init_logging;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::primitives::ByteStream;
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use std::collections::{BTreeMap, HashSet};
+use std::io::{Read, Write};
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+const EC84_NODE_COUNT: usize = 3;
+const EC84_DRIVES_PER_NODE: usize = 4;
+const EC84_DATA_BLOCKS: usize = 8;
+const EC84_PARITY_BLOCKS: usize = 4;
+const EC84_ERASURE_SET_DRIVE_COUNT: usize = EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS;
+const EC84_TARGET_DRIVE_RESTART_CASE: &str = "ec84-target-drive-restart";
+const EC84_TARGET_DRIVE_RESTART_ORACLE: &str = "ec84-target-drive-restart.json";
+
+#[derive(Clone)]
+struct ExpectedShard {
+    key: String,
+    body: Vec<u8>,
+    baseline: VersionShardCensus,
+}
+
+struct ScannerHealEvidenceContext {
+    directory: PathBuf,
+    run: Value,
+}
+
+struct ScannerHealEvidencePayload<'a> {
+    dist: &'a DistCluster,
+    bucket: &'a str,
+    expected: &'a [ExpectedShard],
+    outage_key: &'a str,
+    outage_body: &'a [u8],
+    replaced_drive: &'a Path,
+    pid_before: u32,
+    pid_after: u32,
+    node_listings: Vec<Vec<String>>,
+}
+
+fn file_sha256(path: &Path) -> TestResult<String> {
+    let mut file = std::fs::File::open(path)?;
+    let mut digest = Sha256::new();
+    let mut buffer = [0_u8; 64 * 1024];
+    loop {
+        let read = file.read(&mut buffer)?;
+        if read == 0 {
+            break;
+        }
+        digest.update(&buffer[..read]);
+    }
+    Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
+}
+
+fn compiled_test_identity() -> Value {
+    serde_json::json!({
+        "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
+        "dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
+        "lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
+        "features": env!("RUSTFS_E2E_BUILD_FEATURES"),
+        "target": env!("RUSTFS_E2E_BUILD_TARGET"),
+        "profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
+        "rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
+    })
+}
+
+fn string_field<'a>(value: &'a Value, path: &str) -> TestResult<&'a str> {
+    let mut current = value;
+    for segment in path.split('.') {
+        current = current
+            .get(segment)
+            .ok_or_else(|| format!("scanner/heal run receipt missing {path}"))?;
+    }
+    current
+        .as_str()
+        .filter(|text| !text.is_empty())
+        .ok_or_else(|| format!("scanner/heal run receipt has invalid {path}").into())
+}
+
+fn scanner_heal_evidence_context() -> TestResult<Option<ScannerHealEvidenceContext>> {
+    let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
+        return Ok(None);
+    };
+    let directory = PathBuf::from(directory);
+    let receipt = directory.join("run.json");
+    if receipt.metadata()?.len() > 1024 * 1024 {
+        return Err("oversized scanner/heal execution receipt".into());
+    }
+    let run: Value = serde_json::from_slice(&std::fs::read(receipt)?)?;
+    let built = compiled_test_identity();
+    for key in ["source_revision", "dirty", "lock_blob", "features"] {
+        if built[key] != run["test_build"][key] {
+            return Err(format!("compiled test identity differs for {key}").into());
+        }
+    }
+    let binary_path = PathBuf::from(string_field(&run, "binary.path")?);
+    if file_sha256(&binary_path)? != string_field(&run, "binary.sha256")? {
+        return Err("server binary must match the run receipt".into());
+    }
+    if file_sha256(&std::env::current_exe()?)? != string_field(&run, "test_binary.sha256")? {
+        return Err("test executable must match the run receipt".into());
+    }
+    if directory.join(EC84_TARGET_DRIVE_RESTART_ORACLE).exists() {
+        return Err("scanner/heal oracle already exists; create a new execution receipt".into());
+    }
+    Ok(Some(ScannerHealEvidenceContext { directory, run }))
+}
+
+fn assert_ec84_geometry(census: &VersionShardCensus, key: &str) -> TestResult {
+    if census.data_blocks != Some(EC84_DATA_BLOCKS) || census.parity_blocks != Some(EC84_PARITY_BLOCKS) {
+        return Err(format!("object {key} did not use EC8+4 geometry: {census:?}").into());
+    }
+    let erasure_index = census
+        .erasure_index
+        .ok_or_else(|| format!("object {key} did not record an erasure index: {census:?}"))?;
+    if !(1..=EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS).contains(&erasure_index) {
+        return Err(format!("object {key} has out-of-range erasure index {erasure_index}: {census:?}").into());
+    }
+    if !census.is_complete() || census.expected_part_numbers.is_empty() {
+        return Err(format!("object {key} does not have complete physical shard evidence: {census:?}").into());
+    }
+    Ok(())
+}
+
+async fn write_scanner_heal_evidence(context: ScannerHealEvidenceContext, payload: ScannerHealEvidencePayload<'_>) -> TestResult {
+    let verifier = payload.dist.client(0)?;
+    let mut objects = Vec::new();
+    for item in payload.expected {
+        let actual = get_object_bytes(&verifier, payload.bucket, &item.key).await?;
+        let physical = census_object_version_on_disk(payload.replaced_drive, payload.bucket, &item.key, None)?;
+        objects.push(serde_json::json!({
+            "key": item.key,
+            "version_id": null,
+            "expected_bytes": item.body.len(),
+            "actual_bytes": actual.len(),
+            "expected_sha256": sha256_hex(&item.body),
+            "actual_sha256": sha256_hex(&actual),
+            "expected_physical": item.baseline,
+            "physical": physical,
+        }));
+    }
+    let actual = get_object_bytes(&verifier, payload.bucket, payload.outage_key).await?;
+    let physical = census_object_version_on_disk(payload.replaced_drive, payload.bucket, payload.outage_key, None)?;
+    objects.push(serde_json::json!({
+        "key": payload.outage_key,
+        "version_id": null,
+        "expected_bytes": payload.outage_body.len(),
+        "actual_bytes": actual.len(),
+        "expected_sha256": sha256_hex(payload.outage_body),
+        "actual_sha256": sha256_hex(&actual),
+        "expected_physical": null,
+        "physical": physical,
+    }));
+
+    let evidence = serde_json::json!({
+        "schema": 1,
+        "case": EC84_TARGET_DRIVE_RESTART_CASE,
+        "evidence": "process-restart",
+        "run_id": string_field(&context.run, "run_id")?,
+        "source_revision": string_field(&context.run, "source_revision")?,
+        "test_build": compiled_test_identity(),
+        "binary_sha256": string_field(&context.run, "binary.sha256")?,
+        "test_binary_sha256": string_field(&context.run, "test_binary.sha256")?,
+        "topology": {"nodes": EC84_NODE_COUNT, "drives_per_node": EC84_DRIVES_PER_NODE},
+        "erasure_set_drive_count": EC84_ERASURE_SET_DRIVE_COUNT,
+        "pid_before": payload.pid_before,
+        "pid_after": payload.pid_after,
+        "unclean_shutdown_marker": false,
+        "objects": objects,
+        "node_listings": payload.node_listings,
+    });
+    let data = serde_json::to_vec(&evidence)?;
+    if data.len() > 1024 * 1024 {
+        return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
+    }
+    let mut output = std::fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(context.directory.join(EC84_TARGET_DRIVE_RESTART_ORACLE))?;
+    output.write_all(&data)?;
+    output.sync_all()?;
+    Ok(())
+}
+
+fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> TestResult {
+    for key in keys {
+        let census = census_object_version_on_disk(drive, bucket, key, None)?;
+        if census.has_xl_meta {
+            return Err(format!("replacement drive unexpectedly retained {bucket}/{key}: {census:?}").into());
+        }
+    }
+    Ok(())
+}
+
+async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
+    let mut expected = Vec::new();
+    for index in 0..4 {
+        let key = format!("ec84/prefix-{}/object-{index:04}.bin", index % 2);
+        let body = payload_for(&key, 10 * 1024 * 1024);
+        put_object(client, bucket, &key, body.clone()).await?;
+        expected.push(ExpectedShard {
+            key,
+            body,
+            baseline: VersionShardCensus {
+                version_id: None,
+                has_xl_meta: false,
+                data_dir: None,
+                erasure_index: None,
+                erasure_distribution: None,
+                data_blocks: None,
+                parity_blocks: None,
+                expected_part_numbers: Default::default(),
+                present_part_fingerprints: Default::default(),
+                inline_data_fingerprint: None,
+            },
+        });
+    }
+    Ok(expected)
+}
+
+#[tokio::test]
+async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart() -> TestResult {
+    init_logging();
+    let evidence_context = scanner_heal_evidence_context()?;
+    let mut dist = DistCluster::start_with_env(
+        DistLayout::ThreeByFourEc84,
+        &[
+            ("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
+            ("RUSTFS_HEAL_ENABLED", "true"),
+            ("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"),
+            ("RUSTFS_HEAL_MRF_ENABLE", "false"),
+            ("RUSTFS_SCANNER_ENABLED", "false"),
+        ],
+    )
+    .await?;
+    assert_eq!(dist.cluster.nodes.len(), EC84_NODE_COUNT);
+    assert_eq!(dist.cluster.topology.drives_per_node, EC84_DRIVES_PER_NODE);
+
+    let bucket = unique_bucket("healec84");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let mut expected = put_large_inventory(&writer, &bucket).await?;
+    let replaced_node = 1;
+    let replaced_drive_index = 2;
+    let replaced_drive = PathBuf::from(&dist.cluster.nodes[replaced_node].data_dirs[replaced_drive_index]);
+
+    for item in &mut expected {
+        item.baseline = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
+        assert_ec84_geometry(&item.baseline, &item.key)?;
+    }
+
+    let format_path = replaced_drive.join(".rustfs.sys").join("format.json");
+    let format_json = std::fs::read(&format_path)?;
+    let target_pid_before = dist.cluster.nodes[replaced_node]
+        .process
+        .as_ref()
+        .ok_or("target process is absent before graceful restart")?
+        .id();
+    dist.cluster.stop_node_gracefully(replaced_node).await?;
+    let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
+    std::fs::rename(&replaced_drive, &retired_drive)?;
+    std::fs::create_dir_all(format_path.parent().ok_or("replacement format path has no parent")?)?;
+    std::fs::write(&format_path, format_json)?;
+    assert_replaced_drive_empty(
+        &replaced_drive,
+        &bucket,
+        &expected.iter().map(|item| item.key.clone()).collect::<Vec<_>>(),
+    )?;
+
+    let outage_key = "ec84/written-while-node-restarting.bin";
+    let outage_body = payload_for(outage_key, 10 * 1024 * 1024);
+    writer
+        .put_object()
+        .bucket(&bucket)
+        .key(outage_key)
+        .body(ByteStream::from(outage_body.clone()))
+        .send()
+        .await?;
+
+    dist.cluster.start_node(replaced_node).await?;
+    let target_pid_after = dist.cluster.nodes[replaced_node]
+        .process
+        .as_ref()
+        .ok_or("target process is absent after restart")?
+        .id();
+    let heal_body =
+        r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
+    let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
+    start_root_heal_when_control_ready(&heal_url, heal_body, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
+
+    wait_until(
+        Duration::from_secs(120),
+        || async {
+            for item in &expected {
+                let current = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
+                if !current.matches_manifest(&item.baseline) {
+                    return Ok(false);
+                }
+            }
+            let outage = census_object_version_on_disk(&replaced_drive, &bucket, outage_key, None)?;
+            Ok(outage.is_complete()
+                && outage.data_blocks == Some(EC84_DATA_BLOCKS)
+                && outage.parity_blocks == Some(EC84_PARITY_BLOCKS))
+        },
+        "EC8+4 replacement drive rebuilt baseline and outage shards",
+    )
+    .await?;
+
+    let inventory = expected
+        .iter()
+        .map(|item| (item.key.clone(), item.body.clone()))
+        .chain(std::iter::once((outage_key.to_string(), outage_body.clone())))
+        .collect::<BTreeMap<_, _>>();
+    let expected_keys = inventory.keys().cloned().collect::<HashSet<_>>();
+    let mut node_listings = Vec::new();
+    for node_index in 0..dist.cluster.nodes.len() {
+        let client = dist.client(node_index)?;
+        assert_inventory(&client, &bucket, &inventory).await?;
+        let listing = client.list_objects_v2().bucket(&bucket).send().await?;
+        let observed = listing
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(str::to_owned))
+            .collect::<HashSet<_>>();
+        assert_eq!(observed, expected_keys, "node {node_index} listing diverged after EC8+4 heal");
+        let mut observed = observed.into_iter().collect::<Vec<_>>();
+        observed.sort();
+        node_listings.push(observed);
+    }
+    if let Some(context) = evidence_context {
+        write_scanner_heal_evidence(
+            context,
+            ScannerHealEvidencePayload {
+                dist: &dist,
+                bucket: &bucket,
+                expected: &expected,
+                outage_key,
+                outage_body: &outage_body,
+                replaced_drive: &replaced_drive,
+                pid_before: target_pid_before,
+                pid_after: target_pid_after,
+                node_listings,
+            },
+        )
+        .await?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chaos::is_cluster_heal_coordination_unavailable;
+
+    #[test]
+    fn cluster_heal_coordination_retry_is_exact() {
+        let retryable: Box<dyn std::error::Error + Send + Sync> =
+            "admin POST failed: 500 Internal Server Error cluster heal coordination unavailable".into();
+        assert!(is_cluster_heal_coordination_unavailable(retryable.as_ref()));
+
+        let other_internal: Box<dyn std::error::Error + Send + Sync> =
+            "admin POST failed: 500 Internal Server Error unrelated".into();
+        assert!(!is_cluster_heal_coordination_unavailable(other_internal.as_ref()));
+
+        let wrong_status: Box<dyn std::error::Error + Send + Sync> =
+            "admin POST failed: 503 Service Unavailable cluster heal coordination unavailable".into();
+        assert!(!is_cluster_heal_coordination_unavailable(wrong_status.as_ref()));
+    }
+
+    #[test]
+    fn ec84_drive_restart_evidence_shape_matches_registry() {
+        assert_eq!(EC84_ERASURE_SET_DRIVE_COUNT, EC84_NODE_COUNT * EC84_DRIVES_PER_NODE);
+
+        let registry: Value = serde_json::from_str(include_str!("../../../../.config/scanner-heal-required-tests.json"))
+            .expect("scanner/heal registry is valid JSON");
+        let case = &registry["cases"][EC84_TARGET_DRIVE_RESTART_CASE];
+        assert_eq!(case["erasure_set_drive_count"], EC84_ERASURE_SET_DRIVE_COUNT);
+        assert_eq!(case["topology"]["nodes"], EC84_NODE_COUNT);
+        assert_eq!(case["topology"]["drives_per_node"], EC84_DRIVES_PER_NODE);
+    }
+}

crates/e2e_test/src/distributed/mod.rs

@@ -25,8 +25,10 @@ mod data_integrity_movement_test;
 mod expand_decommission_rebalance_test;
 mod extra_test;
 mod harness;
+mod heal_test;
 mod object_lock_test;
 mod observability_test;
+mod replication_delete_marker_test;
 mod replication_quota_test;
 mod s3_basic_test;
 mod s3_during_data_movement_test;

crates/e2e_test/src/distributed/replication_delete_marker_test.rs

@@ -0,0 +1,137 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Functional REP-105 (rustfs/backlog#2195 item 4): a delete marker created
+//! on a multi-node source cluster must replicate to the bucket-replication
+//! target. Objects converged in seconds while delete markers did not arrive
+//! within 180 s on the shared 3-node functional environment; the single-node
+//! e2e never saw it.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, set_remote_target, unique_bucket,
+    wait_for_replicated_bytes, wait_until,
+};
+use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, init_logging, replication_fast_env, signed_request};
+use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV;
+use aws_sdk_s3::Client;
+use http::{Method, StatusCode};
+use std::time::Duration;
+
+async fn target_has_delete_marker(client: &Client, bucket: &str, key: &str) -> TestResult<bool> {
+    let versions = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
+    Ok(versions.delete_markers().iter().any(|marker| marker.key() == Some(key)))
+}
+
+async fn delete_marker_replicates(
+    source: &DistCluster,
+    source_bucket: &str,
+    target_client: &Client,
+    target_bucket: &str,
+) -> TestResult {
+    let key = "delete-marker/object.bin";
+    let body = b"delete marker replication payload".to_vec();
+    // Write through one node, delete through another: behind a load
+    // balancer consecutive requests land on different nodes.
+    put_object(&source.client(1)?, source_bucket, key, body.clone()).await?;
+    wait_for_replicated_bytes(target_client, target_bucket, key, &body, Duration::from_secs(60)).await?;
+
+    let delete = source
+        .client(2)?
+        .delete_object()
+        .bucket(source_bucket)
+        .key(key)
+        .send()
+        .await?;
+    assert_eq!(
+        delete.delete_marker(),
+        Some(true),
+        "a versioned DELETE without versionId must create a marker"
+    );
+
+    wait_until(
+        Duration::from_secs(90),
+        || async { target_has_delete_marker(target_client, target_bucket, key).await },
+        "delete marker replicated to the target bucket",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_bucket_replication_replicates_delete_marker_to_peer_cluster() -> TestResult {
+    init_logging();
+    let (source, target) = DistCluster::start_replication_pair().await?;
+    let source_bucket = unique_bucket("dm-src");
+    let target_bucket = unique_bucket("dm-dst");
+    source.create_bucket(&source_bucket).await?;
+    target.create_bucket(&target_bucket).await?;
+    enable_versioning(&source.client(0)?, &source_bucket).await?;
+    enable_versioning(&target.client(0)?, &target_bucket).await?;
+
+    let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
+    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
+
+    delete_marker_replicates(&source, &source_bucket, &target.client(0)?, &target_bucket).await
+}
+
+/// The functional environment replicates from a 3-node site to a single-node
+/// target; keep that shape as its own case.
+#[tokio::test]
+async fn four_node_bucket_replication_replicates_delete_marker_to_single_node_target() -> TestResult {
+    init_logging();
+    let mut extra: Vec<(&str, &str)> = replication_fast_env();
+    extra.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+    extra.extend_from_slice(FAST_DATA_USAGE_SCANNER_ENV);
+    let source = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?;
+    let mut target = RustFSTestEnvironment::new().await?;
+    target.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = unique_bucket("dm-src");
+    let target_bucket = unique_bucket("dm-dst");
+    source.create_bucket(&source_bucket).await?;
+    let target_client = target.create_s3_client();
+    target_client.create_bucket().bucket(&target_bucket).send().await?;
+    enable_versioning(&source.client(0)?, &source_bucket).await?;
+    enable_versioning(&target_client, &target_bucket).await?;
+
+    let body = serde_json::json!({
+        "endpoint": target.address,
+        "credentials": { "accessKey": target.access_key, "secretKey": target.secret_key },
+        "targetbucket": target_bucket,
+        "secure": false,
+        "type": "replication"
+    });
+    let url = format!(
+        "{}/rustfs/admin/v3/set-remote-target?bucket={}",
+        source.cluster.nodes[0].url,
+        urlencoding::encode(&source_bucket)
+    );
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &source.cluster.access_key,
+        &source.cluster.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("set remote target failed: {status} {body}").into());
+    }
+    let arn: String = serde_json::from_slice(&response.bytes().await?)?;
+    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
+
+    delete_marker_replicates(&source, &source_bucket, &target_client, &target_bucket).await
+}

crates/e2e_test/src/distributed/s3_during_data_movement_test.rs

@@ -14,9 +14,9 @@
 
 use super::harness::{
     DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
-    decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
-    retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
-    wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
+    decommission_status_json, put_inventory_retrying, rebalance_active, rebalance_status_json, retrying_get_equals, retrying_put,
+    start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
+    wait_for_decommission_running_with_progress, wait_for_rebalance_active, wait_for_rebalance_complete,
 };
 use crate::common::init_logging;
 use std::time::Duration;
@@ -67,7 +67,10 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu
     assert_inventory(&live, &bucket, &inventory).await?;
 
     let rebalance_id = start_rebalance(&dist.cluster).await?;
-    wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    // The status API reads persisted progress, whose first periodic save is
+    // after 30 seconds. A shorter run can remain at zero until completion.
+    // Require Started around the S3 operations and nonzero progress at completion.
+    wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
     retrying_put(
         &live,
         &bucket,
@@ -84,11 +87,26 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu
         Duration::from_secs(30),
     )
     .await?;
+    let listed = live.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed
+            .contents()
+            .iter()
+            .any(|object| object.key() == Some("during-rebalance.bin")),
+        "list during rebalance missed the newly written key"
+    );
     let status = rebalance_status_json(&dist.cluster).await?;
-    if !rebalance_running_with_progress(&status, &rebalance_id)? {
+    if !rebalance_active(&status, &rebalance_id)? {
         return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
     }
     wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
-    assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
+    let after = dist.client(1)?;
+    assert_inventory(&after, &bucket, &inventory).await?;
+    for (key, body) in [
+        ("during-decommission.bin", b"written-while-decommissioning".as_slice()),
+        ("during-rebalance.bin", b"written-while-rebalancing".as_slice()),
+    ] {
+        retrying_get_equals(&after, &bucket, key, body, Duration::from_secs(30)).await?;
+    }
     Ok(())
 }

crates/e2e_test/src/distributed/site_replication_test.rs

@@ -126,3 +126,165 @@ async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResu
     wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
     Ok(())
 }
+
+async fn node_admin(
+    cluster: &crate::common::RustFSTestClusterEnvironment,
+    node_idx: usize,
+    method: Method,
+    path_and_query: &str,
+    body: Option<String>,
+) -> TestResult<(StatusCode, String)> {
+    crate::common::admin_request(
+        &cluster.nodes[node_idx].url,
+        method,
+        path_and_query,
+        body,
+        &cluster.access_key,
+        &cluster.secret_key,
+    )
+    .await
+}
+
+/// Pair two clusters through site A's first node and wait until both report
+/// the two-site topology as enabled.
+async fn pair_sites(site_a: &DistCluster, site_b: &DistCluster) -> TestResult {
+    let sites = vec![
+        PeerSite {
+            name: "site-a".to_string(),
+            endpoint: site_a.cluster.nodes[0].url.clone(),
+            access_key: site_a.cluster.access_key.clone(),
+            secret_key: site_a.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+        PeerSite {
+            name: "site-b".to_string(),
+            endpoint: site_b.cluster.nodes[0].url.clone(),
+            access_key: site_b.cluster.access_key.clone(),
+            secret_key: site_b.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+    ];
+    let add_status = site_replication_add(&site_a.cluster, &sites).await?;
+    assert!(
+        add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
+        "site replication add reported failure: {add_status:?}"
+    );
+    wait_for_site_replication_enabled(&site_a.cluster).await?;
+    wait_for_site_replication_enabled(&site_b.cluster).await?;
+    Ok(())
+}
+
+async fn list_users_contains(
+    cluster: &crate::common::RustFSTestClusterEnvironment,
+    node_idx: usize,
+    access_key: &str,
+) -> TestResult<bool> {
+    let (status, body) = node_admin(cluster, node_idx, Method::GET, "/rustfs/admin/v3/list-users", None).await?;
+    if !status.is_success() {
+        return Err(format!("list-users on node {node_idx} failed: {status} {body}").into());
+    }
+    let users: serde_json::Value = serde_json::from_str(&body)?;
+    Ok(users.get(access_key).is_some())
+}
+
+/// backlog#2367 A-7 / functional SITE-102: an IAM change handled by a node
+/// other than the one that ran `site-replication/add` must still reach the
+/// peer site. Behind a load balancer every admin call may land on a
+/// different node, so the coordinator node is not special.
+#[tokio::test]
+async fn four_node_site_replication_converges_iam_user_created_on_a_non_coordinator_node() -> TestResult {
+    init_logging();
+    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
+    pair_sites(&site_a, &site_b).await?;
+
+    let user = format!("siteuser-{}", &uuid::Uuid::new_v4().simple().to_string()[..8]);
+    let body = serde_json::json!({ "secretKey": "siteuser-secret-key-1234", "status": "enabled" }).to_string();
+    let (status, response) = node_admin(
+        &site_a.cluster,
+        1,
+        Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
+        Some(body),
+    )
+    .await?;
+    assert!(status.is_success(), "add-user on site A node 1 failed: {status} {response}");
+
+    let site_b_cluster = &site_b.cluster;
+    let user_ref = user.as_str();
+    wait_until(
+        Duration::from_secs(90),
+        || async move { list_users_contains(site_b_cluster, 0, user_ref).await },
+        "user created on site A node 1 visible on site B",
+    )
+    .await?;
+    assert!(
+        list_users_contains(&site_a.cluster, 2, &user).await?,
+        "the user must be visible on every site A node"
+    );
+    Ok(())
+}
+
+/// backlog#2367 A-5 / functional SITE-105: a resync started right after
+/// pairing must not report buckets as failed. The bucket carrying an
+/// operator-configured bucket-replication target to the peer (the shape the
+/// functional suite leaves behind) and a plain versioned bucket are both
+/// wired by the pairing itself.
+#[tokio::test]
+async fn four_node_site_replication_resync_start_right_after_pairing_reports_no_failed_bucket() -> TestResult {
+    init_logging();
+    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
+
+    let pre_src = unique_bucket("pre-src");
+    let pre_dst = unique_bucket("pre-dst");
+    let plain = unique_bucket("plain");
+    site_a.create_bucket(&pre_src).await?;
+    site_b.create_bucket(&pre_dst).await?;
+    site_a.create_bucket(&plain).await?;
+    enable_versioning(&site_a.client(0)?, &pre_src).await?;
+    enable_versioning(&site_b.client(0)?, &pre_dst).await?;
+    enable_versioning(&site_a.client(0)?, &plain).await?;
+    let arn = super::harness::set_remote_target(&site_a.cluster, &pre_src, &site_b.cluster, &pre_dst).await?;
+    super::harness::put_bucket_replication(&site_a.cluster, &pre_src, &arn).await?;
+
+    pair_sites(&site_a, &site_b).await?;
+
+    let (status, info) = node_admin(&site_a.cluster, 1, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
+    assert!(status.is_success(), "site-replication/info failed: {status} {info}");
+    let info: serde_json::Value = serde_json::from_str(&info)?;
+    let peer = info["sites"]
+        .as_array()
+        .and_then(|sites| sites.iter().find(|site| site["name"] == "site-b"))
+        .cloned()
+        .ok_or_else(|| format!("site-b peer missing from info: {info}"))?;
+
+    // Through a non-coordinator node, like a load-balanced admin call.
+    let (status, response) = node_admin(
+        &site_a.cluster,
+        1,
+        Method::PUT,
+        "/rustfs/admin/v3/site-replication/resync/op?operation=start",
+        Some(peer.to_string()),
+    )
+    .await?;
+    assert!(status.is_success(), "resync start failed: {status} {response}");
+    let resync: rustfs_madmin::SRResyncOpStatus = serde_json::from_str(&response)?;
+    let failed: Vec<String> = resync
+        .buckets
+        .iter()
+        .filter(|bucket| bucket.status == "failed")
+        .map(|bucket| format!("{}: {}", bucket.bucket, bucket.err_detail))
+        .collect();
+    assert!(
+        failed.is_empty(),
+        "resync right after pairing reported failed buckets: {failed:?} (status={}, detail={})",
+        resync.status,
+        resync.err_detail
+    );
+    assert!(
+        resync.buckets.iter().any(|bucket| bucket.bucket == pre_src)
+            && resync.buckets.iter().any(|bucket| bucket.bucket == plain),
+        "both buckets must be part of the resync: {:?}",
+        resync.buckets
+    );
+    Ok(())
+}

crates/e2e_test/src/inline_fast_path_cluster_test.rs

@@ -929,7 +929,13 @@ async fn assert_case(
         case.label
     );
     assert_storage_layout(cluster, bucket, &key, version_id.as_deref(), case.stored_inline)?;
-    Ok((key, body, put.e_tag().map(str::to_owned), version_id))
+    // A suspended PUT omits the response version, but reads identify the
+    // stored null version explicitly.
+    let read_version_id = match state {
+        VersionState::Suspended => Some("null".to_owned()),
+        _ => version_id,
+    };
+    Ok((key, body, put.e_tag().map(str::to_owned), read_version_id))
 }
 
 async fn get_and_assert(
@@ -1788,6 +1794,8 @@ async fn four_node_inline_fallback_controls() -> TestResult {
     configure_reader_metric_cluster(&mut cluster, &collector);
     let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
     cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", &sse_master_key);
+    // Inspect every disk only after the PUT rename fanout has drained.
+    cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
     cluster.start().await?;
 
     let bucket = "inline-fallback-controls";
@@ -1839,6 +1847,10 @@ async fn four_node_inline_fallback_controls() -> TestResult {
         ),
     )
     .await?;
+    // 16 KiB of plaintext is 8 KiB per data shard on EC 2+2, inside the inline
+    // budget: an SSE-S3 PUT is admitted inline by its plaintext size even though
+    // the ciphertext length is unknown up front (backlog#2393 STOR-115).
+    assert_storage_layout(&cluster, bucket, encrypted_key, None, true)?;
 
     Ok(())
 }
@@ -1851,6 +1863,8 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
     let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
     configure_reader_metric_cluster(&mut cluster, &collector);
     cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
+    // Inspect every disk only after the PUT rename fanout has drained.
+    cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
     cluster.start().await?;
 
     let bucket = "inline-compressed-fallback";
@@ -1871,6 +1885,33 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
         ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, put.e_tag(), None), LEGACY_DUPLEX, COMPRESSED),
     )
     .await?;
+    // 64 KiB of plaintext is 32 KiB per data shard on EC 2+2, inside the inline
+    // budget: a compressed PUT is admitted inline by its plaintext size even
+    // though its stored size is unknown up front (backlog#2393 STOR-115).
+    assert_storage_layout(&cluster, bucket, key, None, true)?;
+
+    // A server-side copy onto another compressible key re-compresses the
+    // decompressed source stream and lands inline the same way (STOR-125).
+    let copy_key = "compressed/copy.txt";
+    let copy = client
+        .copy_object()
+        .bucket(bucket)
+        .key(copy_key)
+        .copy_source(format!("{bucket}/{key}"))
+        .send()
+        .await?;
+    let copy_etag = copy.copy_object_result().and_then(|result| result.e_tag()).map(str::to_owned);
+    assert_reader_path(
+        &collector,
+        &client,
+        ReaderPathExpectation::for_class(
+            ReaderObject::new(bucket, copy_key, &body, copy_etag.as_deref(), None),
+            LEGACY_DUPLEX,
+            COMPRESSED,
+        ),
+    )
+    .await?;
+    assert_storage_layout(&cluster, bucket, copy_key, None, true)?;
 
     Ok(())
 }

crates/e2e_test/src/internode_rpc_signature_e2e_test.rs

@@ -58,10 +58,11 @@
 //!
 //! Every covered handler checks the digest before touching storage, and
 //! `MakeVolume` resolves its disk *after* that check. Aiming at a disk that
-//! cannot exist gives three cleanly separable outcomes with zero side effects
+//! cannot exist gives four cleanly separable outcomes with zero side effects
 //! on the server's real data:
 //!
 //! - `Err(Unauthenticated)` — rejected by `check_auth` (signature layer).
+//! - `Err(Unavailable)` — a valid signature with a stale boot epoch, rejected before execution.
 //! - `Err(PermissionDenied)` — rejected by the handler's body-digest gate.
 //! - `Ok(success: false)` — **authentication passed**; the request reached
 //!   handler logic and only then failed on the bogus disk.
@@ -468,8 +469,8 @@ async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> Te
     rustfs_protos::evict_failed_connection(&url).await;
     assert_rejected(
         call_make_volume(&url, request.clone(), stale_epoch).await,
-        Code::Unauthenticated,
-        None,
+        Code::Unavailable,
+        Some("RPC boot epoch changed"),
         "a replay-scoped signature captured before the receiving process restart",
     );
 

crates/e2e_test/src/kms/README.md

@@ -57,50 +57,46 @@ Broad integration tests that exercise:
    pip install awscurl
    ```
 
-2. **Build RustFS**
+2. **Build RustFS** (from the repository root)
    ```bash
-   cargo build
+   python3 scripts/e2e_binary.py build
    ```
 
 ### Run individual suites
 
+Run every command below from the repository root through `scripts/e2e_binary.py run`; plain `cargo test -p e2e_test` fails with a missing E2E run receipt.
+
 #### Local backend
 ```bash
-cd crates/e2e_test
-cargo test test_local_kms_end_to_end -- --nocapture
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_local_kms_end_to_end -- --nocapture
 ```
 
 #### Vault backend
 ```bash
-cd crates/e2e_test
-cargo test test_vault_kms_end_to_end -- --nocapture
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_vault_kms_end_to_end -- --nocapture
 ```
 
 #### High availability
 ```bash
-cd crates/e2e_test
-cargo test test_vault_kms_high_availability -- --nocapture
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_vault_kms_high_availability -- --nocapture
 ```
 
 #### Comprehensive features (disabled)
 ```bash
-cd crates/e2e_test
 # Disabled due to AWS SDK compatibility gaps
-# cargo test test_comprehensive_kms_functionality -- --nocapture
-# cargo test test_sse_modes_compatibility -- --nocapture
-# cargo test test_kms_api_comprehensive -- --nocapture
+# python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_comprehensive_kms_functionality -- --nocapture
+# python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_sse_modes_compatibility -- --nocapture
+# python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_kms_api_comprehensive -- --nocapture
 ```
 
 ### Run all KMS suites
 ```bash
-cd crates/e2e_test
-cargo test kms -- --nocapture
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test kms -- --nocapture
 ```
 
 ### Run serially (avoid port conflicts)
 ```bash
-cd crates/e2e_test
-cargo test kms -- --nocapture --test-threads=1
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test kms -- --nocapture --test-threads=1
 ```
 
 ## 🔧 Configuration
@@ -120,7 +116,7 @@ export RUST_LOG=debug
 ### Required binaries
 
 Tests look for:
-- `../../target/debug/rustfs` – RustFS server
+- RustFS server – the binary verified by `scripts/e2e_binary.py` (default `target/debug/rustfs`)
 - `vault` – Vault CLI (must be on PATH)
 - `/Users/dandan/Library/Python/3.9/bin/awscurl` – AWS SigV4 helper
 
@@ -174,14 +170,14 @@ which awscurl  # Update the path in tests accordingly
 
 **Q: Tests time out**
 ```bash
-RUST_LOG=debug cargo test test_local_kms_end_to_end -- --nocapture
+RUST_LOG=debug python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_local_kms_end_to_end -- --nocapture
 ```
 
 ### Debug tips
 
 1. **Enable verbose logs**
    ```bash
-   RUST_LOG=rustfs_kms=debug,rustfs=info cargo test -- --nocapture
+   RUST_LOG=rustfs_kms=debug,rustfs=info python3 scripts/e2e_binary.py run -- cargo test -p e2e_test kms -- --nocapture
    ```
 
 2. **Keep temporary files** – comment out cleanup logic to inspect generated configs
@@ -237,9 +233,8 @@ Designed to run inside CI/CD pipelines:
     sudo apt-get install -y vault
     pip install awscurl
 
-    cargo build
-    cd crates/e2e_test
-    cargo test kms -- --nocapture --test-threads=1
+    python3 scripts/e2e_binary.py build
+    python3 scripts/e2e_binary.py run -- cargo test -p e2e_test kms -- --nocapture --test-threads=1
 ```
 
 ## 📚 References

crates/e2e_test/src/kms/bucket_default_encryption_test.rs

@@ -21,6 +21,7 @@
 
 use super::common::LocalKMSTestEnvironment;
 use crate::common::{TEST_BUCKET, init_logging};
+use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
     ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption,
@@ -631,3 +632,92 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s
     info!("Test passed: SSE-KMS without key ID correctly populates default key '{}'", default_key_id);
     Ok(())
 }
+
+/// A default-encryption configuration the write path cannot honour as written
+/// must be refused rather than stored: the write path falls back to AES256 for
+/// any algorithm it does not know, so storing `AES128` would make
+/// GetBucketEncryption report a scheme no object is encrypted under.
+#[tokio::test]
+async fn test_put_bucket_encryption_rejects_unknown_algorithm_and_misplaced_key_id()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut kms_env = LocalKMSTestEnvironment::new().await?;
+    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
+    kms_env.wait_for_kms_ready().await?;
+
+    let s3_client = kms_env.base_env.create_s3_client();
+    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
+
+    let config_with = |algorithm: ServerSideEncryption, key_id: Option<&str>| {
+        let mut by_default = ServerSideEncryptionByDefault::builder().sse_algorithm(algorithm);
+        if let Some(key_id) = key_id {
+            by_default = by_default.kms_master_key_id(key_id);
+        }
+        ServerSideEncryptionConfiguration::builder()
+            .rules(
+                ServerSideEncryptionRule::builder()
+                    .apply_server_side_encryption_by_default(by_default.build().unwrap())
+                    .build(),
+            )
+            .build()
+            .unwrap()
+    };
+
+    // Baseline the bucket on AES256 so a refused update has something to leave untouched.
+    s3_client
+        .put_bucket_encryption()
+        .bucket(TEST_BUCKET)
+        .server_side_encryption_configuration(config_with(ServerSideEncryption::Aes256, None))
+        .send()
+        .await?;
+
+    let unknown = s3_client
+        .put_bucket_encryption()
+        .bucket(TEST_BUCKET)
+        .server_side_encryption_configuration(config_with(ServerSideEncryption::from("AES128"), None))
+        .send()
+        .await
+        .expect_err("an unknown SSEAlgorithm must be refused");
+    assert_eq!(unknown.raw_response().map(|response| response.status().as_u16()), Some(400));
+    assert_eq!(
+        unknown.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("MalformedXML"),
+        "unknown algorithm error was {unknown:?}"
+    );
+
+    let misplaced_key = s3_client
+        .put_bucket_encryption()
+        .bucket(TEST_BUCKET)
+        .server_side_encryption_configuration(config_with(ServerSideEncryption::Aes256, Some(&default_key_id)))
+        .send()
+        .await
+        .expect_err("KMSMasterKeyID with AES256 must be refused");
+    assert_eq!(misplaced_key.raw_response().map(|response| response.status().as_u16()), Some(400));
+    assert_eq!(
+        misplaced_key.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("InvalidArgument"),
+        "misplaced key id error was {misplaced_key:?}"
+    );
+
+    // The refused updates left the baseline in place.
+    let stored = s3_client.get_bucket_encryption().bucket(TEST_BUCKET).send().await?;
+    let by_default = stored
+        .server_side_encryption_configuration()
+        .and_then(|config| config.rules().first())
+        .and_then(|rule| rule.apply_server_side_encryption_by_default())
+        .expect("baseline configuration must still be present");
+    assert_eq!(by_default.sse_algorithm(), &ServerSideEncryption::Aes256);
+    assert_eq!(by_default.kms_master_key_id(), None);
+
+    // aws:kms with a key id stays accepted.
+    s3_client
+        .put_bucket_encryption()
+        .bucket(TEST_BUCKET)
+        .server_side_encryption_configuration(config_with(ServerSideEncryption::AwsKms, Some(&default_key_id)))
+        .send()
+        .await?;
+
+    kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
+    Ok(())
+}