mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-08 13:06:00 +00:00
Compare commits
35 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c03d3cdd59 | |||
| 590fab5c7e | |||
| c02967baf6 | |||
| 0b05b6c6ff | |||
| c507da8f75 | |||
| 550dabeffd | |||
| 4d7f0344d3 | |||
| 084338add6 | |||
| 8d339da706 | |||
| 817ad0a682 | |||
| 80321e5bb4 | |||
| e7475cfa4d | |||
| 8c15025a5a | |||
| 6f2ec66263 | |||
| 62542ddc57 | |||
| a8bb53218d | |||
| 5355d9f8f8 | |||
| 50d7a049ee | |||
| 3149c87cf2 | |||
| b7fa6a4615 | |||
| 6fe83f87a4 | |||
| df6981d88e | |||
| 590adad5ae | |||
| be5d14c985 | |||
| 703086d71d | |||
| a159f312f0 | |||
| 2f6f095298 | |||
| efd8ef005f | |||
| 60fa33773a | |||
| ee752b0b03 | |||
| 99c1f4418b | |||
| 7ce0ac72cf | |||
| 944e26d432 | |||
| f0b0a99260 | |||
| 33ddc10ffd |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=f0c78fdb93471575d9a64c5c46eae6c806bdd0bc10a6e33d7fb574aabd8db5a3
|
||||
sha256-linux=03ed7016cab672de9320e31375a0358eceacb4408b0e79cf063614fa7c878b87
|
||||
sha256-darwin=cca6d0bc1487f472dc354bbffcc2b5c7410edfccb499c6bce6638113fbe6bbec
|
||||
sha256-linux=3b71936f6f4ea0cca3b5db6c2f387c990dddda7e96c982315eb42e3b2e6c92b8
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=a5665318c9bdc0947514fb7008ba1b83b114b739fac775c3c446f207058b7c7a
|
||||
sha256-linux=45d80e1723de5d25bb5b81f3ef5c82f583efc3e4f036a8cd2bb99e4f1eca9e51
|
||||
sha256-darwin=83a7dcaffd5a789517ae9f02a224f66a9713937885cff96fca2ad7e216f197ae
|
||||
sha256-linux=626c10f8c964507ff987b6c86069e9019dc6d2ae7fb02db9be5df5aa8cc5145b
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=0fe8408874ccec3620262a9812d67920ddd72dc9edf0e36e0d0aed3f8bad026e
|
||||
sha256=0e338d305260229e17ccfb2adc48a6212dbdfea36a9ebfb5a4e0d38658e6cc45
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"schema": 2,
|
||||
"cases": {
|
||||
"background-target-restart": {
|
||||
"gate": "G14",
|
||||
@@ -13,6 +13,7 @@
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"erasure": {"data_blocks": 2, "parity_blocks": 2},
|
||||
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
},
|
||||
"background-target-crash": {
|
||||
@@ -27,30 +28,282 @@
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"erasure": {"data_blocks": 2, "parity_blocks": 2},
|
||||
"scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
},
|
||||
"ec84-target-drive-restart": {
|
||||
"gate": "G14",
|
||||
"task": "W20/W21",
|
||||
"lane": "e2e-distributed",
|
||||
"suite": "e2e_test",
|
||||
"name": "distributed::heal_test::three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart",
|
||||
"oracle": "ec84-target-drive-restart.json",
|
||||
"evidence": "process-restart",
|
||||
"unclean_shutdown_marker": false,
|
||||
"min_objects": 5,
|
||||
"max_objects": 5,
|
||||
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||
"scope": "3-node x 4-drive single-set EC8+4, graceful target restart, preformatted replacement drive, exact unversioned S3 bodies and physical target shards; not mixed-version, multi-pool or long-window ABBA."
|
||||
},
|
||||
"background-target-restart-ec8-4": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart",
|
||||
"oracle": "background-target-restart-ec8-4.json",
|
||||
"evidence": "process-restart",
|
||||
"unclean_shutdown_marker": false,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||
"erasure_set_drive_count": 12,
|
||||
"scope": "Target process restart during partial background rebuild on a single 3x4 EC8+4 set; exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
|
||||
},
|
||||
"background-target-crash-ec8-4": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash",
|
||||
"oracle": "background-target-crash-ec8-4.json",
|
||||
"evidence": "process-crash-restart",
|
||||
"unclean_shutdown_marker": true,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||
"erasure_set_drive_count": 12,
|
||||
"scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
|
||||
}
|
||||
},
|
||||
"release_pending": {
|
||||
"G01": "W02/W04 complete root and quota authority coverage",
|
||||
"G02": "W03 bounded checkpoint progress and independent version inventory",
|
||||
"G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
|
||||
"G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
|
||||
"G05": "W06/W07 per-object outcomes and bounded terminal retention",
|
||||
"G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
|
||||
"G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
|
||||
"G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
|
||||
"G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
|
||||
"G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
|
||||
"G11": "W04/W19/W24 maintenance and complete producer coverage",
|
||||
"G12": "W02/W15/W16 both quota paths during reset and settlement",
|
||||
"G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||
"G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
|
||||
"P1": "W20 measured cold-walk share and foreground latency/throughput",
|
||||
"P2": "W20/W24 measured post-stop convergence and cold segment reuse",
|
||||
"P3": "W20 measured two-hour pressure/heal capacity and recovery window",
|
||||
"P4": "W20 measured MRF scale and replay cost with retained responsibility",
|
||||
"R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
|
||||
"R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
|
||||
"R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
|
||||
}
|
||||
"release_lanes": {
|
||||
"single-set-restart": {
|
||||
"status": "implemented",
|
||||
"cases": ["background-target-restart", "background-target-crash"],
|
||||
"covers": ["four-node one-drive topology", "unversioned objects", "target restart/crash"]
|
||||
},
|
||||
"authority-coverage": {
|
||||
"status": "pending",
|
||||
"gates": ["G01", "G12"],
|
||||
"requires": ["root authority coverage", "quota authority coverage"]
|
||||
},
|
||||
"checkpoint-and-crash": {
|
||||
"status": "pending",
|
||||
"gates": ["G02", "G04", "R-E"],
|
||||
"requires": ["bounded checkpoint progress", "boundary crash matrix", "fixed-budget restart evidence"]
|
||||
},
|
||||
"status-and-outcome": {
|
||||
"status": "pending",
|
||||
"gates": ["G05", "G06", "R-D"],
|
||||
"requires": ["per-object outcomes", "legacy status clients", "manager/event/ledger disposition"]
|
||||
},
|
||||
"mrf-responsibility": {
|
||||
"status": "pending",
|
||||
"gates": ["G07", "G08", "P4"],
|
||||
"requires": ["durable MRF responsibility", "disk-full and replica-loss matrix", "MRF replay cost"]
|
||||
},
|
||||
"mixed-version-rollback": {
|
||||
"status": "pending",
|
||||
"gates": ["G03", "G09", "R-L"],
|
||||
"requires": ["mixed-version peers", "rollback payloads", "crash-safe source retirement"]
|
||||
},
|
||||
"scheduler-pressure": {
|
||||
"status": "pending",
|
||||
"gates": ["G10", "P1", "P2", "P3"],
|
||||
"requires": ["bounded scheduling", "foreground latency and throughput", "two-hour pressure evidence"]
|
||||
},
|
||||
"maintenance-producers": {
|
||||
"status": "pending",
|
||||
"gates": ["G11", "G13"],
|
||||
"requires": ["complete producer coverage", "quorum-minus-one and remount matrix"]
|
||||
},
|
||||
"ec8-4-multiset": {
|
||||
"status": "pending",
|
||||
"gates": ["G14"],
|
||||
"requires": ["3x4 EC8+4 topology", "multi-set coverage", "multi-pool coverage"]
|
||||
}
|
||||
},
|
||||
"release_requirements": [
|
||||
{
|
||||
"gate": "G01",
|
||||
"task": "W02/W04",
|
||||
"lane": "authority-coverage",
|
||||
"status": "pending",
|
||||
"description": "Complete root and quota authority coverage",
|
||||
"requires": ["root authority evidence", "quota authority evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G02",
|
||||
"task": "W03",
|
||||
"lane": "checkpoint-and-crash",
|
||||
"status": "pending",
|
||||
"description": "Bounded checkpoint progress and independent version inventory",
|
||||
"requires": ["bounded checkpoint oracle", "independent version inventory"]
|
||||
},
|
||||
{
|
||||
"gate": "G03",
|
||||
"task": "W17/W18",
|
||||
"lane": "mixed-version-rollback",
|
||||
"status": "pending",
|
||||
"description": "Exact scoped ACK with durable publication and mixed peers",
|
||||
"requires": ["durable scoped ACK publication", "mixed-peer evidence"],
|
||||
"evidence_fields": [
|
||||
"durable_root_publication_proof",
|
||||
"scoped_ack_request_identity",
|
||||
"participating_peer_capability_snapshot",
|
||||
"mixed_peer_ack_fallback_oracle"
|
||||
]
|
||||
},
|
||||
{
|
||||
"gate": "G04",
|
||||
"task": "W03/W15/W16",
|
||||
"lane": "checkpoint-and-crash",
|
||||
"status": "pending",
|
||||
"description": "Crash at every cache, root, floor and intent boundary",
|
||||
"requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G05",
|
||||
"task": "W06/W07",
|
||||
"lane": "status-and-outcome",
|
||||
"status": "pending",
|
||||
"description": "Per-object outcomes and bounded terminal retention",
|
||||
"requires": ["per-object outcome oracle", "terminal retention bounds"]
|
||||
},
|
||||
{
|
||||
"gate": "G06",
|
||||
"task": "W06/W08/W23",
|
||||
"lane": "status-and-outcome",
|
||||
"status": "pending",
|
||||
"description": "Concurrent status, legacy clients and truncation",
|
||||
"requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"]
|
||||
},
|
||||
{
|
||||
"gate": "G07",
|
||||
"task": "W12/W13/W14",
|
||||
"lane": "mrf-responsibility",
|
||||
"status": "pending",
|
||||
"description": "Durable MRF responsibility at every commit boundary",
|
||||
"requires": ["MRF responsibility oracle", "commit-boundary crash matrix"]
|
||||
},
|
||||
{
|
||||
"gate": "G08",
|
||||
"task": "W12/W13/W14",
|
||||
"lane": "mrf-responsibility",
|
||||
"status": "pending",
|
||||
"description": "MRF capacity, disk-full and replica-loss matrix",
|
||||
"requires": ["MRF capacity evidence", "disk-full matrix", "replica-loss matrix"]
|
||||
},
|
||||
{
|
||||
"gate": "G09",
|
||||
"task": "W13/W18/W23",
|
||||
"lane": "mixed-version-rollback",
|
||||
"status": "pending",
|
||||
"description": "Actual mixed-version reader/writer and rollback payloads",
|
||||
"requires": ["mixed-version reader evidence", "mixed-version writer evidence", "rollback payload evidence"],
|
||||
"evidence_fields": [
|
||||
"mixed_version_reader_evidence",
|
||||
"mixed_version_writer_evidence",
|
||||
"rollback_payload_evidence"
|
||||
]
|
||||
},
|
||||
{
|
||||
"gate": "G10",
|
||||
"task": "W05/W09/W10/W11",
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Bounded scheduling and pressure recovery",
|
||||
"requires": ["scheduler bound evidence", "pressure recovery evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G11",
|
||||
"task": "W04/W19/W24",
|
||||
"lane": "maintenance-producers",
|
||||
"status": "pending",
|
||||
"description": "Maintenance and complete producer coverage",
|
||||
"requires": ["maintenance producer matrix", "complete producer inventory"]
|
||||
},
|
||||
{
|
||||
"gate": "G12",
|
||||
"task": "W02/W15/W16",
|
||||
"lane": "authority-coverage",
|
||||
"status": "pending",
|
||||
"description": "Both quota paths during reset and settlement",
|
||||
"requires": ["reset quota-path evidence", "settlement quota-path evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G13",
|
||||
"task": "W07/W14",
|
||||
"lane": "maintenance-producers",
|
||||
"status": "pending",
|
||||
"description": "Quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||
"requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G14",
|
||||
"task": "W20/W21",
|
||||
"lane": "ec8-4-multiset",
|
||||
"status": "pending",
|
||||
"description": "Same-window field evidence with 3x4 EC8+4 and multi-set/pool coverage",
|
||||
"requires": ["same-window field evidence", "3x4 EC8+4 evidence", "multi-set evidence", "multi-pool evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "P1",
|
||||
"task": "W20",
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Measured cold-walk share and foreground latency/throughput",
|
||||
"requires": ["cold-walk share measurement", "foreground latency/throughput measurement"]
|
||||
},
|
||||
{
|
||||
"gate": "P2",
|
||||
"task": "W20/W24",
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Measured post-stop convergence and cold segment reuse",
|
||||
"requires": ["post-stop convergence measurement", "cold segment reuse measurement"]
|
||||
},
|
||||
{
|
||||
"gate": "P3",
|
||||
"task": "W20",
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Measured two-hour pressure/heal capacity and recovery window",
|
||||
"requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"]
|
||||
},
|
||||
{
|
||||
"gate": "P4",
|
||||
"task": "W20",
|
||||
"lane": "mrf-responsibility",
|
||||
"status": "pending",
|
||||
"description": "Measured MRF scale and replay cost with retained responsibility",
|
||||
"requires": ["MRF scale measurement", "MRF replay-cost measurement", "retained responsibility evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "R-E",
|
||||
"task": "W03/W05",
|
||||
"lane": "checkpoint-and-crash",
|
||||
"status": "pending",
|
||||
"description": "Fixed-budget real process restart through enumeration and classification",
|
||||
"requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "R-D",
|
||||
"task": "W07/W14",
|
||||
"lane": "status-and-outcome",
|
||||
"status": "pending",
|
||||
"description": "Manager-to-event-to-ledger exact disposition, including grace",
|
||||
"requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"]
|
||||
},
|
||||
{
|
||||
"gate": "R-L",
|
||||
"task": "W13/W14",
|
||||
"lane": "mixed-version-rollback",
|
||||
"status": "pending",
|
||||
"description": "Legacy source conflicts, migration gaps and crash-safe source retirement",
|
||||
"requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Security Audit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
@@ -32,7 +32,7 @@ on:
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
|
||||
@@ -22,7 +22,7 @@ name: Continuous Integration (docs only)
|
||||
on:
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened ]
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Continuous Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
@@ -36,7 +36,7 @@ on:
|
||||
- "flake.lock"
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
# Keep this list in sync with the `paths` list in ci-docs-only.yml, which
|
||||
# reports the required "Test and Lint" check for PRs skipped here.
|
||||
paths-ignore:
|
||||
@@ -872,13 +872,14 @@ jobs:
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||
# every PR, so it is gated to main/release pushes, the merge queue, and manual
|
||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
(github.event_name == 'push' &&
|
||||
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release'))
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 55
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
# Upgrade cases use an independent 1.0.0-rc.2 pin defined below.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
# independent `statfs` capacity. This job runs on GitHub-hosted
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
HTTP_PROXY: ""
|
||||
HTTPS_PROXY: ""
|
||||
# Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
|
||||
# Independent 1.0.0-rc.2 source pin for distributed::upgrade_test.
|
||||
UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
|
||||
UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
|
||||
UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
|
||||
|
||||
@@ -145,6 +145,38 @@ pub fn consume_verified_mrf_repair_events(anchors: &mut Vec<MrfDurableRepairAnch
|
||||
before.saturating_sub(anchors.len())
|
||||
}
|
||||
|
||||
/// Consume recorded verified repairs for one bucket without draining
|
||||
/// unrelated or still-unmatched proofs. If the caller crashes before
|
||||
/// persisting the retained anchor set, the proof may be replayed by repair
|
||||
/// instead of silently deleting the old responsibility.
|
||||
pub fn consume_recorded_verified_mrf_repair_events_for(bucket: &str, anchors: &mut Vec<MrfDurableRepairAnchor>) -> usize {
|
||||
let Some(registry) = MRF_VERIFIED_REPAIR_EVENTS.get() else {
|
||||
return 0;
|
||||
};
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
return 0;
|
||||
};
|
||||
let before = anchors.len();
|
||||
let mut retained = std::collections::VecDeque::with_capacity(events.len());
|
||||
while let Some(event) = events.pop_front() {
|
||||
if event.bucket.as_ref() != bucket {
|
||||
retained.push_back(event);
|
||||
continue;
|
||||
}
|
||||
let mut matched = false;
|
||||
anchors.retain(|anchor| {
|
||||
let proven = anchor.is_proven_by(&event);
|
||||
matched |= proven;
|
||||
!proven
|
||||
});
|
||||
if !matched {
|
||||
retained.push_back(event);
|
||||
}
|
||||
}
|
||||
*events = retained;
|
||||
before.saturating_sub(anchors.len())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct MrfScope {
|
||||
pub pool_index: u32,
|
||||
@@ -821,6 +853,71 @@ mod tests {
|
||||
assert!(retained.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_verified_repair_consumer_retains_unmatched_proofs() {
|
||||
let bucket = Arc::<str>::from(format!("recorded-proof-{}", Uuid::new_v4()));
|
||||
let other_bucket = Arc::<str>::from(format!("recorded-proof-other-{}", Uuid::new_v4()));
|
||||
let incarnation = Uuid::new_v4();
|
||||
let lease = MrfIngressLease::new(21);
|
||||
let retained_anchor = MrfDurableRepairAnchor {
|
||||
kind: MrfKind::PartialWrite,
|
||||
bucket: bucket.clone(),
|
||||
object: Arc::from("retained"),
|
||||
version_id: Some([7; 16]),
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 1,
|
||||
set_index: 2,
|
||||
}),
|
||||
lease,
|
||||
bucket_incarnation_id: incarnation,
|
||||
};
|
||||
let waiting_anchor = MrfDurableRepairAnchor {
|
||||
object: Arc::from("waiting"),
|
||||
lease: MrfIngressLease::new(22),
|
||||
..retained_anchor.clone()
|
||||
};
|
||||
let matched_event = MrfVerifiedRepairEvent {
|
||||
kind: retained_anchor.kind,
|
||||
bucket: bucket.clone(),
|
||||
object: retained_anchor.object.clone(),
|
||||
version_id: retained_anchor.version_id,
|
||||
scope: retained_anchor.scope,
|
||||
lease: Some(retained_anchor.lease),
|
||||
bucket_incarnation_id: retained_anchor.bucket_incarnation_id,
|
||||
disposition: MrfVerifiedRepairDisposition::Repaired,
|
||||
};
|
||||
let same_bucket_unmatched = MrfVerifiedRepairEvent {
|
||||
object: Arc::from("future"),
|
||||
lease: Some(MrfIngressLease::new(23)),
|
||||
..matched_event.clone()
|
||||
};
|
||||
let other_bucket_event = MrfVerifiedRepairEvent {
|
||||
bucket: other_bucket.clone(),
|
||||
..matched_event.clone()
|
||||
};
|
||||
|
||||
note_mrf_verified_repair(matched_event);
|
||||
note_mrf_verified_repair(same_bucket_unmatched.clone());
|
||||
note_mrf_verified_repair(other_bucket_event.clone());
|
||||
|
||||
let mut anchors = vec![retained_anchor, waiting_anchor.clone()];
|
||||
assert_eq!(consume_recorded_verified_mrf_repair_events_for(&bucket, &mut anchors), 1);
|
||||
assert_eq!(anchors, vec![waiting_anchor]);
|
||||
|
||||
let remaining_bucket_events = take_mrf_verified_repair_events_for(&bucket);
|
||||
assert_eq!(
|
||||
remaining_bucket_events,
|
||||
vec![same_bucket_unmatched],
|
||||
"same-bucket proofs without a retained anchor must remain available"
|
||||
);
|
||||
let remaining_other_events = take_mrf_verified_repair_events_for(&other_bucket);
|
||||
assert_eq!(
|
||||
remaining_other_events,
|
||||
vec![other_bucket_event],
|
||||
"proofs for other buckets must not be drained by this consumer"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_send_delivers_and_respects_capacity() {
|
||||
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
||||
|
||||
@@ -44,6 +44,7 @@ use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::error::Error;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::time::{Duration, Instant, sleep};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
use walkdir::WalkDir;
|
||||
@@ -374,6 +375,33 @@ pub(crate) fn census_object_version_on_disk(
|
||||
})
|
||||
}
|
||||
|
||||
/// Wait for the background PUT tail to commit every physical part on one disk.
|
||||
/// Invalid metadata remains an immediate error instead of a retryable absence.
|
||||
pub(crate) async fn wait_for_complete_physical_shard_on_disk(
|
||||
disk: &Path,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
timeout: Duration,
|
||||
) -> ChaosResult<VersionShardCensus> {
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
let census = census_object_version_on_disk(disk, bucket, key, version_id)?;
|
||||
if census.is_complete() && !census.expected_part_numbers.is_empty() {
|
||||
return Ok(census);
|
||||
}
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return Err(format!(
|
||||
"physical shard for {bucket}/{key}@{version_id:?} on {} did not become complete within {timeout:?}: {census:?}",
|
||||
disk.display()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(remaining.min(Duration::from_millis(50))).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST` a signed (SigV4, service `s3`) admin request without relying on the
|
||||
/// external `awscurl` binary. Mirrors the admin heal calls used by the heal
|
||||
/// regression suite.
|
||||
@@ -451,4 +479,30 @@ mod tests {
|
||||
assert!(expected.matches_manifest(&expected));
|
||||
assert!(!changed.matches_manifest(&expected));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn physical_shard_readiness_fails_closed_with_last_census() {
|
||||
let disk = tempfile::tempdir().expect("temporary disk");
|
||||
let error = wait_for_complete_physical_shard_on_disk(disk.path(), "bucket", "missing", None, Duration::ZERO)
|
||||
.await
|
||||
.expect_err("missing physical shards must fail the baseline gate");
|
||||
assert!(error.to_string().contains("has_xl_meta: false"));
|
||||
assert!(error.to_string().contains("bucket/missing"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn physical_shard_readiness_does_not_retry_invalid_metadata() {
|
||||
let disk = tempfile::tempdir().expect("temporary disk");
|
||||
let object = disk.path().join("bucket").join("corrupt");
|
||||
std::fs::create_dir_all(&object).expect("object directory");
|
||||
std::fs::write(object.join("xl.meta"), b"invalid metadata").expect("corrupt metadata fixture");
|
||||
let error = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
wait_for_complete_physical_shard_on_disk(disk.path(), "bucket", "corrupt", None, Duration::from_secs(30)),
|
||||
)
|
||||
.await
|
||||
.expect("corrupt metadata must fail immediately")
|
||||
.expect_err("invalid metadata must not be accepted as a complete baseline");
|
||||
assert!(!error.to_string().contains("did not become complete"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +59,9 @@ const POOL_META_V3_ENV: [(&str, &str); 2] = [
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) enum DistLayout {
|
||||
/// 3 nodes × 4 drives, one erasure pool. With `EC:4` this is the
|
||||
/// release-evidence EC8+4 geometry.
|
||||
ThreeByFourEc84,
|
||||
/// 4 nodes × 4 drives, one erasure pool spanning every endpoint.
|
||||
FourByFour,
|
||||
/// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout).
|
||||
@@ -94,6 +97,7 @@ impl DistCluster {
|
||||
|
||||
pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
|
||||
let topology = match layout {
|
||||
DistLayout::ThreeByFourEc84 => ClusterTopology::single_pool_multidrive(3, DRIVES_PER_NODE),
|
||||
DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE),
|
||||
DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT),
|
||||
DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]),
|
||||
@@ -101,7 +105,7 @@ impl DistCluster {
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?;
|
||||
let pool_storage_roots = match layout {
|
||||
DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?),
|
||||
DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
|
||||
DistLayout::ThreeByFourEc84 | DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
|
||||
};
|
||||
let mut owned_pool_dirs = Vec::new();
|
||||
if let Some(roots) = pool_storage_roots.as_deref() {
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_inventory, payload_for, put_object, unique_bucket, wait_until};
|
||||
use crate::chaos::{
|
||||
VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post, wait_for_complete_physical_shard_on_disk,
|
||||
};
|
||||
use crate::common::{init_logging, rustfs_binary_path};
|
||||
use crate::scanner_heal_evidence::{EvidenceTopology, RestartObservation, ScannerHealEvidenceCase, restart_evidence_run};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
const EC84_NODE_COUNT: usize = 3;
|
||||
const EC84_DRIVES_PER_NODE: usize = 4;
|
||||
const EC84_DATA_BLOCKS: usize = 8;
|
||||
const EC84_PARITY_BLOCKS: usize = 4;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ExpectedShard {
|
||||
key: String,
|
||||
body: Vec<u8>,
|
||||
baseline: VersionShardCensus,
|
||||
}
|
||||
|
||||
fn assert_ec84_geometry(census: &VersionShardCensus, key: &str) -> TestResult {
|
||||
if census.data_blocks != Some(EC84_DATA_BLOCKS) || census.parity_blocks != Some(EC84_PARITY_BLOCKS) {
|
||||
return Err(format!("object {key} did not use EC8+4 geometry: {census:?}").into());
|
||||
}
|
||||
let erasure_index = census
|
||||
.erasure_index
|
||||
.ok_or_else(|| format!("object {key} did not record an erasure index: {census:?}"))?;
|
||||
if !(1..=EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS).contains(&erasure_index) {
|
||||
return Err(format!("object {key} has out-of-range erasure index {erasure_index}: {census:?}").into());
|
||||
}
|
||||
if !census.is_complete() || census.expected_part_numbers.is_empty() {
|
||||
return Err(format!("object {key} does not have complete physical shard evidence: {census:?}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> TestResult {
|
||||
for key in keys {
|
||||
let census = census_object_version_on_disk(drive, bucket, key, None)?;
|
||||
if census.has_xl_meta {
|
||||
return Err(format!("replacement drive unexpectedly retained {bucket}/{key}: {census:?}").into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
|
||||
let mut expected = Vec::new();
|
||||
for index in 0..4 {
|
||||
let key = format!("ec84/prefix-{}/object-{index:04}.bin", index % 2);
|
||||
let body = payload_for(&key, 10 * 1024 * 1024);
|
||||
put_object(client, bucket, &key, body.clone()).await?;
|
||||
expected.push(ExpectedShard {
|
||||
key,
|
||||
body,
|
||||
baseline: VersionShardCensus {
|
||||
version_id: None,
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: Default::default(),
|
||||
present_part_fingerprints: Default::default(),
|
||||
inline_data_fingerprint: None,
|
||||
},
|
||||
});
|
||||
}
|
||||
Ok(expected)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart() -> TestResult {
|
||||
init_logging();
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = restart_evidence_run(
|
||||
&server_binary,
|
||||
ScannerHealEvidenceCase {
|
||||
id: "ec84-target-drive-restart",
|
||||
oracle: "ec84-target-drive-restart.json",
|
||||
evidence: "process-restart",
|
||||
unclean_shutdown_marker: false,
|
||||
topology: EvidenceTopology::new(3, 4),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some("12"),
|
||||
},
|
||||
)?;
|
||||
let mut dist = DistCluster::start_with_env(
|
||||
DistLayout::ThreeByFourEc84,
|
||||
&[
|
||||
("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
|
||||
("RUSTFS_HEAL_ENABLED", "true"),
|
||||
("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"),
|
||||
("RUSTFS_HEAL_MRF_ENABLE", "false"),
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(dist.cluster.nodes.len(), EC84_NODE_COUNT);
|
||||
assert_eq!(dist.cluster.topology.drives_per_node, EC84_DRIVES_PER_NODE);
|
||||
|
||||
let bucket = unique_bucket("healec84");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let mut expected = put_large_inventory(&writer, &bucket).await?;
|
||||
let replaced_node = 1;
|
||||
let replaced_drive_index = 2;
|
||||
let replaced_drive = PathBuf::from(&dist.cluster.nodes[replaced_node].data_dirs[replaced_drive_index]);
|
||||
|
||||
for item in &mut expected {
|
||||
item.baseline =
|
||||
wait_for_complete_physical_shard_on_disk(&replaced_drive, &bucket, &item.key, None, Duration::from_secs(10)).await?;
|
||||
assert_ec84_geometry(&item.baseline, &item.key)?;
|
||||
}
|
||||
|
||||
let format_path = replaced_drive.join(".rustfs.sys").join("format.json");
|
||||
let format_json = std::fs::read(&format_path)?;
|
||||
let pid_before = dist.cluster.nodes[replaced_node]
|
||||
.process
|
||||
.as_ref()
|
||||
.ok_or("target process is absent")?
|
||||
.id();
|
||||
dist.cluster.stop_node_gracefully(replaced_node).await?;
|
||||
let unclean_shutdown_marker = Path::new(&dist.cluster.nodes[replaced_node].data_dir)
|
||||
.join(".rustfs.sys")
|
||||
.join("unclean-shutdown")
|
||||
.is_file();
|
||||
assert!(!unclean_shutdown_marker, "graceful target shutdown must remove its unclean marker");
|
||||
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
|
||||
std::fs::rename(&replaced_drive, &retired_drive)?;
|
||||
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path has no parent")?)?;
|
||||
std::fs::write(&format_path, format_json)?;
|
||||
assert_replaced_drive_empty(
|
||||
&replaced_drive,
|
||||
&bucket,
|
||||
&expected.iter().map(|item| item.key.clone()).collect::<Vec<_>>(),
|
||||
)?;
|
||||
|
||||
let outage_key = "ec84/written-while-node-restarting.bin";
|
||||
let outage_body = payload_for(outage_key, 10 * 1024 * 1024);
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(outage_key)
|
||||
.body(ByteStream::from(outage_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
dist.cluster.start_node(replaced_node).await?;
|
||||
let heal_body =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
|
||||
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
|
||||
wait_until(
|
||||
Duration::from_secs(120),
|
||||
|| async {
|
||||
for item in &expected {
|
||||
let current = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
|
||||
if !current.matches_manifest(&item.baseline) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
let outage = census_object_version_on_disk(&replaced_drive, &bucket, outage_key, None)?;
|
||||
Ok(outage.is_complete()
|
||||
&& outage.data_blocks == Some(EC84_DATA_BLOCKS)
|
||||
&& outage.parity_blocks == Some(EC84_PARITY_BLOCKS))
|
||||
},
|
||||
"EC8+4 replacement drive rebuilt baseline and outage shards",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let inventory = expected
|
||||
.iter()
|
||||
.map(|item| (item.key.clone(), item.body.clone()))
|
||||
.chain(std::iter::once((outage_key.to_string(), outage_body.clone())))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let expected_keys = inventory.keys().cloned().collect::<HashSet<_>>();
|
||||
let mut node_listings = Vec::new();
|
||||
for node_index in 0..dist.cluster.nodes.len() {
|
||||
let client = dist.client(node_index)?;
|
||||
assert_inventory(&client, &bucket, &inventory).await?;
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let observed = listing
|
||||
.contents()
|
||||
.iter()
|
||||
.filter_map(|object| object.key().map(str::to_owned))
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(observed, expected_keys, "node {node_index} listing diverged after EC8+4 heal");
|
||||
let mut keys = observed.into_iter().collect::<Vec<_>>();
|
||||
keys.sort();
|
||||
node_listings.push(keys);
|
||||
}
|
||||
|
||||
if let Some(evidence_run) = evidence_run {
|
||||
let target_client = dist.client(replaced_node)?;
|
||||
let mut objects = Vec::with_capacity(inventory.len());
|
||||
for (key, body) in &inventory {
|
||||
let response = target_client.get_object().bucket(&bucket).key(key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(actual.as_ref(), body.as_slice(), "object body changed for {key}");
|
||||
let physical = census_object_version_on_disk(&replaced_drive, &bucket, key, None)?;
|
||||
assert_ec84_geometry(&physical, key)?;
|
||||
let baseline = expected.iter().find(|item| item.key == *key).map(|item| &item.baseline);
|
||||
objects.push(serde_json::json!({
|
||||
"key": key, "version_id": null,
|
||||
"expected_bytes": body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(body), "actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": baseline, "physical": physical,
|
||||
}));
|
||||
}
|
||||
let pid_after = dist.cluster.nodes[replaced_node]
|
||||
.process
|
||||
.as_ref()
|
||||
.ok_or("restarted target is absent")?
|
||||
.id();
|
||||
evidence_run.write(
|
||||
&server_binary,
|
||||
RestartObservation {
|
||||
nodes: dist.cluster.nodes.len(),
|
||||
drives_per_node: dist.cluster.topology.drives_per_node,
|
||||
pid_before,
|
||||
pid_after,
|
||||
unclean_shutdown_marker,
|
||||
objects,
|
||||
node_listings,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -25,6 +25,7 @@ mod data_integrity_movement_test;
|
||||
mod expand_decommission_rebalance_test;
|
||||
mod extra_test;
|
||||
mod harness;
|
||||
mod heal_test;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod replication_quota_test;
|
||||
|
||||
@@ -16,18 +16,20 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
|
||||
use crate::chaos::{
|
||||
VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post,
|
||||
wait_for_complete_physical_shard_on_disk,
|
||||
};
|
||||
use crate::common::{
|
||||
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
|
||||
rustfs_binary_path,
|
||||
};
|
||||
use crate::scanner_heal_evidence::{EvidenceTopology, RestartObservation, ScannerHealEvidenceCase, restart_evidence_run};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -39,34 +41,14 @@ mod tests {
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RestartEvidenceRun {
|
||||
schema: u32,
|
||||
run_id: String,
|
||||
source_revision: String,
|
||||
test_build: serde_json::Value,
|
||||
binary: EvidenceBuild,
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ScannerHealEvidenceCase {
|
||||
id: &'static str,
|
||||
oracle: &'static str,
|
||||
evidence: &'static str,
|
||||
unclean_shutdown_marker: bool,
|
||||
}
|
||||
|
||||
const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-restart",
|
||||
oracle: "background-target-restart.json",
|
||||
evidence: "process-restart",
|
||||
unclean_shutdown_marker: false,
|
||||
topology: EvidenceTopology::new(4, 1),
|
||||
storage_class_standard: None,
|
||||
erasure_set_drive_count: None,
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
@@ -74,82 +56,30 @@ mod tests {
|
||||
oracle: "background-target-crash.json",
|
||||
evidence: "process-crash-restart",
|
||||
unclean_shutdown_marker: true,
|
||||
topology: EvidenceTopology::new(4, 1),
|
||||
storage_class_standard: None,
|
||||
erasure_set_drive_count: None,
|
||||
};
|
||||
|
||||
struct RestartEvidenceContext {
|
||||
directory: PathBuf,
|
||||
run: RestartEvidenceRun,
|
||||
case: ScannerHealEvidenceCase,
|
||||
}
|
||||
const BACKGROUND_TARGET_RESTART_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-restart-ec8-4",
|
||||
oracle: "background-target-restart-ec8-4.json",
|
||||
evidence: "process-restart",
|
||||
unclean_shutdown_marker: false,
|
||||
topology: EvidenceTopology::new(3, 4),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some("12"),
|
||||
};
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..read]);
|
||||
}
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
fn restart_evidence_run(
|
||||
binary: &Path,
|
||||
case: ScannerHealEvidenceCase,
|
||||
) -> Result<Option<RestartEvidenceContext>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
if case.id.is_empty()
|
||||
|| case.oracle.is_empty()
|
||||
|| !case.oracle.ends_with(".json")
|
||||
|| case.oracle.contains('/')
|
||||
|| case.oracle.contains('\\')
|
||||
|| case.oracle.contains("..")
|
||||
|| !matches!(case.evidence, "process-restart" | "process-crash-restart")
|
||||
|| (case.evidence == "process-crash-restart") != case.unclean_shutdown_marker
|
||||
{
|
||||
return Err("invalid scanner/heal evidence case".into());
|
||||
}
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
return Err("oversized scanner/heal execution receipt".into());
|
||||
}
|
||||
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
|
||||
return Err("invalid scanner/heal execution identity".into());
|
||||
}
|
||||
let built = compiled_test_identity();
|
||||
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
|
||||
}
|
||||
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
|
||||
assert_eq!(
|
||||
file_sha256(&std::env::current_exe()?)?,
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join(case.oracle).exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some(RestartEvidenceContext { directory, run, case }))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||
})
|
||||
}
|
||||
const BACKGROUND_TARGET_CRASH_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-crash-ec8-4",
|
||||
oracle: "background-target-crash-ec8-4.json",
|
||||
evidence: "process-crash-restart",
|
||||
unclean_shutdown_marker: true,
|
||||
topology: EvidenceTopology::new(3, 4),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some("12"),
|
||||
};
|
||||
|
||||
struct TcpPortBlackhole {
|
||||
port: u16,
|
||||
@@ -332,11 +262,10 @@ mod tests {
|
||||
|
||||
// Healing may rewrite non-identity bookkeeping in xl.meta. The census
|
||||
// therefore compares the canonical selected metadata fields plus every
|
||||
// physical shard, while the payload seed makes object mix-ups observable.
|
||||
// physical shard.
|
||||
#[derive(Debug)]
|
||||
struct PhysicalObjectManifest {
|
||||
key: String,
|
||||
payload_seed: u8,
|
||||
shard_census: VersionShardCensus,
|
||||
}
|
||||
|
||||
@@ -892,7 +821,10 @@ mod tests {
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
previous_cycle_end: u64,
|
||||
) -> Result<u64, Box<dyn Error + Send + Sync>> {
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
let started = Instant::now();
|
||||
let mut deadline = started + Duration::from_secs(60);
|
||||
let catch_up_deadline = deadline + Duration::from_secs(300);
|
||||
let mut catch_up_wait_observed = false;
|
||||
loop {
|
||||
let mut latest_cycle_end = 0;
|
||||
let mut versions_observed = false;
|
||||
@@ -920,24 +852,61 @@ mod tests {
|
||||
let versions_scanned = metrics["versions_scanned"]
|
||||
.as_u64()
|
||||
.ok_or("scanner status is missing its version-coverage counter")?;
|
||||
latest_cycle_end = latest_cycle_end.max(cycle_end);
|
||||
let cycle_result = metrics["last_cycle_result"]
|
||||
.as_str()
|
||||
.ok_or("scanner status is missing its cycle result")?;
|
||||
if cycle_result == "success" {
|
||||
latest_cycle_end = latest_cycle_end.max(cycle_end);
|
||||
}
|
||||
versions_observed |= versions_scanned > 0;
|
||||
let backlog = &status["pause_backlog"];
|
||||
if !catch_up_wait_observed
|
||||
&& backlog["persistence_state"].as_str() == Some("healthy")
|
||||
&& backlog["durable"].as_bool() == Some(true)
|
||||
&& backlog["phase"].as_str() == Some("catching_up")
|
||||
&& backlog["rate_limited"].as_bool() == Some(true)
|
||||
&& backlog["retry_exhausted"].as_bool() == Some(false)
|
||||
{
|
||||
let next_attempt = backlog["next_attempt_at_unix_secs"]
|
||||
.as_u64()
|
||||
.ok_or("rate-limited scanner backlog is missing its next attempt")?;
|
||||
let interval = backlog["thresholds"]["catch_up_min_interval_seconds"]
|
||||
.as_u64()
|
||||
.ok_or("rate-limited scanner backlog is missing its catch-up interval")?;
|
||||
let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_secs();
|
||||
let remaining = next_attempt.saturating_sub(now);
|
||||
if remaining > 0 {
|
||||
if interval > 300 || remaining > interval {
|
||||
return Err(
|
||||
format!("scanner catch-up schedule exceeds the bounded recovery budget: {backlog}").into()
|
||||
);
|
||||
}
|
||||
// The durable catch-up interval overrides SCANNER_CYCLE=1.
|
||||
// Honor one observed retry without restarting the deadline on every poll.
|
||||
deadline = deadline
|
||||
.max(Instant::now() + Duration::from_secs(remaining + 60))
|
||||
.min(catch_up_deadline);
|
||||
catch_up_wait_observed = true;
|
||||
}
|
||||
}
|
||||
observations.push(format!(
|
||||
"node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}",
|
||||
"node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}, backlog={}",
|
||||
metrics["current_cycle"],
|
||||
metrics["current_cycle_active"],
|
||||
metrics["leader_lock_state"],
|
||||
metrics["last_cycle_result"],
|
||||
backlog,
|
||||
));
|
||||
}
|
||||
// The coordinator records cycle completion, but remote workers
|
||||
// record scanned versions. Both witnesses need not share a node.
|
||||
// Only a successful coordinator cycle counts as completion; deferred
|
||||
// and superseded attempts also advance its end timestamp. Remote
|
||||
// workers record version coverage, so the witnesses can span nodes.
|
||||
if latest_cycle_end > previous_cycle_end && versions_observed {
|
||||
return Ok(latest_cycle_end);
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"enabled scanner did not complete an object-scanning cycle after {previous_cycle_end}: {observations:?}"
|
||||
"enabled scanner did not complete a successful object-scanning cycle after {previous_cycle_end}: {observations:?}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
@@ -957,7 +926,7 @@ mod tests {
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_background_target_restart()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
Duration::from_secs(720),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestart),
|
||||
)
|
||||
.await?
|
||||
@@ -967,17 +936,37 @@ mod tests {
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_background_target_crash()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
Duration::from_secs(720),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrash),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(720),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(720),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrashEc84),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
Duration::from_secs(720),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundCoordinatorRestart),
|
||||
)
|
||||
.await?
|
||||
@@ -1010,29 +999,38 @@ mod tests {
|
||||
IsolatedTargetRestart,
|
||||
BackgroundTargetRestart,
|
||||
BackgroundTargetCrash,
|
||||
BackgroundTargetRestartEc84,
|
||||
BackgroundTargetCrashEc84,
|
||||
BackgroundCoordinatorRestart,
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = match scenario {
|
||||
InterruptionScenario::BackgroundTargetRestart => {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)?
|
||||
}
|
||||
InterruptionScenario::BackgroundTargetCrash => {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_CRASH_EVIDENCE)?
|
||||
}
|
||||
let evidence_case = match scenario {
|
||||
InterruptionScenario::BackgroundTargetRestart => Some(BACKGROUND_TARGET_RESTART_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetCrash => Some(BACKGROUND_TARGET_CRASH_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetRestartEc84 => Some(BACKGROUND_TARGET_RESTART_EC84_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetCrashEc84 => Some(BACKGROUND_TARGET_CRASH_EC84_EVIDENCE),
|
||||
_ => None,
|
||||
};
|
||||
let evidence_run = match evidence_case {
|
||||
Some(case) => restart_evidence_run(&server_binary, case)?,
|
||||
None => None,
|
||||
};
|
||||
let mut evidence_objects = Vec::new();
|
||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
||||
InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"),
|
||||
InterruptionScenario::BackgroundTargetRestartEc84 => (true, 1, "background_target_restart_ec8_4"),
|
||||
InterruptionScenario::BackgroundTargetCrashEc84 => (true, 1, "background_target_crash_ec8_4"),
|
||||
InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
|
||||
InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
|
||||
};
|
||||
let topology = evidence_case
|
||||
.map(|case| case.topology)
|
||||
.unwrap_or_else(|| EvidenceTopology::new(4, 1));
|
||||
init_logging();
|
||||
info!(
|
||||
event = "heal_interruption_started",
|
||||
@@ -1044,9 +1042,15 @@ mod tests {
|
||||
"Starting root-heal interruption test"
|
||||
);
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology.cluster_topology()).await?;
|
||||
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
if let Some(storage_class) = evidence_case.and_then(|case| case.storage_class_standard) {
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", storage_class);
|
||||
}
|
||||
if let Some(erasure_set_drive_count) = evidence_case.and_then(|case| case.erasure_set_drive_count) {
|
||||
cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count);
|
||||
}
|
||||
// Heal control uses the first lexicographically sorted grid host.
|
||||
// Keep that coordinator distinct from the remote target at index 1.
|
||||
cluster.nodes.sort_by(|left, right| left.url.cmp(&right.url));
|
||||
@@ -1069,10 +1073,19 @@ mod tests {
|
||||
let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG")
|
||||
.unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string());
|
||||
cluster.set_env("RUST_LOG", server_rust_log);
|
||||
let log_dir = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR").unwrap_or_else(|_| format!("{}/logs", cluster.temp_dir));
|
||||
let log_dir = if let Some(directory) = std::env::var_os("RUSTFS_HEAL_CHAOS_LOG_DIR") {
|
||||
PathBuf::from(directory)
|
||||
} else if let Some(directory) = std::env::var_os("RUSTFS_E2E_LOG_DIR") {
|
||||
let cluster_name = Path::new(&cluster.temp_dir)
|
||||
.file_name()
|
||||
.ok_or("cluster directory has no name")?;
|
||||
PathBuf::from(directory).join(cluster_name).join("heal")
|
||||
} else {
|
||||
PathBuf::from(&cluster.temp_dir).join("logs")
|
||||
};
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
cluster.set_node_capture_log_path(node_index, log_dir.join(format!("node{node_index}.log")).to_string_lossy())?;
|
||||
}
|
||||
cluster.start_with_binary(&server_binary).await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
@@ -1095,11 +1108,22 @@ mod tests {
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.unwrap_or(4 * 1024 * 1024)
|
||||
.clamp(1024 * 1024, 16 * 1024 * 1024);
|
||||
let mut created_online_objects = Vec::with_capacity(online_object_count);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
let mut unclean_shutdown_marker_observed = None;
|
||||
for index in 0..online_object_count {
|
||||
let key = format!("cluster/online/object-{index:04}.bin");
|
||||
let payload_seed = u8::try_from(index + 1).expect("clamped object count must fit in u8");
|
||||
let mut attempt_count = 0usize;
|
||||
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
|
||||
while expected_manifests.len() < online_object_count {
|
||||
if attempt_count >= max_online_attempts {
|
||||
return Err(format!(
|
||||
"target replacement drive held only {}/{} baseline object shards after {attempt_count} writes",
|
||||
expected_manifests.len(),
|
||||
online_object_count
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let key = format!("cluster/online/object-{attempt_count:04}.bin");
|
||||
let payload_seed = ((attempt_count % 251) + 1) as u8;
|
||||
timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[0]
|
||||
@@ -1111,6 +1135,13 @@ mod tests {
|
||||
)
|
||||
.await??;
|
||||
let shard_census = census_object_version_on_disk(&replaced_disk, bucket, &key, None)?;
|
||||
if !shard_census.has_xl_meta {
|
||||
timeout(Duration::from_secs(30), clients[0].delete_object().bucket(bucket).key(&key).send()).await??;
|
||||
attempt_count += 1;
|
||||
continue;
|
||||
}
|
||||
let shard_census =
|
||||
wait_for_complete_physical_shard_on_disk(&replaced_disk, bucket, &key, None, Duration::from_secs(10)).await?;
|
||||
assert!(
|
||||
shard_census.is_complete(),
|
||||
"node 1 should hold a complete baseline shard for {key}: {shard_census:?}"
|
||||
@@ -1119,11 +1150,9 @@ mod tests {
|
||||
!shard_census.expected_part_numbers.is_empty(),
|
||||
"chaos objects must use physical part shards rather than inline data: {shard_census:?}"
|
||||
);
|
||||
expected_manifests.push(PhysicalObjectManifest {
|
||||
key,
|
||||
payload_seed,
|
||||
shard_census,
|
||||
});
|
||||
created_online_objects.push((key.clone(), payload_seed));
|
||||
expected_manifests.push(PhysicalObjectManifest { key, shard_census });
|
||||
attempt_count += 1;
|
||||
}
|
||||
|
||||
let expected_pool_metadata = if background_enabled {
|
||||
@@ -1169,31 +1198,38 @@ mod tests {
|
||||
if node_index == 1 {
|
||||
continue;
|
||||
}
|
||||
let census = census_object_version_on_disk(Path::new(&node.data_dir), bucket, outage_key, None)?;
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census
|
||||
.erasure_index
|
||||
.ok_or_else(|| format!("online node {node_index} outage-object shard has no erasure index: {census:?}"))?;
|
||||
assert!(
|
||||
(1..=cluster.nodes.len()).contains(&erasure_index),
|
||||
"online node {node_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online nodes"
|
||||
);
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
let census = census_object_version_on_disk(Path::new(drive), bucket, outage_key, None)?;
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census.erasure_index.ok_or_else(|| {
|
||||
format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
|
||||
})?;
|
||||
assert!(
|
||||
(1..=topology.total_drives()).contains(&erasure_index),
|
||||
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online drives"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
outage_peer_erasure_indices.len(),
|
||||
cluster.nodes.len().saturating_sub(1),
|
||||
"every online node must contribute one unique outage-object erasure index"
|
||||
topology.total_drives().saturating_sub(cluster.nodes[1].data_dirs.len()),
|
||||
"every online drive must contribute one unique outage-object erasure index"
|
||||
);
|
||||
let missing_outage_erasure_indices = (1..=topology.total_drives())
|
||||
.filter(|index| !outage_peer_erasure_indices.contains(index))
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
missing_outage_erasure_indices.len(),
|
||||
cluster.nodes[1].data_dirs.len(),
|
||||
"the stopped node must account for every missing outage-object erasure index"
|
||||
);
|
||||
let expected_outage_target_erasure_index = (1..=cluster.nodes.len())
|
||||
.find(|index| !outage_peer_erasure_indices.contains(index))
|
||||
.ok_or("online outage-object shards leave no erasure index for the replacement target")?;
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
if !background_enabled {
|
||||
@@ -1326,7 +1362,7 @@ mod tests {
|
||||
let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body)
|
||||
.map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?;
|
||||
let pre_interrupt_replacement = replacement_recovery_status(&cluster).await?;
|
||||
let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?;
|
||||
let coordinator_log = std::fs::read_to_string(log_dir.join("node0.log"))?;
|
||||
assert!(
|
||||
coordinator_log
|
||||
.lines()
|
||||
@@ -1463,7 +1499,11 @@ mod tests {
|
||||
"Restored target endpoint forwarding"
|
||||
);
|
||||
} else {
|
||||
if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
let graceful_restart = matches!(
|
||||
scenario,
|
||||
InterruptionScenario::BackgroundTargetRestart | InterruptionScenario::BackgroundTargetRestartEc84
|
||||
);
|
||||
if graceful_restart {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
@@ -1485,7 +1525,7 @@ mod tests {
|
||||
if background_enabled {
|
||||
let marker_exists = unclean_shutdown_marker.is_file();
|
||||
unclean_shutdown_marker_observed = Some(marker_exists);
|
||||
let expected_marker = !matches!(scenario, InterruptionScenario::BackgroundTargetRestart);
|
||||
let expected_marker = !graceful_restart;
|
||||
assert!(
|
||||
marker_exists == expected_marker,
|
||||
"background restart/crash lane observed unexpected unclean-shutdown marker state"
|
||||
@@ -1586,29 +1626,35 @@ mod tests {
|
||||
"outage object must have a complete target shard: {outage_census:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
outage_census
|
||||
.erasure_index
|
||||
.filter(|index| missing_outage_erasure_indices.contains(index)),
|
||||
outage_census.erasure_index,
|
||||
Some(expected_outage_target_erasure_index),
|
||||
"the outage object must be rebuilt into its own missing erasure slot"
|
||||
"the outage object must be rebuilt into one of the stopped node's missing erasure slots"
|
||||
);
|
||||
|
||||
if let Some(cycle_end) = scanner_cycle_floor {
|
||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||
}
|
||||
|
||||
let mut expected_keys = expected_manifests
|
||||
let mut expected_keys = created_online_objects
|
||||
.iter()
|
||||
.map(|manifest| manifest.key.clone())
|
||||
.map(|(key, _)| key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for expected in &expected_manifests {
|
||||
let response = target_client.get_object().bucket(bucket).key(&expected.key).send().await?;
|
||||
for (key, payload_seed) in &created_online_objects {
|
||||
let response = target_client.get_object().bucket(bucket).key(key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
||||
if evidence_run.is_some() {
|
||||
let expected_body = deterministic_object_body(object_size_bytes, *payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {key}");
|
||||
if evidence_run.is_some()
|
||||
&& let Some(expected) = expected_manifests
|
||||
.iter()
|
||||
.find(|manifest| manifest.key.as_str() == key.as_str())
|
||||
{
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": expected.key, "version_id": expected.shard_census.version_id,
|
||||
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
|
||||
@@ -1682,32 +1728,18 @@ mod tests {
|
||||
if let Some(evidence_context) = evidence_run {
|
||||
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
|
||||
assert_ne!(target_pid, restarted_pid, "target must be a new process");
|
||||
assert_eq!(
|
||||
file_sha256(&server_binary)?,
|
||||
evidence_context.run.binary.sha256,
|
||||
"server build changed during restart"
|
||||
);
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": evidence_context.case.id, "evidence": evidence_context.case.evidence,
|
||||
"run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": evidence_context.run.binary.sha256,
|
||||
"test_binary_sha256": evidence_context.run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
}
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(evidence_context.directory.join(evidence_context.case.oracle))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
evidence_context.write(
|
||||
&server_binary,
|
||||
RestartObservation {
|
||||
nodes: cluster.nodes.len(),
|
||||
drives_per_node: cluster.nodes[0].data_dirs.len(),
|
||||
pid_before: target_pid,
|
||||
pid_after: restarted_pid,
|
||||
unclean_shutdown_marker: unclean_shutdown_marker_observed.ok_or("missing shutdown marker observation")?,
|
||||
objects: evidence_objects,
|
||||
node_listings,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
//! One S3 GET can select readers on multiple EC nodes, so the counter tracks
|
||||
//! distributed reader selection rather than HTTP request count.
|
||||
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, signal_process};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
@@ -2207,6 +2207,33 @@ async fn four_node_manual_transition_job_status_survives_node_restart() -> TestR
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct SuspendedTransitionTarget<'a> {
|
||||
// Keep the owned child borrowed until it is resumed so its PID cannot be reused.
|
||||
child: &'a std::process::Child,
|
||||
suspended: bool,
|
||||
}
|
||||
|
||||
impl<'a> SuspendedTransitionTarget<'a> {
|
||||
fn suspend(child: &'a std::process::Child) -> TestResult<Self> {
|
||||
signal_process(child.id(), "STOP")?;
|
||||
Ok(Self { child, suspended: true })
|
||||
}
|
||||
|
||||
fn resume(&mut self) -> TestResult {
|
||||
signal_process(self.child.id(), "CONT")?;
|
||||
self.suspended = false;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SuspendedTransitionTarget<'_> {
|
||||
fn drop(&mut self) {
|
||||
if self.suspended {
|
||||
let _ = signal_process(self.child.id(), "CONT");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_manual_transition_distributed_admission_conflict_reports_status_and_backpressure() -> TestResult {
|
||||
init_logging();
|
||||
@@ -2234,7 +2261,6 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
let bucket = format!("distributed-admission-{}", Uuid::new_v4().simple());
|
||||
let prefix = "transition/distributed-admission/";
|
||||
hot_client.create_bucket().bucket(&bucket).send().await?;
|
||||
put_lifecycle_with_transition_retry(&hot_client, &bucket, &tier_name).await?;
|
||||
for index in 0u8..64 {
|
||||
let key = format!("{prefix}object-{index:02}.bin");
|
||||
hot_client
|
||||
@@ -2245,6 +2271,21 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
// Lifecycle PUT starts its own backfill. Keep its first page on a separate
|
||||
// node and stop it at queue backpressure before it reaches the tested prefix:
|
||||
// one active worker, one queued item, then the first rejected item.
|
||||
for index in 0u8..3 {
|
||||
hot_client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(format!("transition/automatic-admission/object-{index:02}.bin"))
|
||||
.body(ByteStream::from(payload(KIB, index)))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
let mut suspended_cold = SuspendedTransitionTarget::suspend(cold.process.as_ref().ok_or("cold-tier process missing")?)?;
|
||||
let lifecycle_client = hot.create_s3_client(2)?;
|
||||
put_lifecycle_with_transition_retry(&lifecycle_client, &bucket, &tier_name).await?;
|
||||
|
||||
let (node0, node1) = tokio::join!(
|
||||
start_manual_transition_job_on_node(&hot, 0, &bucket, prefix, &tier_name, false, 64),
|
||||
@@ -2304,6 +2345,31 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
assert_eq!(status["job_id"].as_str(), Some(job_id));
|
||||
assert_eq!(status["status_endpoint"].as_str(), Some(status_endpoint));
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let status = read_manual_transition_job_status_endpoint(&hot, accepted.0, status_endpoint).await?;
|
||||
assert_eq!(
|
||||
status["status"].as_str(),
|
||||
Some("running"),
|
||||
"blocked cold tier must keep the admitted job running: {status}"
|
||||
);
|
||||
if status["report"]["skipped_queue_full"].as_u64().is_some_and(|count| count > 0) {
|
||||
assert!(
|
||||
status["report"]["enqueued"].as_u64().is_some_and(|count| count > 0),
|
||||
"the job must own pending transitions while the cold tier is suspended: {status}"
|
||||
);
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"manual transition job did not reach queue backpressure while the cold tier was suspended: {status}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
suspended_cold.resume()?;
|
||||
let terminal = wait_for_manual_transition_job_terminal(&hot, conflict.0, job_id, false).await?;
|
||||
assert_eq!(terminal["job_id"].as_str(), Some(job_id));
|
||||
assert_eq!(terminal["bucket"].as_str(), Some(bucket.as_str()));
|
||||
|
||||
@@ -59,36 +59,6 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
|
||||
|
||||
assert_eq!(put_response.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
|
||||
// A missing key in the healthy store is a client error, unlike a store outage.
|
||||
let missing_key_object = "test-missing-kms-key";
|
||||
let missing_key_error = s3_client
|
||||
.put_object()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(missing_key_object)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"must not be published"))
|
||||
.server_side_encryption(ServerSideEncryption::AwsKms)
|
||||
.ssekms_key_id("rustfs-e2e-test-missing-key")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unknown key in a healthy Local KMS store must reject the write");
|
||||
assert_eq!(missing_key_error.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
assert_eq!(
|
||||
missing_key_error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("KMS.NotFoundException")
|
||||
);
|
||||
let missing_key_absence = s3_client
|
||||
.get_object()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(missing_key_object)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a write rejected by a missing KMS key must not publish an object");
|
||||
assert_eq!(missing_key_absence.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
assert_eq!(
|
||||
missing_key_absence.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey")
|
||||
);
|
||||
|
||||
// Temporarily rename the key directory to simulate unavailability
|
||||
info!("🔧 Simulating key directory unavailability");
|
||||
let backup_dir = format!("{}.backup", kms_env.kms_keys_dir);
|
||||
|
||||
@@ -23,6 +23,9 @@ pub mod common;
|
||||
#[cfg(test)]
|
||||
pub mod chaos;
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_heal_evidence;
|
||||
|
||||
// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8)
|
||||
// and on-demand-migration source scenarios (backlog#2151).
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -79,6 +79,11 @@ pub async fn test_sftp_compliance_suite() -> Result<()> {
|
||||
.await
|
||||
.map_err(|e| anyhow!("{}", e))?;
|
||||
|
||||
// Protocol listeners can accept connections before IAM is initialized.
|
||||
// A signed S3 request establishes readiness before the first SFTP login.
|
||||
let s3 = build_test_s3_client(&format!("http://{COMPLIANCE_RW_S3_ADDRESS}"));
|
||||
wait_for_s3_ready(&s3, 30).await?;
|
||||
|
||||
let (session, sftp) = connect_sftp_to(COMPLIANCE_RW_SFTP_ADDRESS).await?;
|
||||
|
||||
cmptst_01::run_medium_binary_round_trip(&sftp).await?;
|
||||
@@ -101,8 +106,6 @@ pub async fn test_sftp_compliance_suite() -> Result<()> {
|
||||
// reach the finalised object as x-amz-meta-* user metadata
|
||||
// through the CreateMultipartUpload input field. The S3 client
|
||||
// connects to the same rustfs process this suite already drives.
|
||||
let s3 = build_test_s3_client(&format!("http://{COMPLIANCE_RW_S3_ADDRESS}"));
|
||||
wait_for_s3_ready(&s3, 30).await?;
|
||||
cmptst_34::run_open_attrs_round_trip_multipart(&sftp, &s3).await?;
|
||||
|
||||
drop(sftp);
|
||||
|
||||
@@ -168,6 +168,10 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
.await
|
||||
.map_err(|e| anyhow!("{}", e))?;
|
||||
|
||||
// Protocol listeners can accept connections before IAM is initialized.
|
||||
let s3 = build_test_s3_client(S3_ENDPOINT);
|
||||
wait_for_s3_ready(&s3, S3_READY_ATTEMPTS).await?;
|
||||
|
||||
let (session, sftp) = connect_sftp().await?;
|
||||
|
||||
// --- 1. Subsystem canary: SFTP session reachable after password auth ---
|
||||
@@ -348,16 +352,6 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
let _ = bad_session.disconnect(russh::Disconnect::ByApplication, "", "en").await;
|
||||
info!("PASS: bad-password authentication rejected");
|
||||
|
||||
// --- Cross-protocol setup: aws-sdk-s3 client against the same server ---
|
||||
// The rustfs binary spawned for this suite serves both SFTP on port
|
||||
// 9022 and S3 on port 9000. The S3 stack may need a moment to finish
|
||||
// initialising after TCP is listening, so list_buckets is polled
|
||||
// until it succeeds before any cross-protocol assertion runs.
|
||||
info!("Testing SFTP: prepare aws-sdk-s3 client and wait for S3 readiness");
|
||||
let s3 = build_test_s3_client(S3_ENDPOINT);
|
||||
wait_for_s3_ready(&s3, S3_READY_ATTEMPTS).await?;
|
||||
info!("PASS: S3 endpoint reachable from cross-protocol client");
|
||||
|
||||
// --- SFTP write, S3 read: SHA256 round-trip ---
|
||||
// SFTP creates the object, then assert_cross_protocol_sha_match
|
||||
// fetches it via both S3 GetObject and SFTP READ and compares
|
||||
@@ -522,6 +516,9 @@ pub async fn test_sftp_idle_timeout_disconnects() -> Result<()> {
|
||||
.await
|
||||
.map_err(|e| anyhow!("{}", e))?;
|
||||
|
||||
let s3 = build_test_s3_client(&format!("http://{IDLE_S3_ADDRESS}"));
|
||||
wait_for_s3_ready(&s3, S3_READY_ATTEMPTS).await?;
|
||||
|
||||
let (session, sftp) = connect_sftp_to(IDLE_SFTP_ADDRESS).await?;
|
||||
|
||||
// Confirm the session is live before the wait so a failure in the
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Build-bound evidence for scanner and heal restart tests.
|
||||
|
||||
use crate::common::ClusterTopology;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RestartEvidenceRun {
|
||||
schema: u32,
|
||||
run_id: String,
|
||||
source_revision: String,
|
||||
test_build: serde_json::Value,
|
||||
binary: EvidenceBuild,
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct ScannerHealEvidenceCase {
|
||||
pub(crate) id: &'static str,
|
||||
pub(crate) oracle: &'static str,
|
||||
pub(crate) evidence: &'static str,
|
||||
pub(crate) unclean_shutdown_marker: bool,
|
||||
pub(crate) topology: EvidenceTopology,
|
||||
pub(crate) storage_class_standard: Option<&'static str>,
|
||||
pub(crate) erasure_set_drive_count: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct EvidenceTopology {
|
||||
pub(crate) nodes: usize,
|
||||
pub(crate) drives_per_node: usize,
|
||||
}
|
||||
|
||||
impl EvidenceTopology {
|
||||
pub(crate) const fn new(nodes: usize, drives_per_node: usize) -> Self {
|
||||
Self { nodes, drives_per_node }
|
||||
}
|
||||
|
||||
pub(crate) fn total_drives(self) -> usize {
|
||||
self.nodes * self.drives_per_node
|
||||
}
|
||||
|
||||
pub(crate) fn cluster_topology(self) -> ClusterTopology {
|
||||
ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RestartEvidenceContext {
|
||||
directory: PathBuf,
|
||||
run: RestartEvidenceRun,
|
||||
case: ScannerHealEvidenceCase,
|
||||
}
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..read]);
|
||||
}
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
pub(crate) fn restart_evidence_run(
|
||||
binary: &Path,
|
||||
case: ScannerHealEvidenceCase,
|
||||
) -> Result<Option<RestartEvidenceContext>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
if case.id.is_empty()
|
||||
|| case.oracle.is_empty()
|
||||
|| !case.oracle.ends_with(".json")
|
||||
|| case.oracle.contains('/')
|
||||
|| case.oracle.contains('\\')
|
||||
|| case.oracle.contains("..")
|
||||
|| !matches!(case.evidence, "process-restart" | "process-crash-restart")
|
||||
|| (case.evidence == "process-crash-restart") != case.unclean_shutdown_marker
|
||||
{
|
||||
return Err("invalid scanner/heal evidence case".into());
|
||||
}
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
return Err("oversized scanner/heal execution receipt".into());
|
||||
}
|
||||
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
|
||||
return Err("invalid scanner/heal execution identity".into());
|
||||
}
|
||||
let built = compiled_test_identity();
|
||||
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
|
||||
}
|
||||
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
|
||||
assert_eq!(
|
||||
file_sha256(&std::env::current_exe()?)?,
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join(case.oracle).exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some(RestartEvidenceContext { directory, run, case }))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) struct RestartObservation {
|
||||
pub(crate) nodes: usize,
|
||||
pub(crate) drives_per_node: usize,
|
||||
pub(crate) pid_before: u32,
|
||||
pub(crate) pid_after: u32,
|
||||
pub(crate) unclean_shutdown_marker: bool,
|
||||
pub(crate) objects: Vec<serde_json::Value>,
|
||||
pub(crate) node_listings: Vec<Vec<String>>,
|
||||
}
|
||||
|
||||
impl RestartEvidenceContext {
|
||||
pub(crate) fn write(self, binary: &Path, observed: RestartObservation) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
assert_ne!(observed.pid_before, observed.pid_after, "target must be a new process");
|
||||
assert_eq!(file_sha256(binary)?, self.run.binary.sha256, "server build changed during restart");
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": self.case.id, "evidence": self.case.evidence,
|
||||
"run_id": self.run.run_id, "source_revision": self.run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": self.run.binary.sha256,
|
||||
"test_binary_sha256": self.run.test_binary.sha256,
|
||||
"topology": {"nodes": observed.nodes, "drives_per_node": observed.drives_per_node},
|
||||
"pid_before": observed.pid_before, "pid_after": observed.pid_after,
|
||||
"unclean_shutdown_marker": observed.unclean_shutdown_marker,
|
||||
"objects": observed.objects, "node_listings": observed.node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
}
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(self.directory.join(self.case.oracle))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -563,7 +563,7 @@ pub mod store_list {
|
||||
}
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
|
||||
@@ -8220,7 +8220,7 @@ mod tests {
|
||||
}
|
||||
mod multipart_transport_tests {
|
||||
use super::super::super::replication_filemeta_boundary::ObjectPartInfo;
|
||||
use super::super::super::replication_storage_boundary::ObjectIO as _;
|
||||
use super::super::super::replication_storage_boundary::{ObjectIO as _, ReadPlan};
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use http_body_util::{BodyExt, Full};
|
||||
@@ -8229,6 +8229,7 @@ mod tests {
|
||||
#[derive(Debug)]
|
||||
struct Source {
|
||||
body: Bytes,
|
||||
stored: Option<Bytes>,
|
||||
info: ObjectInfo,
|
||||
ranges: StdMutex<Vec<(i64, i64)>>,
|
||||
full_reads: std::sync::atomic::AtomicUsize,
|
||||
@@ -8257,6 +8258,17 @@ mod tests {
|
||||
self.info.version_id.map(|id| id.to_string()),
|
||||
"every read retains the selected source version"
|
||||
);
|
||||
if let Some(stored) = &self.stored {
|
||||
if let Some(range) = &range {
|
||||
self.ranges.lock().expect("range journal lock").push((range.start, range.end));
|
||||
} else {
|
||||
self.full_reads.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
let plan = ReadPlan::build_for_request(range, &self.info, opts, &HeaderMap::new(), None).await?;
|
||||
let start = plan.storage_offset();
|
||||
let end = start + usize::try_from(plan.storage_length()).expect("nonnegative storage length");
|
||||
return plan.into_object_reader(Box::new(std::io::Cursor::new(stored.slice(start..end))), &self.info);
|
||||
}
|
||||
if range.is_none() {
|
||||
self.full_reads.fetch_add(1, Ordering::Relaxed);
|
||||
return Ok(GetObjectReader {
|
||||
@@ -8415,6 +8427,7 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
body: body.clone(),
|
||||
stored: Some(Bytes::from(stored)),
|
||||
ranges: StdMutex::new(Vec::new()),
|
||||
full_reads: std::sync::atomic::AtomicUsize::new(0),
|
||||
});
|
||||
@@ -8656,6 +8669,7 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
body: body.clone(),
|
||||
stored: None,
|
||||
ranges: StdMutex::new(Vec::new()),
|
||||
full_reads: std::sync::atomic::AtomicUsize::new(0),
|
||||
});
|
||||
|
||||
@@ -22,7 +22,7 @@ pub(crate) use crate::object_api::{
|
||||
GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence};
|
||||
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence, ReadPlan};
|
||||
pub(crate) use crate::storage_api_contracts::list::{
|
||||
ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
|
||||
};
|
||||
|
||||
@@ -285,6 +285,20 @@ fn peer_replay_state(audience: &str) -> PeerReplayState {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) fn clear_peer_replay_state_for_addr(addr: &str) -> std::io::Result<()> {
|
||||
let uri = addr
|
||||
.parse::<Uri>()
|
||||
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
|
||||
let audience = uri
|
||||
.authority()
|
||||
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
|
||||
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))??;
|
||||
if let Ok(mut states) = PEER_REPLAY_STATES.lock() {
|
||||
states.remove(&audience);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_peer_replay_response(
|
||||
audience: String,
|
||||
sent_state: PeerReplayState,
|
||||
@@ -619,6 +633,13 @@ mod tests {
|
||||
.remove(audience);
|
||||
}
|
||||
|
||||
fn set_peer_capability(audience: &str, state: PeerReplayState) {
|
||||
PEER_REPLAY_STATES
|
||||
.lock()
|
||||
.expect("peer capability cache lock must not be poisoned")
|
||||
.insert(audience.to_string(), state);
|
||||
}
|
||||
|
||||
fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> {
|
||||
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest {
|
||||
args: "canonical mutation request".to_string(),
|
||||
@@ -1090,6 +1111,23 @@ mod tests {
|
||||
clear_peer_capability(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_peer_replay_state_for_addr_removes_normalized_audience() {
|
||||
let audience = "clear-peer-replay-state-test:9000";
|
||||
let boot_epoch = Uuid::new_v4();
|
||||
set_peer_capability(
|
||||
audience,
|
||||
PeerReplayState {
|
||||
boot_epoch: Some(boot_epoch),
|
||||
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
|
||||
},
|
||||
);
|
||||
|
||||
clear_peer_replay_state_for_addr("http://clear-peer-replay-state-test:9000").expect("peer URI should clear replay state");
|
||||
|
||||
assert_eq!(peer_replay_state(audience), PeerReplayState::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -13,8 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
AuthenticatedChannel, TonicInterceptor, clear_peer_replay_state_for_addr, embedded_tonic_status,
|
||||
gen_tonic_signature_interceptor, heal_control_time_out_client, is_network_like_status, message_has_network_needle,
|
||||
node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -544,6 +545,16 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
|
||||
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
|
||||
}
|
||||
|
||||
fn heal_control_auth_may_need_replay_scope_refresh(err: &Error) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
Error::Io(io_err)
|
||||
if embedded_tonic_status(io_err).is_some_and(|status| {
|
||||
status.code() == tonic::Code::Unauthenticated && status.message() == "No valid auth token"
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
|
||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
|
||||
if topology_member != expected_member {
|
||||
@@ -1720,45 +1731,72 @@ impl PeerRestClient {
|
||||
return Err(Error::other("heal control command exceeds size limit"));
|
||||
}
|
||||
let capability_probe = rustfs_protos::is_heal_control_capability_probe(&command);
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self
|
||||
.get_heal_control_client()
|
||||
.await?
|
||||
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
|
||||
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
|
||||
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, &topology_fingerprint, &command)
|
||||
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
|
||||
let mut request = Request::new(HealControlRequest {
|
||||
version,
|
||||
topology_fingerprint: topology_fingerprint.clone(),
|
||||
command: command.clone().into(),
|
||||
});
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||
let response = client.heal_control(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
|
||||
));
|
||||
}
|
||||
if !capability_probe {
|
||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
version,
|
||||
&topology_fingerprint,
|
||||
&command,
|
||||
&response.result,
|
||||
)
|
||||
let result = self
|
||||
.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
|
||||
.await;
|
||||
if result
|
||||
.as_ref()
|
||||
.err()
|
||||
.is_some_and(heal_control_auth_may_need_replay_scope_refresh)
|
||||
{
|
||||
self.prepare_heal_control_auth_retry().await;
|
||||
return self
|
||||
.finalize_result(
|
||||
self.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
|
||||
.await,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.finalize_result(result).await
|
||||
}
|
||||
|
||||
async fn prepare_heal_control_auth_retry(&self) {
|
||||
if let Err(err) = clear_peer_replay_state_for_addr(&self.grid_host) {
|
||||
debug!(
|
||||
peer = %self.grid_host,
|
||||
error = %err,
|
||||
"could not clear heal control replay state before retry"
|
||||
);
|
||||
}
|
||||
self.evict_connection().await;
|
||||
}
|
||||
|
||||
async fn heal_control_once(
|
||||
&self,
|
||||
version: u32,
|
||||
topology_fingerprint: &str,
|
||||
command: &[u8],
|
||||
capability_probe: bool,
|
||||
) -> Result<Vec<u8>> {
|
||||
let mut client = self
|
||||
.get_heal_control_client()
|
||||
.await?
|
||||
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
|
||||
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
|
||||
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, topology_fingerprint, command)
|
||||
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
|
||||
let mut request = Request::new(HealControlRequest {
|
||||
version,
|
||||
topology_fingerprint: topology_fingerprint.to_string(),
|
||||
command: command.to_vec().into(),
|
||||
});
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||
let response = client.heal_control(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
|
||||
));
|
||||
}
|
||||
if !capability_probe {
|
||||
let canonical_response =
|
||||
rustfs_protos::canonical_heal_control_response_body(version, topology_fingerprint, command, &response.result)
|
||||
.map_err(|_| Error::other("heal control response length cannot be represented"))?;
|
||||
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
|
||||
}
|
||||
Ok(response.result.to_vec())
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
|
||||
}
|
||||
Ok(response.result.to_vec())
|
||||
}
|
||||
|
||||
/// Confirms that a peer supports the current heal-control coordination
|
||||
@@ -3728,6 +3766,22 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_control_auth_retry_is_limited_to_transport_auth_rejection() {
|
||||
assert!(heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||
tonic::Status::unauthenticated("No valid auth token")
|
||||
)));
|
||||
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||
tonic::Status::permission_denied("bad signature")
|
||||
)));
|
||||
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||
tonic::Status::unauthenticated("application rejected heal control")
|
||||
)));
|
||||
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::other(
|
||||
"Io error: code: 'Unauthenticated', message: \"No valid auth token\""
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_keeps_slow_peers_online() {
|
||||
// The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s)
|
||||
|
||||
@@ -378,6 +378,26 @@ impl ECStore {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Whether this replacement set owns the pool's metadata replica.
|
||||
///
|
||||
/// Pool metadata follows normal object placement within each pool. A valid
|
||||
/// non-owner set has no replica to repair; missing metadata on the owner
|
||||
/// set still requires healing and target-specific readback.
|
||||
pub fn replacement_pool_metadata_applies(&self, pool_index: usize, set_index: usize) -> Result<bool> {
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
|
||||
let selected = pool.get_disks_for_heal_object(
|
||||
POOL_META_NAME,
|
||||
&HealOpts {
|
||||
set: Some(set_index),
|
||||
..Default::default()
|
||||
},
|
||||
)?;
|
||||
Ok(Arc::ptr_eq(&selected, &pool.get_disks_by_key(POOL_META_NAME)))
|
||||
}
|
||||
|
||||
#[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
|
||||
pub async fn replacement_targets_have_version(
|
||||
&self,
|
||||
@@ -829,6 +849,73 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_pool_metadata_applies_to_the_written_replica_in_each_pool() {
|
||||
let mut store = minimal_heal_store().await;
|
||||
for pool_index in 0..store.pools.len() {
|
||||
assert!(
|
||||
store
|
||||
.replacement_pool_metadata_applies(pool_index, 0)
|
||||
.expect("a valid single-set pool should have a metadata owner")
|
||||
);
|
||||
}
|
||||
store.ctx = Arc::new(InstanceContext::new());
|
||||
for algorithm in [
|
||||
crate::disk::format::DistributionAlgoVersion::V1,
|
||||
crate::disk::format::DistributionAlgoVersion::V2,
|
||||
crate::disk::format::DistributionAlgoVersion::V3,
|
||||
] {
|
||||
let mut temp_dirs = Vec::new();
|
||||
for pool_index in 0..store.pools.len() {
|
||||
let (dirs, mut pool) =
|
||||
crate::core::sets::make_local_two_set_sets_for_pool_with_ctx(Arc::clone(&store.ctx), pool_index).await;
|
||||
temp_dirs.extend(dirs);
|
||||
Arc::get_mut(&mut pool)
|
||||
.expect("fixture pool should have one owner")
|
||||
.distribution_algo = algorithm.clone();
|
||||
store.pools[pool_index] = pool;
|
||||
}
|
||||
for (pool_index, pool) in store.pools.iter().enumerate() {
|
||||
let mut required_sets = 0;
|
||||
for set_index in 0..pool.disk_set.len() {
|
||||
required_sets += usize::from(
|
||||
store
|
||||
.replacement_pool_metadata_applies(pool_index, set_index)
|
||||
.expect("valid replacement topology should be classified before metadata exists"),
|
||||
);
|
||||
}
|
||||
assert_eq!(required_sets, 1, "missing metadata cannot exempt the owner set");
|
||||
save_config(pool.clone(), POOL_META_NAME, b"pool metadata placement".to_vec())
|
||||
.await
|
||||
.expect("normal config writes should persist one metadata replica per pool");
|
||||
for (set_index, set) in pool.disk_set.iter().enumerate() {
|
||||
let applies = store
|
||||
.replacement_pool_metadata_applies(pool_index, set_index)
|
||||
.expect("valid replacement topology should be classified");
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
let replica = disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await;
|
||||
if applies {
|
||||
replica.expect("the metadata owner must match actual persisted shards");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(replica, Err(crate::disk::error::DiskError::FileNotFound)),
|
||||
"non-owner sets must have no persisted metadata shard; observed error: {:?}",
|
||||
replica.as_ref().err()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.replacement_pool_metadata_applies(pool_index, pool.disk_set.len())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(store.replacement_pool_metadata_applies(store.pools.len(), 0).is_err());
|
||||
}
|
||||
|
||||
async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
|
||||
let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
|
||||
let missing_disk = target_set.disks.read().await[0]
|
||||
|
||||
@@ -34,7 +34,7 @@ use std::time::{Duration, UNIX_EPOCH};
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::{DiskStore, EcstoreError};
|
||||
use super::{DiskStore, EcstoreError, POOL_META_NAME, RUSTFS_META_BUCKET};
|
||||
|
||||
/// Outcome of classifying an error returned by [`HealStorageAPI::heal_object`].
|
||||
enum HealObjectOutcome {
|
||||
@@ -68,6 +68,13 @@ struct PageConcurrencyGuard {
|
||||
set_label: String,
|
||||
}
|
||||
|
||||
struct ErasureSetPassCounters<'a> {
|
||||
processed_objects: &'a mut u64,
|
||||
successful_objects: &'a mut u64,
|
||||
failed_objects: &'a mut u64,
|
||||
skipped_objects: &'a mut u64,
|
||||
}
|
||||
|
||||
impl PageConcurrencyGuard {
|
||||
fn new(in_flight: Arc<AtomicUsize>, set_label: String) -> Self {
|
||||
let current = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
@@ -834,6 +841,21 @@ impl ErasureSetHealer {
|
||||
current_object_index = 0;
|
||||
}
|
||||
|
||||
if failed_objects == 0 && skipped_objects == 0 && failed_buckets == 0 {
|
||||
self.heal_replacement_pool_metadata(
|
||||
set_disk_id,
|
||||
&mut ErasureSetPassCounters {
|
||||
processed_objects: &mut processed_objects,
|
||||
successful_objects: &mut successful_objects,
|
||||
failed_objects: &mut failed_objects,
|
||||
skipped_objects: &mut skipped_objects,
|
||||
},
|
||||
resume_manager,
|
||||
checkpoint_manager,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// 5. finalize. Only declare the set healed when nothing failed AND
|
||||
// nothing was transiently skipped — otherwise the resume/checkpoint
|
||||
// state must survive so the failed/skipped versions are retried instead
|
||||
@@ -919,6 +941,211 @@ impl ErasureSetHealer {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn heal_replacement_pool_metadata(
|
||||
&self,
|
||||
set_disk_id: &str,
|
||||
counters: &mut ErasureSetPassCounters<'_>,
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
) -> Result<()> {
|
||||
if self.replacement_task_id.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
if self.target_endpoints.is_empty() {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Replacement pool metadata heal requires target endpoints".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if !self.storage.replacement_pool_metadata_applies(&self.heal_opts).await? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
|
||||
let checkpoint_key = compose_key(&object_key, None);
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
if checkpoint.processed_objects.contains(&checkpoint_key)
|
||||
|| checkpoint.failed_objects.contains(&checkpoint_key)
|
||||
|| checkpoint.skipped_objects.contains(&checkpoint_key)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
drop(checkpoint);
|
||||
|
||||
self.verify_replacement_identity_fence("pool metadata").await?;
|
||||
resume_manager
|
||||
.set_current_item(Some(RUSTFS_META_BUCKET.to_string()), Some(POOL_META_NAME.to_string()))
|
||||
.await?;
|
||||
|
||||
let result = match self
|
||||
.storage
|
||||
.heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, None, &self.heal_opts)
|
||||
.await
|
||||
{
|
||||
Ok((result, None)) if target_outcomes_complete(&result, &self.target_endpoints) => {
|
||||
let object_size = result_object_size_u64(&result);
|
||||
match self
|
||||
.storage
|
||||
.replacement_targets_have_version(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
None,
|
||||
&self.heal_opts,
|
||||
&self.target_endpoints,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(true) => (object_size, Ok(())),
|
||||
Ok(false) => (
|
||||
object_size,
|
||||
Err(Error::transient_skip(
|
||||
"Skipped replacement pool metadata heal because target readback did not confirm the committed version",
|
||||
)),
|
||||
),
|
||||
Err(err) => (
|
||||
object_size,
|
||||
Err(Error::transient_skip(format!(
|
||||
"Skipped replacement pool metadata heal because target readback failed: {err}"
|
||||
))),
|
||||
),
|
||||
}
|
||||
}
|
||||
Ok((result, None)) => (
|
||||
result_object_size_u64(&result),
|
||||
Err(Error::transient_skip(
|
||||
"Skipped replacement pool metadata heal because a replacement target was not committed",
|
||||
)),
|
||||
),
|
||||
Ok((result, Some(err))) => {
|
||||
let object_size = result_object_size_u64(&result);
|
||||
match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent | HealObjectOutcome::Transient => (
|
||||
object_size,
|
||||
Err(Error::transient_skip(format!(
|
||||
"Skipped replacement pool metadata heal due to transient error: {err}"
|
||||
))),
|
||||
),
|
||||
HealObjectOutcome::Failed => (object_size, Err(err)),
|
||||
}
|
||||
}
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(err) => match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent | HealObjectOutcome::Transient => (
|
||||
0,
|
||||
Err(Error::transient_skip(format!(
|
||||
"Skipped replacement pool metadata heal due to transient error: {err}"
|
||||
))),
|
||||
),
|
||||
HealObjectOutcome::Failed => (0, Err(err)),
|
||||
},
|
||||
};
|
||||
|
||||
let (object_size, result) = result;
|
||||
let mut bytes_processed = self.progress.read().await.bytes_processed;
|
||||
let mut telemetry_unknown = false;
|
||||
let checkpoint_outcome = match result {
|
||||
Ok(()) => {
|
||||
telemetry_unknown |= !increment_counter(counters.successful_objects);
|
||||
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket = RUSTFS_META_BUCKET,
|
||||
object = POOL_META_NAME,
|
||||
state = "healed",
|
||||
"Replacement pool metadata healed"
|
||||
);
|
||||
CheckpointObjectOutcome::Processed
|
||||
}
|
||||
Err(Error::TransientSkip { message }) => {
|
||||
telemetry_unknown |= !increment_counter(counters.skipped_objects);
|
||||
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||
warn!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket = RUSTFS_META_BUCKET,
|
||||
object = POOL_META_NAME,
|
||||
state = "transient_skip",
|
||||
error = %message,
|
||||
"Replacement pool metadata heal skipped due to transient error"
|
||||
);
|
||||
CheckpointObjectOutcome::Skipped
|
||||
}
|
||||
Err(err) => {
|
||||
telemetry_unknown |= !increment_counter(counters.failed_objects);
|
||||
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||
warn!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket = RUSTFS_META_BUCKET,
|
||||
object = POOL_META_NAME,
|
||||
state = "failed",
|
||||
error = %err,
|
||||
"Replacement pool metadata heal failed"
|
||||
);
|
||||
CheckpointObjectOutcome::Failed
|
||||
}
|
||||
};
|
||||
|
||||
telemetry_unknown |= !increment_counter(counters.processed_objects);
|
||||
let (outcome_record, counter_unknown, skipped_new_versions, skipped_ilm_expired) = {
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(object_key.clone()));
|
||||
progress.update_object_progress(
|
||||
*counters.processed_objects,
|
||||
*counters.successful_objects,
|
||||
*counters.failed_objects,
|
||||
*counters.skipped_objects,
|
||||
bytes_processed,
|
||||
);
|
||||
if telemetry_unknown {
|
||||
progress.mark_unknown();
|
||||
}
|
||||
(
|
||||
CheckpointObjectOutcomeRecord {
|
||||
object: checkpoint_key,
|
||||
outcome: checkpoint_outcome,
|
||||
successful: progress.objects_healed,
|
||||
failed: progress.objects_failed,
|
||||
skipped: progress.skipped_objects,
|
||||
bytes: progress.bytes_processed,
|
||||
skipped_new_versions: progress.skipped_new_versions,
|
||||
skipped_ilm_expired: progress.skipped_ilm_expired,
|
||||
counter_unknown: progress.counter_unknown,
|
||||
},
|
||||
progress.counter_unknown,
|
||||
progress.skipped_new_versions,
|
||||
progress.skipped_ilm_expired,
|
||||
)
|
||||
};
|
||||
checkpoint_manager.record_object_outcome(outcome_record).await?;
|
||||
resume_manager
|
||||
.update_progress_with_bytes(
|
||||
*counters.processed_objects,
|
||||
*counters.successful_objects,
|
||||
*counters.failed_objects,
|
||||
*counters.skipped_objects,
|
||||
bytes_processed,
|
||||
)
|
||||
.await?;
|
||||
resume_manager
|
||||
.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired)
|
||||
.await?;
|
||||
if counter_unknown {
|
||||
resume_manager.mark_counter_unknown().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// heal single bucket with resume
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
|
||||
@@ -1679,7 +1906,8 @@ mod resume_loop_tests {
|
||||
use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage_api::status::BucketInfo;
|
||||
use crate::heal::{
|
||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk,
|
||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, POOL_META_NAME, RUSTFS_META_BUCKET,
|
||||
new_disk,
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource};
|
||||
@@ -1773,6 +2001,28 @@ mod resume_loop_tests {
|
||||
assert!(!target_outcomes_complete(&duplicate, &["replacement-a".to_string()]));
|
||||
}
|
||||
|
||||
fn replacement_target_ok_result(endpoint: &str, object: &str) -> HealResultItem {
|
||||
HealResultItem {
|
||||
object: object.to_string(),
|
||||
object_size: 1024,
|
||||
before: Infos {
|
||||
drives: vec![HealDriveInfo {
|
||||
endpoint: endpoint.to_string(),
|
||||
state: "missing".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
after: Infos {
|
||||
drives: vec![HealDriveInfo {
|
||||
endpoint: endpoint.to_string(),
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Page {
|
||||
items: Vec<HealListItem>,
|
||||
@@ -1783,6 +2033,8 @@ mod resume_loop_tests {
|
||||
#[derive(Clone)]
|
||||
enum HealOutcome {
|
||||
Ok,
|
||||
/// The object has no metadata on any disk in the selected set.
|
||||
FileNotFound,
|
||||
/// The version vanished before heal ran (deleted mid-heal).
|
||||
VersionNotFound,
|
||||
/// A transient infrastructure condition (offline disk / unmet quorum):
|
||||
@@ -1808,6 +2060,8 @@ mod resume_loop_tests {
|
||||
/// Target-specific physical readback evidence per `compose_key`; the
|
||||
/// fake models a healthy backend unless a test explicitly revokes it.
|
||||
replacement_commit_evidence: Mutex<HashMap<String, ReplacementCommitEvidence>>,
|
||||
pool_metadata_not_applicable: AtomicBool,
|
||||
fail_pool_metadata_scope: AtomicBool,
|
||||
lifecycle_expired: Mutex<HashSet<String>>,
|
||||
/// every heal_object call recorded as (name, version_id)
|
||||
heal_calls: Mutex<Vec<(String, Option<String>)>>,
|
||||
@@ -1909,6 +2163,7 @@ mod resume_loop_tests {
|
||||
let outcome = self.outcomes.lock().unwrap().get(&key).cloned().unwrap_or(HealOutcome::Ok);
|
||||
match outcome {
|
||||
HealOutcome::Ok => Ok((self.results.lock().unwrap().get(&key).cloned().unwrap_or_default(), None)),
|
||||
HealOutcome::FileNotFound => Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::FileNotFound)))),
|
||||
HealOutcome::VersionNotFound => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::FileVersionNotFound))))
|
||||
}
|
||||
@@ -1922,6 +2177,17 @@ mod resume_loop_tests {
|
||||
async fn heal_format(&self, _dry: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
Ok((HealResultItem::default(), None))
|
||||
}
|
||||
async fn replacement_pool_metadata_applies(&self, opts: &HealOpts) -> Result<bool> {
|
||||
if self.fail_pool_metadata_scope.load(Ordering::SeqCst) {
|
||||
return Err(Error::other("injected pool metadata scope failure"));
|
||||
}
|
||||
if self.pool_metadata_not_applicable.load(Ordering::SeqCst) {
|
||||
assert_eq!(opts.pool, Some(0));
|
||||
assert_eq!(opts.set, Some(1));
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -2444,6 +2710,8 @@ mod resume_loop_tests {
|
||||
HealRequestSource::AutoHeal,
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||
env.storage
|
||||
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||
|
||||
healer
|
||||
.heal_erasure_set(&["b".to_string()], "pool_0_set_0")
|
||||
@@ -2461,9 +2729,164 @@ mod resume_loop_tests {
|
||||
CheckpointManager::has_checkpoint(&env.healer.disk, &replacement_task_id).await,
|
||||
"the checkpoint must survive until the caller clears the healing marker"
|
||||
);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
drop(checkpoint);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_pool_metadata_non_owner_completes_but_missing_owner_retries() {
|
||||
for owns_pool_metadata in [false, true] {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let replacement_task_id = ResumeUtils::generate_task_id();
|
||||
let set_index = usize::from(!owns_pool_metadata);
|
||||
let set_disk_id = format!("pool_0_set_{set_index}");
|
||||
ResumeManager::new_replacement_intent(
|
||||
env.healer.disk.clone(),
|
||||
replacement_task_id.clone(),
|
||||
set_disk_id.clone(),
|
||||
vec!["b".to_string()],
|
||||
vec!["replacement-a".to_string()],
|
||||
vec![crate::heal::resume::ReplacementTargetIdentity {
|
||||
endpoint: "replacement-a".to_string(),
|
||||
canonical_path: "/mnt/replacement-a".to_string(),
|
||||
physical_device_ids: vec!["device-a".to_string()],
|
||||
filesystem_identity: "1:2:3".to_string(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("replacement intent should persist");
|
||||
env.storage
|
||||
.pool_metadata_not_applicable
|
||||
.store(!owns_pool_metadata, Ordering::SeqCst);
|
||||
env.storage.set_outcome(POOL_META_NAME, None, HealOutcome::FileNotFound);
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(set_index),
|
||||
..Default::default()
|
||||
},
|
||||
HealRequestSource::AutoHeal,
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||
|
||||
let result = healer.heal_erasure_set(&["b".to_string()], &set_disk_id).await;
|
||||
let state = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id)
|
||||
.await
|
||||
.expect("replacement state must remain until marker cleanup")
|
||||
.get_state()
|
||||
.await;
|
||||
if owns_pool_metadata {
|
||||
let error = result.expect_err("missing metadata in the owner set must keep replacement incomplete");
|
||||
assert!(error.to_string().contains("Replacement erasure set heal incomplete"));
|
||||
assert!(!state.completed);
|
||||
assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent);
|
||||
assert_eq!(state.retry_count, 1);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
} else {
|
||||
result.expect("a non-owner set must complete without a pool metadata replica");
|
||||
assert!(state.completed);
|
||||
assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Verified);
|
||||
assert_eq!(state.retry_count, 0);
|
||||
assert!(env.storage.calls().is_empty(), "non-owner sets must not attempt pool metadata repair");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_pool_metadata_unknown_scope_cannot_complete() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
HealRequestSource::AutoHeal,
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some("generation-a".to_string()));
|
||||
env.storage.fail_pool_metadata_scope.store(true, Ordering::SeqCst);
|
||||
env.storage
|
||||
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||
let mut processed_objects = 0;
|
||||
let mut successful_objects = 0;
|
||||
let mut failed_objects = 0;
|
||||
let mut skipped_objects = 0;
|
||||
let error = healer
|
||||
.heal_replacement_pool_metadata(
|
||||
"pool_0_set_0",
|
||||
&mut super::ErasureSetPassCounters {
|
||||
processed_objects: &mut processed_objects,
|
||||
successful_objects: &mut successful_objects,
|
||||
failed_objects: &mut failed_objects,
|
||||
skipped_objects: &mut skipped_objects,
|
||||
},
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
)
|
||||
.await
|
||||
.expect_err("unknown metadata placement must keep replacement incomplete");
|
||||
assert!(error.to_string().contains("injected pool metadata scope failure"));
|
||||
assert!(env.storage.calls().is_empty());
|
||||
assert_eq!((processed_objects, successful_objects, failed_objects, skipped_objects), (0, 0, 0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_pool_metadata_readback_failure_schedules_retry() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let replacement_task_id = ResumeUtils::generate_task_id();
|
||||
ResumeManager::new_replacement_intent(
|
||||
env.healer.disk.clone(),
|
||||
replacement_task_id.clone(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["b".to_string()],
|
||||
vec!["replacement-a".to_string()],
|
||||
vec![crate::heal::resume::ReplacementTargetIdentity {
|
||||
endpoint: "replacement-a".to_string(),
|
||||
canonical_path: "/mnt/replacement-a".to_string(),
|
||||
physical_device_ids: vec!["device-a".to_string()],
|
||||
filesystem_identity: "1:2:3".to_string(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("replacement intent should persist");
|
||||
env.storage
|
||||
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||
env.storage.set_replacement_commit_evidence(POOL_META_NAME, None, false);
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts::default(),
|
||||
HealRequestSource::AutoHeal,
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||
|
||||
let error = healer
|
||||
.heal_erasure_set(&["b".to_string()], "pool_0_set_0")
|
||||
.await
|
||||
.expect_err("unconfirmed pool metadata readback must keep the replacement incomplete");
|
||||
|
||||
assert!(error.to_string().contains("Replacement erasure set heal incomplete"));
|
||||
let state = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id)
|
||||
.await
|
||||
.expect("replacement retry state must remain")
|
||||
.get_state()
|
||||
.await;
|
||||
assert!(!state.completed);
|
||||
assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent);
|
||||
assert_eq!(state.retry_count, 1);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_exhaustion_keeps_resume_artifacts_for_recovery() {
|
||||
let env = make_env().await;
|
||||
|
||||
@@ -506,6 +506,25 @@ fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key:
|
||||
.map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
|
||||
}
|
||||
|
||||
fn request_matches_task(request: &HealRequest, task: &HealTask) -> bool {
|
||||
request.heal_type == task.heal_type
|
||||
&& request.options == task.options
|
||||
&& request.priority == task.priority
|
||||
&& request.source == task.source
|
||||
&& request.retry_attempts == task.retry_attempts
|
||||
&& request.heal_endpoints == task.heal_endpoints
|
||||
}
|
||||
|
||||
fn request_matches_request(request: &HealRequest, existing: &HealRequest) -> bool {
|
||||
request.heal_type == existing.heal_type
|
||||
&& request.options == existing.options
|
||||
&& request.priority == existing.priority
|
||||
&& request.source == existing.source
|
||||
&& request.force_start == existing.force_start
|
||||
&& request.retry_attempts == existing.retry_attempts
|
||||
&& request.heal_endpoints == existing.heal_endpoints
|
||||
}
|
||||
|
||||
fn retrying_heal_for_dedup_key(retrying_heals: &HashMap<String, RetryingHeal>, key: &str) -> Option<(String, HealType)> {
|
||||
retrying_heals
|
||||
.iter()
|
||||
@@ -1548,6 +1567,18 @@ impl HealManager {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn durable_mrf_repair_anchor(
|
||||
&self,
|
||||
intent: &rustfs_common::mrf_channel::MrfIntent,
|
||||
) -> Option<rustfs_common::mrf_channel::MrfDurableRepairAnchor> {
|
||||
match self.storage.mrf_bucket_incarnation_id(intent.bucket.as_ref()).await {
|
||||
Ok(Some(bucket_incarnation_id)) => {
|
||||
rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(intent, bucket_incarnation_id)
|
||||
}
|
||||
Ok(None) | Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn submit_heal_request_with_receipt_alias_and_mrf_notice(
|
||||
&self,
|
||||
request: HealRequest,
|
||||
@@ -1613,6 +1644,64 @@ impl HealManager {
|
||||
pause_duplicate_admission_after_active_lock(&request.id).await;
|
||||
let mut queue = self.heal_queue.lock().await;
|
||||
let retrying_heals = self.retrying_heals.lock().await;
|
||||
|
||||
let request_id_admission = active_heals
|
||||
.get(&request.id)
|
||||
.map(|task| (request_matches_task(&request, task), "active"))
|
||||
.or_else(|| {
|
||||
queue
|
||||
.requests()
|
||||
.find(|queued| queued.id == request.id)
|
||||
.map(|queued| (request_matches_request(&request, queued), "queued"))
|
||||
})
|
||||
.or_else(|| {
|
||||
retrying_heals
|
||||
.get(&request.id)
|
||||
.map(|retrying| (request_matches_request(&request, &retrying.request), "retrying"))
|
||||
});
|
||||
if let Some((matches_existing, duplicate_state)) = request_id_admission {
|
||||
let admission = if matches_existing {
|
||||
HealAdmissionResult::Accepted
|
||||
} else {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
};
|
||||
if matches!(admission, HealAdmissionResult::Accepted | HealAdmissionResult::Merged)
|
||||
&& let Some(target) = mrf_notice_target
|
||||
{
|
||||
let mut targets = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets);
|
||||
Self::insert_mrf_repair_notice_target(&mut targets, &request.id, target);
|
||||
}
|
||||
drop(retrying_heals);
|
||||
drop(queue);
|
||||
drop(active_heals);
|
||||
let lock_phase = lock_phase_start.elapsed();
|
||||
Self::record_admission_metric(request.source, admission, "duplicate");
|
||||
self.record_admission_observation(HealAdmissionObservation {
|
||||
source,
|
||||
result: admission,
|
||||
context: "duplicate",
|
||||
force_start,
|
||||
displaced: false,
|
||||
start_duration: admission_start.elapsed(),
|
||||
lock_phase,
|
||||
});
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
duplicate_state,
|
||||
result = admission.result_label(),
|
||||
reason = admission.reason_label(),
|
||||
"Heal queue admission reused an existing request id"
|
||||
);
|
||||
return Ok(HealAdmissionReceipt {
|
||||
result: admission,
|
||||
task_id: request.id,
|
||||
});
|
||||
}
|
||||
|
||||
let duplicate = (!request.force_start).then(|| {
|
||||
active_heal_for_dedup_key(&active_heals, &dedup_key)
|
||||
.map(|(task_id, _)| (task_id, "active"))
|
||||
|
||||
@@ -733,23 +733,30 @@ pub(super) fn mrf_verified_repair_event_for_target(
|
||||
HealObjectDisposition::AuthoritativelyAbsent => MrfVerifiedRepairDisposition::AuthoritativelyAbsent,
|
||||
_ => return None,
|
||||
};
|
||||
if target.kind != MrfKind::PartialWrite {
|
||||
return None;
|
||||
}
|
||||
let expected_kind = HealObjectKind::Object;
|
||||
let expected_kind = match target.kind {
|
||||
MrfKind::DecodeFailure => HealObjectKind::Decode,
|
||||
MrfKind::MetadataCorruption => HealObjectKind::Metadata,
|
||||
MrfKind::PartialWrite => HealObjectKind::Object,
|
||||
};
|
||||
if outcome.identity.kind != expected_kind
|
||||
|| outcome.identity.bucket.as_str() != target.bucket.as_ref()
|
||||
|| outcome.identity.object.as_str() != target.object.as_ref()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let version_id = target.version_id.filter(|bytes| *bytes != [0; 16]);
|
||||
let version_id = (!matches!(target.kind, MrfKind::MetadataCorruption))
|
||||
.then_some(target.version_id)
|
||||
.flatten()
|
||||
.filter(|bytes| *bytes != [0; 16]);
|
||||
let expected_version = version_id.map(|bytes| uuid::Uuid::from_bytes(bytes).to_string());
|
||||
if outcome.identity.version_id != expected_version {
|
||||
return None;
|
||||
}
|
||||
let expected_pool = target.scope.and_then(|scope| usize::try_from(scope.pool_index).ok());
|
||||
let expected_set = target.scope.and_then(|scope| usize::try_from(scope.set_index).ok());
|
||||
let scope = (!matches!(target.kind, MrfKind::MetadataCorruption))
|
||||
.then_some(target.scope)
|
||||
.flatten();
|
||||
let expected_pool = scope.and_then(|scope| usize::try_from(scope.pool_index).ok());
|
||||
let expected_set = scope.and_then(|scope| usize::try_from(scope.set_index).ok());
|
||||
if outcome.identity.pool_index != expected_pool || outcome.identity.set_index != expected_set {
|
||||
return None;
|
||||
}
|
||||
@@ -759,7 +766,7 @@ pub(super) fn mrf_verified_repair_event_for_target(
|
||||
bucket: target.bucket.clone(),
|
||||
object: target.object.clone(),
|
||||
version_id,
|
||||
scope: target.scope,
|
||||
scope,
|
||||
lease: target.lease,
|
||||
bucket_incarnation_id,
|
||||
disposition,
|
||||
|
||||
@@ -1142,18 +1142,48 @@ fn mrf_verified_repair_event_requires_positive_exact_identity() {
|
||||
assert_eq!(event.bucket_incarnation_id, incarnation);
|
||||
assert_eq!(event.disposition, MrfVerifiedRepairDisposition::Repaired);
|
||||
|
||||
assert!(
|
||||
mrf_verified_repair_event_for_target(
|
||||
&MrfRepairNoticeTarget {
|
||||
kind: MrfKind::DecodeFailure,
|
||||
..target.clone()
|
||||
},
|
||||
&matching
|
||||
)
|
||||
.is_none(),
|
||||
"only receipt-producing partial-write object heals can publish verified events today"
|
||||
let decode_target = MrfRepairNoticeTarget {
|
||||
kind: MrfKind::DecodeFailure,
|
||||
..target.clone()
|
||||
};
|
||||
let decode_outcome = HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Decode,
|
||||
..matching.identity.clone()
|
||||
},
|
||||
..matching.clone()
|
||||
};
|
||||
let decode_event =
|
||||
mrf_verified_repair_event_for_target(&decode_target, &decode_outcome).expect("decode repairs publish exact proofs");
|
||||
assert_eq!(decode_event.kind, MrfKind::DecodeFailure);
|
||||
assert_eq!(
|
||||
decode_event.scope,
|
||||
Some(MrfScope {
|
||||
pool_index: 1,
|
||||
set_index: 2
|
||||
})
|
||||
);
|
||||
|
||||
let metadata_target = MrfRepairNoticeTarget {
|
||||
kind: MrfKind::MetadataCorruption,
|
||||
..target.clone()
|
||||
};
|
||||
let metadata_outcome = HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Metadata,
|
||||
version_id: None,
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
..matching.identity.clone()
|
||||
},
|
||||
..matching.clone()
|
||||
};
|
||||
let metadata_event =
|
||||
mrf_verified_repair_event_for_target(&metadata_target, &metadata_outcome).expect("metadata repairs publish exact proofs");
|
||||
assert_eq!(metadata_event.kind, MrfKind::MetadataCorruption);
|
||||
assert_eq!(metadata_event.version_id, None);
|
||||
assert_eq!(metadata_event.scope, None);
|
||||
|
||||
for rejected in [
|
||||
HealObjectOutcome {
|
||||
disposition: HealObjectDisposition::Unknown,
|
||||
@@ -1839,7 +1869,7 @@ async fn test_submit_heal_request_returns_merged_for_duplicate() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let request = HealRequest::new(
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
@@ -1856,6 +1886,7 @@ async fn test_submit_heal_request_returns_merged_for_duplicate() {
|
||||
.expect("first request should be accepted"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
request.id = uuid::Uuid::new_v4().to_string();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
@@ -3695,7 +3726,7 @@ async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
|
||||
}),
|
||||
);
|
||||
|
||||
let request = HealRequest::new(
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
@@ -3712,6 +3743,7 @@ async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
|
||||
.expect("first request should be accepted"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
request.id = uuid::Uuid::new_v4().to_string();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
@@ -4537,6 +4569,48 @@ async fn test_force_start_marks_dedup_key_for_future_duplicates() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn same_request_id_replay_reuses_existing_task_without_force_start_duplication() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let mut original = admin_prefix_request("bucket", "logs/");
|
||||
original.force_start = true;
|
||||
let original_id = original.id.clone();
|
||||
let accepted = manager
|
||||
.submit_heal_request_with_receipt(original.clone())
|
||||
.await
|
||||
.expect("original forceStart request should queue");
|
||||
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
|
||||
assert_eq!(accepted.task_id, original_id);
|
||||
|
||||
let replayed = manager
|
||||
.submit_heal_request_with_receipt(original.clone())
|
||||
.await
|
||||
.expect("same request id and payload should reuse the existing task");
|
||||
assert_eq!(replayed.result, HealAdmissionResult::Accepted);
|
||||
assert_eq!(replayed.task_id, original_id);
|
||||
assert_eq!(
|
||||
manager.get_queue_length().await,
|
||||
1,
|
||||
"exact forceStart replay must not create a second queued task"
|
||||
);
|
||||
|
||||
let mut changed = original;
|
||||
changed.options.remove_corrupted = true;
|
||||
let changed = manager
|
||||
.submit_heal_request_with_receipt(changed)
|
||||
.await
|
||||
.expect("same request id with a changed payload should fail closed");
|
||||
assert_eq!(changed.result, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning));
|
||||
assert_eq!(changed.task_id, original_id);
|
||||
assert_eq!(
|
||||
manager.get_queue_length().await,
|
||||
1,
|
||||
"same-id conflict must not displace or duplicate the original task"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_running_heal_set_counts_groups_set_scoped_tasks() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
@@ -27,10 +27,10 @@ pub mod task;
|
||||
pub mod utils;
|
||||
|
||||
use storage_api::owner::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskOption,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO,
|
||||
ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk,
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME,
|
||||
ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes,
|
||||
EcstoreDiskError, EcstoreDiskOption, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
|
||||
EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk,
|
||||
};
|
||||
|
||||
pub use erasure_healer::ErasureSetHealer;
|
||||
@@ -41,6 +41,7 @@ pub use task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
|
||||
pub(crate) const DATA_USAGE_CACHE_NAME: &str = ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||
pub(crate) const BUCKET_META_PREFIX: &str = ECSTORE_BUCKET_META_PREFIX;
|
||||
pub(crate) const RUSTFS_META_BUCKET: &str = ECSTORE_RUSTFS_META_BUCKET;
|
||||
pub(crate) const POOL_META_NAME: &str = ECSTORE_POOL_META_NAME;
|
||||
|
||||
/// Marker written to every local disk while the process runs; removed by
|
||||
/// [`clear_unclean_shutdown_markers`] on graceful shutdown. Finding it at
|
||||
|
||||
@@ -36,9 +36,9 @@
|
||||
use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
|
||||
use crate::heal::manager::{HealManager, MrfRepairNoticeTarget};
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIngressResult, MrfIntent};
|
||||
use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfDurableRepairAnchor, MrfIngressResult, MrfIntent};
|
||||
use rustfs_heal_contracts::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
@@ -134,7 +134,10 @@ struct MrfQueueKey {
|
||||
}
|
||||
|
||||
fn queue_key(intent: &MrfIntent) -> MrfQueueKey {
|
||||
let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]);
|
||||
let version_id = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
|
||||
.then_some(intent.version_id)
|
||||
.flatten()
|
||||
.filter(|bytes| *bytes != [0; 16]);
|
||||
let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
|
||||
.then_some(intent.scope)
|
||||
.flatten();
|
||||
@@ -527,29 +530,107 @@ struct MrfRuntime {
|
||||
/// True while a journal snapshot exists on disk that may still be needed
|
||||
/// for replay or cleanup.
|
||||
journal_on_disk: bool,
|
||||
/// True when replay observed a responsibility that cannot be discharged by
|
||||
/// a complete verified repair proof in this process.
|
||||
retain_replay_journal: bool,
|
||||
/// Partial-write responsibilities accepted from replay and waiting for an
|
||||
/// exact storage-owned proof before the startup journal can be deleted.
|
||||
durable_replay_anchors: Vec<MrfDurableRepairAnchor>,
|
||||
/// Startup responsibilities remain in every successor snapshot until an
|
||||
/// exact verified repair discharges them. Live admissions never grow this
|
||||
/// set, so its size is bounded by the decoded startup journal.
|
||||
retained_replay_intents: HashMap<MrfQueueKey, MrfIntent>,
|
||||
/// Earliest instant a full-admission retry may proceed.
|
||||
backoff_until: Option<tokio::time::Instant>,
|
||||
}
|
||||
|
||||
impl MrfRuntime {
|
||||
fn snapshot(&self) -> (Vec<u8>, Vec<u8>) {
|
||||
fn enqueue_batch(&mut self, intents: impl IntoIterator<Item = MrfIntent>) -> usize {
|
||||
// Reserve retained startup work before admitting live hints. Computing
|
||||
// the union once per batch avoids scanning it for every incoming hint.
|
||||
let mut snapshot_count = self.queue.depth();
|
||||
let mut snapshot_bytes = self.queue.bytes();
|
||||
for (key, intent) in &self.retained_replay_intents {
|
||||
if !self.queue.pending_keys.contains(key) {
|
||||
snapshot_count = snapshot_count.saturating_add(1);
|
||||
snapshot_bytes = snapshot_bytes.saturating_add(intent.estimated_bytes());
|
||||
}
|
||||
}
|
||||
let mut enqueued = 0;
|
||||
for intent in intents {
|
||||
let key = queue_key(&intent);
|
||||
let retained = self.retained_replay_intents.get(&key);
|
||||
let additional = retained.is_none() && !self.queue.pending_keys.contains(&key);
|
||||
let next_count = snapshot_count.saturating_add(usize::from(additional));
|
||||
let next_bytes = snapshot_bytes.saturating_add(if additional { intent.estimated_bytes() } else { 0 });
|
||||
let result = if retained.is_some_and(|retained| retained.lease != intent.lease)
|
||||
|| next_count > self.queue.capacity
|
||||
|| next_bytes > self.queue.byte_budget
|
||||
{
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
|
||||
MrfQueuePushResult::Rejected
|
||||
} else {
|
||||
self.queue.try_push_typed(intent.clone())
|
||||
};
|
||||
match result {
|
||||
MrfQueuePushResult::Enqueued => {
|
||||
snapshot_count = next_count;
|
||||
snapshot_bytes = next_bytes;
|
||||
enqueued += 1;
|
||||
self.new_since_flush += 1;
|
||||
self.dirty = true;
|
||||
}
|
||||
MrfQueuePushResult::Coalesced | MrfQueuePushResult::Rejected => {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
}
|
||||
}
|
||||
enqueued
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> Option<(Vec<u8>, Vec<u8>)> {
|
||||
let mut authoritative = Vec::new();
|
||||
let mut legacy = Vec::new();
|
||||
for intent in self.queue.intents() {
|
||||
let mut encoded = HashSet::new();
|
||||
for intent in self.queue.intents().chain(self.retained_replay_intents.values()) {
|
||||
let key = queue_key(intent);
|
||||
if self
|
||||
.retained_replay_intents
|
||||
.get(&key)
|
||||
.is_some_and(|retained| retained.lease != intent.lease)
|
||||
{
|
||||
// Legacy records cannot distinguish concurrent responsibilities
|
||||
// with different leases. Preserve the existing disk anchor.
|
||||
return None;
|
||||
}
|
||||
if !encoded.insert(key) {
|
||||
continue;
|
||||
}
|
||||
if encoded.len() > self.queue.capacity {
|
||||
return None;
|
||||
}
|
||||
let scoped_identity =
|
||||
!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) && intent.scope.is_some();
|
||||
if !encode_intent(intent, &mut authoritative) {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
|
||||
return None;
|
||||
}
|
||||
if authoritative.len() > self.queue.byte_budget {
|
||||
return None;
|
||||
}
|
||||
if !scoped_identity && !encode_intent(intent, &mut legacy) {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
|
||||
return None;
|
||||
}
|
||||
}
|
||||
(authoritative, legacy)
|
||||
Some((authoritative, legacy))
|
||||
}
|
||||
|
||||
async fn flush(&mut self) {
|
||||
let (authoritative, legacy) = self.snapshot();
|
||||
let Some((authoritative, legacy)) = self.snapshot() else {
|
||||
self.dirty = true;
|
||||
return;
|
||||
};
|
||||
let authoritative_persisted = write_journal(MRF_SCOPED_JOURNAL_PATH, &authoritative).await;
|
||||
if !authoritative.is_empty() {
|
||||
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
|
||||
@@ -588,11 +669,31 @@ impl MrfRuntime {
|
||||
// attempts counter) changes the encoded snapshot; mark it dirty
|
||||
// either way.
|
||||
self.dirty = true;
|
||||
let replay_key = queue_key(&intent);
|
||||
let replayed = self
|
||||
.retained_replay_intents
|
||||
.get(&replay_key)
|
||||
.is_some_and(|retained| retained.lease == intent.lease);
|
||||
if replayed {
|
||||
if !matches!(
|
||||
rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut intent),
|
||||
MrfIngressResult::Enqueued
|
||||
) {
|
||||
self.retain_replay_journal = true;
|
||||
self.queue.push_back(intent);
|
||||
self.backoff_until = Some(tokio::time::Instant::now() + self.config.admission_backoff);
|
||||
break;
|
||||
}
|
||||
self.retained_replay_intents.insert(replay_key, intent.clone());
|
||||
}
|
||||
match submit_mrf_heal_request(manager, &intent).await {
|
||||
// Accepted intents leave the pending set; the next flush persists the
|
||||
// smaller snapshot. This is not a durable successor receipt and
|
||||
// does not discharge the producer's existing retry hints.
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
// Admission removes executable work from the pending queue,
|
||||
// but startup responsibilities still require an exact proof.
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {
|
||||
if replayed && let Some(anchor) = manager.durable_mrf_repair_anchor(&intent).await {
|
||||
self.durable_replay_anchors.push(anchor);
|
||||
}
|
||||
}
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts >= MRF_MAX_ATTEMPTS {
|
||||
@@ -624,6 +725,37 @@ impl MrfRuntime {
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(self.queue.depth()));
|
||||
gauge!("rustfs_heal_mrf_queue_bytes").set(metric_f64(self.queue.bytes()));
|
||||
}
|
||||
|
||||
fn retained_replay_journal(&self) -> bool {
|
||||
self.retain_replay_journal || !self.retained_replay_intents.is_empty()
|
||||
}
|
||||
|
||||
fn discharge_durable_replay_anchors(&mut self) {
|
||||
if self.durable_replay_anchors.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut discharged_leases: HashSet<_> = self.durable_replay_anchors.iter().map(|anchor| anchor.lease).collect();
|
||||
let mut buckets: Vec<Arc<str>> = self
|
||||
.durable_replay_anchors
|
||||
.iter()
|
||||
.map(|anchor| anchor.bucket.clone())
|
||||
.collect();
|
||||
buckets.sort_unstable();
|
||||
buckets.dedup();
|
||||
for bucket in buckets {
|
||||
rustfs_common::mrf_channel::consume_recorded_verified_mrf_repair_events_for(
|
||||
bucket.as_ref(),
|
||||
&mut self.durable_replay_anchors,
|
||||
);
|
||||
}
|
||||
for anchor in &self.durable_replay_anchors {
|
||||
discharged_leases.remove(&anchor.lease);
|
||||
}
|
||||
let before = self.retained_replay_intents.len();
|
||||
self.retained_replay_intents
|
||||
.retain(|_, intent| !intent.lease.is_some_and(|lease| discharged_leases.contains(&lease)));
|
||||
self.dirty |= self.retained_replay_intents.len() != before;
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize the global MRF channel (honoring `RUSTFS_HEAL_MRF_ENABLE`) and
|
||||
@@ -673,10 +805,73 @@ pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
|
||||
struct ReplayOutcome {
|
||||
replayed: usize,
|
||||
journal_on_disk: bool,
|
||||
retain_journal_for_replay: bool,
|
||||
durable_replay_anchors: Vec<MrfDurableRepairAnchor>,
|
||||
retained_replay_intents: HashMap<MrfQueueKey, MrfIntent>,
|
||||
}
|
||||
|
||||
fn replay_must_retain_journal(rearm_incomplete: bool, pending_depth: usize) -> bool {
|
||||
rearm_incomplete || pending_depth > 0
|
||||
fn replay_must_retain_journal(
|
||||
rearm_incomplete: bool,
|
||||
pending_depth: usize,
|
||||
accepted_without_durable_anchor: bool,
|
||||
durable_replay_anchors: usize,
|
||||
) -> bool {
|
||||
rearm_incomplete || pending_depth > 0 || accepted_without_durable_anchor || durable_replay_anchors > 0
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum ReplayCleanup {
|
||||
Legacy,
|
||||
Committed { sequence: u64 },
|
||||
}
|
||||
|
||||
struct ReplaySource {
|
||||
data: Vec<u8>,
|
||||
cleanup: ReplayCleanup,
|
||||
}
|
||||
|
||||
async fn read_replay_source(max_bytes: usize) -> Result<Option<ReplaySource>, snapshot::SnapshotError> {
|
||||
if let Some(committed) = snapshot::inspect_local_committed_snapshot(max_bytes).await? {
|
||||
return Ok(Some(ReplaySource {
|
||||
data: committed.payload().to_vec(),
|
||||
cleanup: ReplayCleanup::Committed {
|
||||
sequence: committed.sequence(),
|
||||
},
|
||||
}));
|
||||
}
|
||||
// The scoped file is a complete authoritative legacy snapshot. Fall back
|
||||
// to the v1 mirror only when the authoritative path is unavailable;
|
||||
// merging both files could combine records from different flush epochs.
|
||||
let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await {
|
||||
Some(data) => data,
|
||||
None => match read_journal(MRF_JOURNAL_PATH).await {
|
||||
Some(data) => data,
|
||||
None => return Ok(None),
|
||||
},
|
||||
};
|
||||
Ok(Some(ReplaySource {
|
||||
data,
|
||||
cleanup: ReplayCleanup::Legacy,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn delete_replay_source(cleanup: ReplayCleanup, max_bytes: usize) -> bool {
|
||||
let committed_deleted = match cleanup {
|
||||
ReplayCleanup::Legacy => true,
|
||||
ReplayCleanup::Committed { sequence } => match snapshot::delete_committed_snapshots_through(sequence, max_bytes).await {
|
||||
Ok(deleted) => deleted,
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
target: "rustfs::heal::mrf",
|
||||
error = %err,
|
||||
sequence,
|
||||
"MRF committed replay checkpoint cleanup failed"
|
||||
);
|
||||
false
|
||||
}
|
||||
},
|
||||
};
|
||||
committed_deleted && delete_journals().await
|
||||
}
|
||||
|
||||
/// Shared replay core: read + decode + re-arm, then drain what fits. The
|
||||
@@ -687,21 +882,34 @@ async fn replay_into(
|
||||
queue: &mut MrfQueue,
|
||||
backoff_until: &mut Option<tokio::time::Instant>,
|
||||
) -> ReplayOutcome {
|
||||
// The scoped file is a complete authoritative snapshot. Fall back to the
|
||||
// legacy mirror only when the authoritative path is unavailable; merging
|
||||
// both files could combine records from different flush epochs.
|
||||
let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await {
|
||||
Some(data) => data,
|
||||
None => match read_journal(MRF_JOURNAL_PATH).await {
|
||||
Some(data) => data,
|
||||
None => {
|
||||
return ReplayOutcome {
|
||||
replayed: 0,
|
||||
journal_on_disk: false,
|
||||
};
|
||||
}
|
||||
},
|
||||
let source = match read_replay_source(queue.byte_budget).await {
|
||||
Ok(Some(source)) => source,
|
||||
Ok(None) => {
|
||||
return ReplayOutcome {
|
||||
replayed: 0,
|
||||
journal_on_disk: false,
|
||||
retain_journal_for_replay: false,
|
||||
durable_replay_anchors: Vec::new(),
|
||||
retained_replay_intents: HashMap::new(),
|
||||
};
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
target: "rustfs::heal::mrf",
|
||||
error = %err,
|
||||
"MRF committed replay checkpoint could not be inspected"
|
||||
);
|
||||
return ReplayOutcome {
|
||||
replayed: 0,
|
||||
journal_on_disk: true,
|
||||
retain_journal_for_replay: true,
|
||||
durable_replay_anchors: Vec::new(),
|
||||
retained_replay_intents: HashMap::new(),
|
||||
};
|
||||
}
|
||||
};
|
||||
let cleanup = source.cleanup;
|
||||
let data = source.data;
|
||||
let (decoded, truncated) = decode_journal(&data);
|
||||
let replayed = decoded.len();
|
||||
let intents = decoded;
|
||||
@@ -722,6 +930,8 @@ async fn replay_into(
|
||||
// prefix.
|
||||
queue.raise_limits_for_replay(intents.len(), replay_bytes);
|
||||
let mut rearm_incomplete = false;
|
||||
let mut accepted_without_durable_anchor = false;
|
||||
let mut durable_replay_anchors = Vec::new();
|
||||
for intent in intents {
|
||||
let result = queue.try_push_typed(intent.clone());
|
||||
match result {
|
||||
@@ -733,6 +943,7 @@ async fn replay_into(
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut retained_replay_intents: HashMap<_, _> = queue.intents().map(|intent| (queue_key(intent), intent.clone())).collect();
|
||||
|
||||
// Drain the replayed intents immediately; whatever the manager refuses
|
||||
// stays armed in `queue` for the consumer's retry loop.
|
||||
@@ -747,8 +958,15 @@ async fn replay_into(
|
||||
*backoff_until = Some(tokio::time::Instant::now());
|
||||
break;
|
||||
}
|
||||
retained_replay_intents.insert(queue_key(&intent), intent.clone());
|
||||
match submit_mrf_heal_request(manager, &intent).await {
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {
|
||||
if let Some(anchor) = manager.durable_mrf_repair_anchor(&intent).await {
|
||||
durable_replay_anchors.push(anchor);
|
||||
} else {
|
||||
accepted_without_durable_anchor = true;
|
||||
}
|
||||
}
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts < MRF_MAX_ATTEMPTS {
|
||||
@@ -762,6 +980,7 @@ async fn replay_into(
|
||||
break;
|
||||
}
|
||||
Ok(HealAdmissionResult::Dropped(_)) => {
|
||||
retained_replay_intents.remove(&queue_key(&intent));
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
Err(_) => {
|
||||
@@ -779,14 +998,24 @@ async fn replay_into(
|
||||
}
|
||||
}
|
||||
}
|
||||
let journal_on_disk = if replay_must_retain_journal(rearm_incomplete, queue.depth()) {
|
||||
let must_retain_journal = replay_must_retain_journal(
|
||||
rearm_incomplete,
|
||||
queue.depth(),
|
||||
accepted_without_durable_anchor,
|
||||
durable_replay_anchors.len(),
|
||||
);
|
||||
let retain_journal_for_replay = rearm_incomplete || accepted_without_durable_anchor;
|
||||
let journal_on_disk = if must_retain_journal {
|
||||
true
|
||||
} else {
|
||||
!delete_journals().await
|
||||
!delete_replay_source(cleanup, queue.byte_budget).await
|
||||
};
|
||||
ReplayOutcome {
|
||||
replayed,
|
||||
journal_on_disk,
|
||||
retain_journal_for_replay,
|
||||
durable_replay_anchors,
|
||||
retained_replay_intents,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -800,6 +1029,9 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: false,
|
||||
retain_replay_journal: false,
|
||||
durable_replay_anchors: Vec::new(),
|
||||
retained_replay_intents: HashMap::new(),
|
||||
backoff_until: None,
|
||||
};
|
||||
|
||||
@@ -807,6 +1039,9 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
// on disk whenever any replayed intent still needs a successor snapshot.
|
||||
let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
||||
runtime.journal_on_disk = replay.journal_on_disk;
|
||||
runtime.retain_replay_journal = replay.retain_journal_for_replay;
|
||||
runtime.durable_replay_anchors = replay.durable_replay_anchors;
|
||||
runtime.retained_replay_intents = replay.retained_replay_intents;
|
||||
// Anything still pending (e.g. the manager was full and backoff armed)
|
||||
// must be re-persisted by the next flush before replay can delete the
|
||||
// startup anchor.
|
||||
@@ -833,27 +1068,19 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
);
|
||||
return;
|
||||
}
|
||||
for intent in batch.drain(..) {
|
||||
match runtime.queue.try_push_typed(intent.clone()) {
|
||||
MrfQueuePushResult::Enqueued => {
|
||||
runtime.new_since_flush += 1;
|
||||
runtime.dirty = true;
|
||||
}
|
||||
MrfQueuePushResult::Coalesced | MrfQueuePushResult::Rejected => {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
}
|
||||
}
|
||||
runtime.enqueue_batch(batch.drain(..));
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
if runtime.new_since_flush >= runtime.config.flush_threshold {
|
||||
runtime.flush().await;
|
||||
}
|
||||
}
|
||||
_ = flush_tick.tick() => {
|
||||
runtime.discharge_durable_replay_anchors();
|
||||
match tick_action(
|
||||
runtime.dirty,
|
||||
runtime.queue.depth(),
|
||||
runtime.journal_on_disk,
|
||||
runtime.retained_replay_journal(),
|
||||
) {
|
||||
TickAction::Flush => {
|
||||
runtime.flush().await;
|
||||
@@ -897,12 +1124,12 @@ enum TickAction {
|
||||
Idle,
|
||||
}
|
||||
|
||||
fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction {
|
||||
fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool, retain_replay_journal: bool) -> TickAction {
|
||||
if dirty {
|
||||
TickAction::Flush
|
||||
} else if depth > 0 {
|
||||
TickAction::Retry
|
||||
} else if journal_on_disk {
|
||||
} else if journal_on_disk && !retain_replay_journal {
|
||||
TickAction::DeleteJournal
|
||||
} else {
|
||||
TickAction::Idle
|
||||
@@ -912,7 +1139,7 @@ fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_common::mrf_channel::{MrfIntent, MrfKind};
|
||||
use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfVerifiedRepairDisposition, MrfVerifiedRepairEvent};
|
||||
use std::sync::Arc as StdArc;
|
||||
|
||||
fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent {
|
||||
@@ -934,35 +1161,271 @@ mod tests {
|
||||
|
||||
// Dirty dominates: a changed pending set flushes even when idle
|
||||
// otherwise.
|
||||
assert!(matches!(tick_action(true, 0, false), Flush));
|
||||
assert!(matches!(tick_action(true, 3, true), Flush));
|
||||
assert!(matches!(tick_action(true, 0, false, false), Flush));
|
||||
assert!(matches!(tick_action(true, 3, true, false), Flush));
|
||||
|
||||
// Clean backlog: no rewrite, but keep draining so an expired
|
||||
// admission backoff retries on time.
|
||||
assert!(matches!(tick_action(false, 1, false), Retry));
|
||||
assert!(matches!(tick_action(false, 2, true), Retry));
|
||||
assert!(matches!(tick_action(false, 1, false, false), Retry));
|
||||
assert!(matches!(tick_action(false, 2, true, false), Retry));
|
||||
|
||||
// Quiescent with a stale journal file on disk: remove it.
|
||||
assert!(matches!(tick_action(false, 0, true), DeleteJournal));
|
||||
assert!(matches!(tick_action(false, 0, true, false), DeleteJournal));
|
||||
assert!(matches!(tick_action(false, 0, true, true), Idle));
|
||||
|
||||
// Fully quiescent: nothing to do.
|
||||
assert!(matches!(tick_action(false, 0, false), Idle));
|
||||
assert!(matches!(tick_action(false, 0, false, false), Idle));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cleanup_retains_journal_for_unarmed_or_refused_records() {
|
||||
assert!(
|
||||
replay_must_retain_journal(true, 0),
|
||||
replay_must_retain_journal(true, 0, false, 0),
|
||||
"a rejected replay record still needs its disk anchor"
|
||||
);
|
||||
assert!(
|
||||
replay_must_retain_journal(false, 1),
|
||||
replay_must_retain_journal(false, 1, false, 0),
|
||||
"a Full admission retry must keep the startup journal until the next snapshot"
|
||||
);
|
||||
assert!(
|
||||
!replay_must_retain_journal(false, 0),
|
||||
"only a fully consumed replay snapshot may be deleted"
|
||||
!replay_must_retain_journal(false, 0, false, 0),
|
||||
"a fully consumed replay snapshot without accepts may be deleted"
|
||||
);
|
||||
assert!(
|
||||
replay_must_retain_journal(false, 0, true, 0),
|
||||
"accepted or merged replay records without a proof identity still need a durable successor"
|
||||
);
|
||||
assert!(
|
||||
replay_must_retain_journal(false, 0, false, 1),
|
||||
"accepted replay records with a durable proof anchor must retain the journal until proof arrives"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_releases_retained_replay_journal_after_verified_repair_proof() {
|
||||
let mut intent = intent("proof-bucket", "proof-object", 0);
|
||||
intent.kind = MrfKind::PartialWrite;
|
||||
assert_eq!(
|
||||
rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut intent),
|
||||
MrfIngressResult::Enqueued
|
||||
);
|
||||
let bucket_incarnation_id = uuid::Uuid::new_v4();
|
||||
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, bucket_incarnation_id)
|
||||
.expect("fresh replay lease and bucket incarnation build a durable anchor");
|
||||
let mut runtime = MrfRuntime {
|
||||
queue: MrfQueue::new(2, usize::MAX),
|
||||
config: MrfConsumerConfig::default(),
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: true,
|
||||
retain_replay_journal: false,
|
||||
durable_replay_anchors: vec![anchor],
|
||||
retained_replay_intents: HashMap::from([(queue_key(&intent), intent.clone())]),
|
||||
backoff_until: None,
|
||||
};
|
||||
rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent {
|
||||
kind: intent.kind,
|
||||
bucket: intent.bucket.clone(),
|
||||
object: intent.object.clone(),
|
||||
version_id: intent.version_id,
|
||||
scope: intent.scope,
|
||||
lease: intent.lease,
|
||||
bucket_incarnation_id,
|
||||
disposition: MrfVerifiedRepairDisposition::Repaired,
|
||||
});
|
||||
|
||||
assert!(
|
||||
runtime.retained_replay_journal(),
|
||||
"anchor must retain the startup journal before proof is consumed"
|
||||
);
|
||||
assert_eq!(
|
||||
decode_journal(&runtime.snapshot().expect("retained snapshot").0).0.len(),
|
||||
1,
|
||||
"an admitted responsibility must remain in the successor before proof"
|
||||
);
|
||||
runtime.discharge_durable_replay_anchors();
|
||||
assert!(
|
||||
!runtime.retained_replay_journal(),
|
||||
"matching verified proof discharges the durable replay anchor"
|
||||
);
|
||||
assert!(runtime.dirty, "proof removal must be persisted by the next flush");
|
||||
assert!(runtime.snapshot().expect("discharged snapshot").0.is_empty());
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_snapshot_retains_admitted_replay_and_pending_successor() {
|
||||
let accepted = intent("snapshot-bucket", "accepted", 0);
|
||||
let pending = intent("snapshot-bucket", "pending", 2);
|
||||
let mut queue = MrfQueue::new(4, 4096);
|
||||
assert!(queue.try_push(pending.clone()));
|
||||
let runtime = MrfRuntime {
|
||||
queue,
|
||||
config: MrfConsumerConfig::default(),
|
||||
new_since_flush: 0,
|
||||
dirty: true,
|
||||
journal_on_disk: true,
|
||||
retain_replay_journal: true,
|
||||
durable_replay_anchors: Vec::new(),
|
||||
retained_replay_intents: HashMap::from([
|
||||
(queue_key(&accepted), accepted),
|
||||
(queue_key(&pending), intent("snapshot-bucket", "pending", 0)),
|
||||
]),
|
||||
backoff_until: None,
|
||||
};
|
||||
|
||||
let (authoritative, legacy) = runtime.snapshot().expect("complete successor should fit");
|
||||
for snapshot in [authoritative, legacy] {
|
||||
let (recovered, truncated) = decode_journal(&snapshot);
|
||||
assert_eq!(truncated, 0);
|
||||
assert_eq!(recovered.len(), 2, "admission must not discard an unproven startup responsibility");
|
||||
assert!(recovered.iter().any(|intent| intent.object.as_ref() == "accepted"));
|
||||
assert!(
|
||||
recovered
|
||||
.iter()
|
||||
.any(|intent| intent.object.as_ref() == "pending" && intent.attempts == 2)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_snapshot_preserves_anchor_when_successor_exceeds_budget_or_changes_lease() {
|
||||
let mut retained = intent("bounded-bucket", "retained", 0);
|
||||
assert_eq!(
|
||||
rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut retained),
|
||||
MrfIngressResult::Enqueued
|
||||
);
|
||||
let mut queue = MrfQueue::new(1, 4096);
|
||||
assert!(queue.try_push(intent("bounded-bucket", "pending", 0)));
|
||||
let mut runtime = MrfRuntime {
|
||||
queue,
|
||||
config: MrfConsumerConfig::default(),
|
||||
new_since_flush: 0,
|
||||
dirty: true,
|
||||
journal_on_disk: true,
|
||||
retain_replay_journal: false,
|
||||
durable_replay_anchors: Vec::new(),
|
||||
retained_replay_intents: HashMap::from([(queue_key(&retained), retained.clone())]),
|
||||
backoff_until: None,
|
||||
};
|
||||
assert!(runtime.snapshot().is_none(), "combined count must honor the queue ceiling");
|
||||
runtime.queue.capacity = 2;
|
||||
runtime.queue.byte_budget = 1;
|
||||
assert!(runtime.snapshot().is_none(), "oversized successor must not replace the startup journal");
|
||||
|
||||
runtime.queue = MrfQueue::new(2, 4096);
|
||||
let mut newer = retained.clone();
|
||||
newer.lease = None;
|
||||
assert_eq!(
|
||||
rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut newer),
|
||||
MrfIngressResult::Enqueued
|
||||
);
|
||||
assert_ne!(retained.lease, newer.lease);
|
||||
assert!(runtime.queue.try_push(newer));
|
||||
assert!(
|
||||
runtime.snapshot().is_none(),
|
||||
"legacy encoding cannot conflate distinct responsibilities for one object"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_admission_reserves_replay_budget_until_verified_repair() {
|
||||
for count_limited in [true, false] {
|
||||
let mut retained = intent("reserved-bucket", "retained", 0);
|
||||
assert_eq!(
|
||||
rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut retained),
|
||||
MrfIngressResult::Enqueued
|
||||
);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let anchor = MrfDurableRepairAnchor::from_intent(&retained, incarnation).expect("replay anchor");
|
||||
let mut runtime = MrfRuntime {
|
||||
queue: MrfQueue::new(if count_limited { 1 } else { 2 }, retained.estimated_bytes()),
|
||||
config: MrfConsumerConfig::default(),
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: true,
|
||||
retain_replay_journal: false,
|
||||
durable_replay_anchors: vec![anchor],
|
||||
retained_replay_intents: HashMap::from([(queue_key(&retained), retained.clone())]),
|
||||
backoff_until: None,
|
||||
};
|
||||
if count_limited {
|
||||
runtime.queue.byte_budget = 4096;
|
||||
}
|
||||
let pending = intent("reserved-bucket", "pending", 0);
|
||||
assert_eq!(runtime.enqueue_batch([pending.clone()]), 0, "retained work consumes admission budget");
|
||||
assert_eq!(runtime.queue.depth(), 0);
|
||||
let (snapshot, _) = runtime.snapshot().expect("rejection must leave a writable retained snapshot");
|
||||
let (recovered, truncated) = decode_journal(&snapshot);
|
||||
assert_eq!(truncated, 0);
|
||||
assert_eq!(recovered.len(), 1);
|
||||
assert_eq!(recovered[0].object.as_ref(), "retained");
|
||||
|
||||
rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent {
|
||||
kind: retained.kind,
|
||||
bucket: retained.bucket.clone(),
|
||||
object: retained.object.clone(),
|
||||
version_id: retained.version_id,
|
||||
scope: retained.scope,
|
||||
lease: retained.lease,
|
||||
bucket_incarnation_id: incarnation,
|
||||
disposition: MrfVerifiedRepairDisposition::Repaired,
|
||||
});
|
||||
runtime.discharge_durable_replay_anchors();
|
||||
assert_eq!(runtime.enqueue_batch([pending]), 1, "proof must release admission capacity for retry");
|
||||
let (snapshot, _) = runtime.snapshot().expect("new admitted work must be persistable");
|
||||
let (recovered, truncated) = decode_journal(&snapshot);
|
||||
assert_eq!(truncated, 0);
|
||||
assert_eq!(recovered.len(), 1);
|
||||
assert_eq!(recovered[0].object.as_ref(), "pending");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_admission_rejects_new_lease_without_blocking_other_successors() {
|
||||
let mut retained = intent("lease-bucket", "retained", 0);
|
||||
assert_eq!(
|
||||
rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut retained),
|
||||
MrfIngressResult::Enqueued
|
||||
);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let anchor = MrfDurableRepairAnchor::from_intent(&retained, incarnation).expect("replay anchor");
|
||||
let mut runtime = MrfRuntime {
|
||||
queue: MrfQueue::new(2, 4096),
|
||||
config: MrfConsumerConfig::default(),
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: true,
|
||||
retain_replay_journal: false,
|
||||
durable_replay_anchors: vec![anchor],
|
||||
retained_replay_intents: HashMap::from([(queue_key(&retained), retained.clone())]),
|
||||
backoff_until: None,
|
||||
};
|
||||
let mut newer = retained.clone();
|
||||
newer.lease = None;
|
||||
assert_eq!(
|
||||
rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut newer),
|
||||
MrfIngressResult::Enqueued
|
||||
);
|
||||
assert_ne!(newer.lease, retained.lease);
|
||||
assert_eq!(runtime.enqueue_batch([newer.clone(), intent("lease-bucket", "pending", 0)]), 1);
|
||||
assert_eq!(runtime.queue.intents().next().expect("unrelated successor").object.as_ref(), "pending");
|
||||
assert_eq!(decode_journal(&runtime.snapshot().expect("unblocked successor").0).0.len(), 2);
|
||||
|
||||
rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent {
|
||||
kind: retained.kind,
|
||||
bucket: retained.bucket.clone(),
|
||||
object: retained.object.clone(),
|
||||
version_id: retained.version_id,
|
||||
scope: retained.scope,
|
||||
lease: retained.lease,
|
||||
bucket_incarnation_id: incarnation,
|
||||
disposition: MrfVerifiedRepairDisposition::Repaired,
|
||||
});
|
||||
runtime.discharge_durable_replay_anchors();
|
||||
assert_eq!(runtime.enqueue_batch([newer.clone()]), 1);
|
||||
assert!(runtime.queue.intents().any(|intent| intent.lease == newer.lease));
|
||||
assert_eq!(decode_journal(&runtime.snapshot().expect("new lease successor").0).0.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -999,6 +1462,46 @@ mod tests {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&replay);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_replay_canonicalization_preserves_one_bounded_responsibility() {
|
||||
let mut legacy = intent("metadata-replay-bucket", "object", 0);
|
||||
legacy.kind = MrfKind::MetadataCorruption;
|
||||
let mut bytes = Vec::new();
|
||||
assert!(encode_intent(&legacy, &mut bytes));
|
||||
let (mut decoded, truncated) = decode_journal(&bytes);
|
||||
assert_eq!(truncated, 0);
|
||||
let mut replay = decoded.pop().expect("legacy metadata record");
|
||||
assert!(replay.version_id.is_some(), "the legacy wire record carries an ignored version");
|
||||
let original_key = queue_key(&replay);
|
||||
let mut retained_replay_intents = HashMap::from([(original_key.clone(), replay.clone())]);
|
||||
assert_eq!(
|
||||
rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut replay),
|
||||
MrfIngressResult::Enqueued
|
||||
);
|
||||
assert!(replay.version_id.is_none());
|
||||
assert_eq!(queue_key(&replay), original_key, "rearm must not create another retained key");
|
||||
retained_replay_intents.insert(queue_key(&replay), replay);
|
||||
assert_eq!(retained_replay_intents.len(), 1);
|
||||
let runtime = MrfRuntime {
|
||||
queue: MrfQueue::new(1, 4096),
|
||||
config: MrfConsumerConfig::default(),
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: true,
|
||||
retain_replay_journal: true,
|
||||
durable_replay_anchors: Vec::new(),
|
||||
retained_replay_intents,
|
||||
backoff_until: None,
|
||||
};
|
||||
let (snapshot, _) = runtime
|
||||
.snapshot()
|
||||
.expect("canonical metadata fits the original one-record budget");
|
||||
let (recovered, truncated) = decode_journal(&snapshot);
|
||||
assert_eq!(truncated, 0);
|
||||
assert_eq!(recovered.len(), 1);
|
||||
assert!(recovered[0].version_id.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_can_arm_more_records_than_live_queue_budget() {
|
||||
let mut queue = MrfQueue::new(1, intent("bucket", "object-0", 0).estimated_bytes());
|
||||
|
||||
@@ -17,17 +17,21 @@
|
||||
//! Each of two slots has a payload and a commit manifest. The manifest binds
|
||||
//! the writer identity, persistent sequence, length and whole-payload digest.
|
||||
//! Replacing the inactive slot must leave the previous committed slot intact.
|
||||
//! Production publication and reclamation are deliberately not enabled here.
|
||||
//! Production publication is deliberately reader-first; reclamation only
|
||||
//! removes manifest entries after the owning replay path has discharged every
|
||||
//! responsibility through a newer durable snapshot or a verified repair proof.
|
||||
//! An unreadable commit path cannot prove that only legacy data exists. This
|
||||
//! explicit inspection API fails closed and never mutates recovery anchors.
|
||||
//! It is not wired into the legacy consumer: that transition requires the
|
||||
//! ownership-aware replay and producer handoff before writer activation.
|
||||
//! It is wired into the replay reader before writer activation, but the writer
|
||||
//! remains gated on ownership-aware handoff.
|
||||
//! One surviving committed replica supports process restart recovery only;
|
||||
//! this reader does not establish a replication quorum or a power-loss policy.
|
||||
|
||||
use super::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_journal};
|
||||
use crate::heal::RUSTFS_META_BUCKET;
|
||||
use crate::heal::storage_api::owner::{EcstoreDiskAPI, EcstoreDiskError, EcstoreDiskStore};
|
||||
use crate::heal::storage_api::owner::{
|
||||
EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskStore,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -54,6 +58,8 @@ pub enum SnapshotError {
|
||||
TooLarge,
|
||||
#[error("MRF checkpoint replicas disagree at the same sequence")]
|
||||
Conflict,
|
||||
#[error("MRF checkpoint has no writable replica")]
|
||||
NoWritableReplica,
|
||||
#[error("MRF checkpoint storage is unavailable")]
|
||||
Disk(#[source] EcstoreDiskError),
|
||||
#[error("MRF checkpoint body could not be read")]
|
||||
@@ -121,6 +127,7 @@ impl Manifest {
|
||||
pub struct CommittedSnapshot {
|
||||
manifest: Manifest,
|
||||
payload: Vec<u8>,
|
||||
slot: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -141,13 +148,18 @@ impl CommittedSnapshot {
|
||||
self.manifest.owner
|
||||
}
|
||||
|
||||
/// Slot that supplied this committed checkpoint.
|
||||
pub fn slot(&self) -> usize {
|
||||
self.slot
|
||||
}
|
||||
|
||||
/// Complete, checksum-validated record bytes. Inspection does not consume
|
||||
/// these records or acknowledge completion to any producer.
|
||||
pub fn payload(&self) -> &[u8] {
|
||||
&self.payload
|
||||
}
|
||||
|
||||
fn decode(manifest: &[u8], payload: Vec<u8>, limit: usize) -> Result<Self, SnapshotError> {
|
||||
fn decode(slot: usize, manifest: &[u8], payload: Vec<u8>, limit: usize) -> Result<Self, SnapshotError> {
|
||||
let manifest = Manifest::decode(manifest, limit)?;
|
||||
let checksum: [u8; 32] = Sha256::digest(&payload).into();
|
||||
if payload.len() != manifest.payload_len || checksum != manifest.payload_digest {
|
||||
@@ -156,10 +168,29 @@ impl CommittedSnapshot {
|
||||
if decode_journal(&payload).1 != 0 {
|
||||
return Err(SnapshotError::Corrupt);
|
||||
}
|
||||
Ok(Self { manifest, payload })
|
||||
Ok(Self { manifest, payload, slot })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotPublication {
|
||||
pub owner: Uuid,
|
||||
pub sequence: u64,
|
||||
pub slot: usize,
|
||||
pub payload_len: usize,
|
||||
pub payload_replicas: usize,
|
||||
pub manifest_replicas: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotReclamation {
|
||||
pub owner: Uuid,
|
||||
pub sequence: u64,
|
||||
pub reclaimed_slot: usize,
|
||||
pub manifest_replicas: usize,
|
||||
pub payload_replicas: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum RecoverySnapshot {
|
||||
/// An intact legacy snapshot, without a comparable commit sequence.
|
||||
@@ -230,7 +261,7 @@ async fn read_committed_with_stats(
|
||||
let mut damaged = None;
|
||||
let mut identities = HashMap::new();
|
||||
for disk in disks {
|
||||
for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) {
|
||||
for (slot, (manifest_path, payload_path)) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS).enumerate() {
|
||||
let candidate = async {
|
||||
let Some(manifest) = read_bounded_with_stats(disk, manifest_path, MANIFEST_LEN, stats.as_deref_mut()).await?
|
||||
else {
|
||||
@@ -240,7 +271,7 @@ async fn read_committed_with_stats(
|
||||
let payload = read_bounded_with_stats(disk, payload_path, header.payload_len, stats.as_deref_mut())
|
||||
.await?
|
||||
.ok_or(SnapshotError::Corrupt)?;
|
||||
CommittedSnapshot::decode(&manifest, payload, limit).map(Some)
|
||||
CommittedSnapshot::decode(slot, &manifest, payload, limit).map(Some)
|
||||
}
|
||||
.await;
|
||||
match candidate {
|
||||
@@ -272,6 +303,199 @@ async fn read_committed_with_stats(
|
||||
}
|
||||
}
|
||||
|
||||
async fn cas_replace(
|
||||
disk: &EcstoreDiskStore,
|
||||
path: &str,
|
||||
replacement: &[u8],
|
||||
limit: usize,
|
||||
) -> Result<EcstoreConditionalFileUpdate, SnapshotError> {
|
||||
let expected = read_bounded(disk, path, limit).await?.map(EcstoreDiskBytes::from);
|
||||
cas_replace_expected(disk, path, expected, replacement).await
|
||||
}
|
||||
|
||||
async fn cas_replace_expected(
|
||||
disk: &EcstoreDiskStore,
|
||||
path: &str,
|
||||
expected: Option<EcstoreDiskBytes>,
|
||||
replacement: &[u8],
|
||||
) -> Result<EcstoreConditionalFileUpdate, SnapshotError> {
|
||||
EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
expected,
|
||||
Some(EcstoreDiskBytes::copy_from_slice(replacement)),
|
||||
)
|
||||
.await
|
||||
.map_err(SnapshotError::Disk)
|
||||
}
|
||||
|
||||
async fn cas_delete_expected(
|
||||
disk: &EcstoreDiskStore,
|
||||
path: &str,
|
||||
expected: EcstoreDiskBytes,
|
||||
) -> Result<EcstoreConditionalFileUpdate, SnapshotError> {
|
||||
EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, path, Some(expected), None)
|
||||
.await
|
||||
.map_err(SnapshotError::Disk)
|
||||
}
|
||||
|
||||
fn validate_reusable_manifest_slot(existing: Option<&[u8]>, sequence: u64, payload_limit: usize) -> Result<(), SnapshotError> {
|
||||
let Some(existing) = existing else {
|
||||
return Ok(());
|
||||
};
|
||||
let manifest = Manifest::decode(existing, payload_limit)?;
|
||||
if manifest.sequence >= sequence {
|
||||
return Err(SnapshotError::Conflict);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Publish a committed checkpoint into the inactive slot.
|
||||
///
|
||||
/// The writer is a narrow production primitive for the ownership-aware MRF
|
||||
/// handoff: it validates the whole journal payload, preserves the previous
|
||||
/// committed slot, and publishes the manifest only after the successor payload
|
||||
/// reaches the same disk. It does not delete legacy journals, tombstone older
|
||||
/// anchors, or activate the live consumer.
|
||||
pub async fn publish_committed_snapshot(
|
||||
disks: &[EcstoreDiskStore],
|
||||
owner: Uuid,
|
||||
sequence: u64,
|
||||
payload: &[u8],
|
||||
limit: usize,
|
||||
) -> Result<SnapshotPublication, SnapshotError> {
|
||||
if disks.is_empty() {
|
||||
return Err(SnapshotError::NoWritableReplica);
|
||||
}
|
||||
if owner.is_nil() || sequence == 0 || sequence == u64::MAX {
|
||||
return Err(SnapshotError::Corrupt);
|
||||
}
|
||||
if payload.len() > limit || decode_journal(payload).1 != 0 {
|
||||
return Err(SnapshotError::Corrupt);
|
||||
}
|
||||
let current = read_committed(disks, limit).await?;
|
||||
if current.as_ref().is_some_and(|snapshot| snapshot.sequence() >= sequence) {
|
||||
return Err(SnapshotError::Conflict);
|
||||
}
|
||||
let slot = current.as_ref().map_or(0, |snapshot| 1usize.saturating_sub(snapshot.slot()));
|
||||
let manifest = Manifest::encode(owner, sequence, payload)?;
|
||||
let mut payload_replicas = 0usize;
|
||||
let mut manifest_replicas = 0usize;
|
||||
let mut first_error = None;
|
||||
for disk in disks {
|
||||
let expected_manifest = match read_bounded(disk, MANIFEST_PATHS[slot], MANIFEST_LEN).await {
|
||||
Ok(expected) => expected,
|
||||
Err(error) => {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(error);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Err(error) = validate_reusable_manifest_slot(expected_manifest.as_deref(), sequence, limit) {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(error);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match cas_replace(disk, PAYLOAD_PATHS[slot], payload, limit).await {
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => payload_replicas += 1,
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue,
|
||||
Err(error) => {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(error);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
match cas_replace_expected(disk, MANIFEST_PATHS[slot], expected_manifest.map(EcstoreDiskBytes::from), &manifest).await {
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => manifest_replicas += 1,
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => {}
|
||||
Err(error) => {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if manifest_replicas == 0 {
|
||||
return Err(first_error.unwrap_or(SnapshotError::NoWritableReplica));
|
||||
}
|
||||
Ok(SnapshotPublication {
|
||||
owner,
|
||||
sequence,
|
||||
slot,
|
||||
payload_len: payload.len(),
|
||||
payload_replicas,
|
||||
manifest_replicas,
|
||||
})
|
||||
}
|
||||
|
||||
/// Reclaim the slot superseded by an already committed checkpoint.
|
||||
///
|
||||
/// This is a narrow cleanup primitive: it first reads back the current
|
||||
/// committed checkpoint and only removes the opposite slot when that slot is a
|
||||
/// complete, older checkpoint for the same owner. Incomplete, damaged, equal or
|
||||
/// newer evidence is retained.
|
||||
pub async fn reclaim_committed_snapshot_predecessor(
|
||||
disks: &[EcstoreDiskStore],
|
||||
owner: Uuid,
|
||||
sequence: u64,
|
||||
limit: usize,
|
||||
) -> Result<SnapshotReclamation, SnapshotError> {
|
||||
let current = read_committed(disks, limit).await?.ok_or(SnapshotError::Conflict)?;
|
||||
if current.owner() != owner || current.sequence() != sequence {
|
||||
return Err(SnapshotError::Conflict);
|
||||
}
|
||||
let reclaimed_slot = 1usize.saturating_sub(current.slot());
|
||||
let mut manifest_replicas = 0usize;
|
||||
let mut payload_replicas = 0usize;
|
||||
for disk in disks {
|
||||
let Some(manifest_bytes) = read_bounded(disk, MANIFEST_PATHS[reclaimed_slot], MANIFEST_LEN).await? else {
|
||||
continue;
|
||||
};
|
||||
let manifest = match Manifest::decode(&manifest_bytes, limit) {
|
||||
Ok(manifest) if manifest.owner == owner && manifest.sequence < sequence => manifest,
|
||||
Ok(_) | Err(SnapshotError::Corrupt) | Err(SnapshotError::TooLarge) => continue,
|
||||
Err(SnapshotError::Unsupported) => return Err(SnapshotError::Unsupported),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let Some(payload_bytes) = read_bounded(disk, PAYLOAD_PATHS[reclaimed_slot], manifest.payload_len).await? else {
|
||||
continue;
|
||||
};
|
||||
if CommittedSnapshot::decode(reclaimed_slot, &manifest_bytes, payload_bytes.clone(), limit).is_err() {
|
||||
continue;
|
||||
}
|
||||
match cas_delete_expected(disk, MANIFEST_PATHS[reclaimed_slot], EcstoreDiskBytes::copy_from_slice(&manifest_bytes)).await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => manifest_replicas += 1,
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
PAYLOAD_PATHS[reclaimed_slot],
|
||||
Some(EcstoreDiskBytes::copy_from_slice(&payload_bytes)),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(SnapshotError::Disk)?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => payload_replicas += 1,
|
||||
EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch => {}
|
||||
}
|
||||
}
|
||||
Ok(SnapshotReclamation {
|
||||
owner,
|
||||
sequence,
|
||||
reclaimed_slot,
|
||||
manifest_replicas,
|
||||
payload_replicas,
|
||||
})
|
||||
}
|
||||
|
||||
async fn read_legacy(disks: &[EcstoreDiskStore], path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
|
||||
let mut selected = None;
|
||||
let mut incomplete: Option<Vec<u8>> = None;
|
||||
@@ -320,6 +544,91 @@ pub async fn inspect_local_recovery_snapshot(max_bytes: usize) -> Result<Option<
|
||||
read_recovery_snapshot(&super::journal_disks().await, max_bytes).await
|
||||
}
|
||||
|
||||
/// Inspect only committed MRF checkpoints.
|
||||
///
|
||||
/// The legacy replay path keeps its historical torn-tail behavior, so startup
|
||||
/// replay uses this narrower API to avoid turning a legacy torn tail into a
|
||||
/// committed-format failure. A corrupt or conflicting committed checkpoint is
|
||||
/// still authoritative: callers must fail closed instead of falling back to a
|
||||
/// stale legacy mirror.
|
||||
pub async fn inspect_local_committed_snapshot(max_bytes: usize) -> Result<Option<CommittedSnapshot>, SnapshotError> {
|
||||
read_committed(&super::journal_disks().await, max_bytes).await
|
||||
}
|
||||
|
||||
/// Remove committed manifests whose sequence is no newer than
|
||||
/// `committed_through`.
|
||||
///
|
||||
/// Payload files are intentionally left as orphans after their manifest is
|
||||
/// removed. Readers cannot discover a payload without its matching manifest,
|
||||
/// and deleting manifests first prevents an older retained slot from becoming
|
||||
/// visible again after the newest replay has been fully discharged.
|
||||
pub async fn delete_committed_snapshots_through(committed_through: u64, max_bytes: usize) -> Result<bool, SnapshotError> {
|
||||
let disks = super::journal_disks().await;
|
||||
delete_committed_snapshots_through_on(&disks, committed_through, max_bytes).await
|
||||
}
|
||||
|
||||
async fn delete_committed_snapshots_through_on(
|
||||
disks: &[EcstoreDiskStore],
|
||||
committed_through: u64,
|
||||
max_bytes: usize,
|
||||
) -> Result<bool, SnapshotError> {
|
||||
if disks.is_empty() {
|
||||
return Err(SnapshotError::NoWritableReplica);
|
||||
}
|
||||
let mut any_changed = false;
|
||||
let mut first_error = None;
|
||||
for disk in disks {
|
||||
for path in MANIFEST_PATHS {
|
||||
let existing = match read_bounded(disk, path, MANIFEST_LEN).await {
|
||||
Ok(Some(existing)) => existing,
|
||||
Ok(None) => continue,
|
||||
Err(error) => {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(error);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let manifest = match Manifest::decode(&existing, max_bytes) {
|
||||
Ok(manifest) => manifest,
|
||||
Err(error) => {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(error);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if manifest.sequence > committed_through {
|
||||
continue;
|
||||
}
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
Some(EcstoreDiskBytes::from(existing)),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => any_changed = true,
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => {}
|
||||
Err(error) => {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(SnapshotError::Disk(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if any_changed {
|
||||
Ok(true)
|
||||
} else if let Some(error) = first_error {
|
||||
Err(error)
|
||||
} else {
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_recovery_snapshot(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<RecoverySnapshot>, SnapshotError> {
|
||||
if let Some(snapshot) = read_committed(disks, limit).await? {
|
||||
return Ok(Some(RecoverySnapshot::Committed(snapshot)));
|
||||
@@ -337,7 +646,6 @@ async fn read_recovery_snapshot(disks: &[EcstoreDiskStore], limit: usize) -> Res
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::heal::mrf_queue::encode_intent;
|
||||
use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes};
|
||||
use crate::heal::{DiskOption, Endpoint, new_disk};
|
||||
use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope};
|
||||
use std::sync::Arc;
|
||||
@@ -359,6 +667,24 @@ mod tests {
|
||||
bytes
|
||||
}
|
||||
|
||||
fn many_record_payload(records: usize) -> Vec<u8> {
|
||||
let mut bytes = Vec::new();
|
||||
for index in 0..records {
|
||||
let intent = MrfIntent {
|
||||
bucket: Arc::from("b"),
|
||||
object: Arc::from(format!("o-{index:06}")),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
scope: None,
|
||||
lease: None,
|
||||
enqueued_at_ms: 1234,
|
||||
attempts: 0,
|
||||
};
|
||||
assert!(encode_intent(&intent, &mut bytes), "large fixture record must encode");
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
fn manifest(owner: Uuid, sequence: u64, payload: &[u8]) -> Vec<u8> {
|
||||
let mut bytes = Vec::with_capacity(MANIFEST_LEN);
|
||||
bytes.extend_from_slice(MAGIC);
|
||||
@@ -417,7 +743,7 @@ mod tests {
|
||||
fn manifest_validates_identity_sequence_length_and_digest() {
|
||||
let bytes = payload("object");
|
||||
let owner = Uuid::new_v4();
|
||||
assert!(CommittedSnapshot::decode(&manifest(owner, 1, &bytes), bytes.clone(), bytes.len()).is_ok());
|
||||
assert!(CommittedSnapshot::decode(0, &manifest(owner, 1, &bytes), bytes.clone(), bytes.len()).is_ok());
|
||||
for (owner, sequence) in [(Uuid::nil(), 1), (owner, 0), (owner, u64::MAX)] {
|
||||
assert!(matches!(
|
||||
Manifest::decode(&manifest(owner, sequence, &bytes), bytes.len()),
|
||||
@@ -442,12 +768,12 @@ mod tests {
|
||||
let owner = Uuid::new_v4();
|
||||
let header = manifest(owner, 1, &bytes);
|
||||
assert!(matches!(
|
||||
CommittedSnapshot::decode(&header, bytes[..bytes.len() - 1].to_vec(), bytes.len()),
|
||||
CommittedSnapshot::decode(0, &header, bytes[..bytes.len() - 1].to_vec(), bytes.len()),
|
||||
Err(SnapshotError::Corrupt)
|
||||
));
|
||||
let invalid = b"not an MRF record".to_vec();
|
||||
assert!(matches!(
|
||||
CommittedSnapshot::decode(&manifest(owner, 2, &invalid), invalid, bytes.len()),
|
||||
CommittedSnapshot::decode(0, &manifest(owner, 2, &invalid), invalid, bytes.len()),
|
||||
Err(SnapshotError::Corrupt)
|
||||
));
|
||||
}
|
||||
@@ -585,6 +911,271 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_snapshot_writer_uses_inactive_slot_and_reports_replicas() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let first = disk(&root, "first").await;
|
||||
let second = disk(&root, "second").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let old = payload("old");
|
||||
let next = payload("next");
|
||||
commit(&first, 0, owner, 1, &old).await;
|
||||
commit(&second, 0, owner, 1, &old).await;
|
||||
|
||||
let publication = publish_committed_snapshot(&[first.clone(), second.clone()], owner, 2, &next, 4096)
|
||||
.await
|
||||
.expect("publish successor");
|
||||
|
||||
assert_eq!(publication.slot, 1);
|
||||
assert_eq!(publication.payload_len, next.len());
|
||||
assert_eq!(publication.payload_replicas, 2);
|
||||
assert_eq!(publication.manifest_replicas, 2);
|
||||
let recovered = read_committed(&[first.clone(), second.clone()], 4096)
|
||||
.await
|
||||
.expect("read committed")
|
||||
.expect("successor committed");
|
||||
assert_eq!(recovered.sequence(), 2);
|
||||
assert_eq!(recovered.slot(), 1);
|
||||
assert_eq!(recovered.payload(), next.as_slice());
|
||||
for disk in [&first, &second] {
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
|
||||
.await
|
||||
.expect("old payload retained")
|
||||
.as_ref(),
|
||||
old.as_slice()
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
|
||||
.await
|
||||
.expect("old manifest retained")
|
||||
.as_ref(),
|
||||
manifest(owner, 1, &old).as_slice()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_snapshot_reclaim_deletes_only_the_superseded_slot_after_successor_readback() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let first = disk(&root, "first").await;
|
||||
let second = disk(&root, "second").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let old = payload("old");
|
||||
let next = payload("next");
|
||||
commit(&first, 0, owner, 1, &old).await;
|
||||
commit(&second, 0, owner, 1, &old).await;
|
||||
publish_committed_snapshot(&[first.clone(), second.clone()], owner, 2, &next, 4096)
|
||||
.await
|
||||
.expect("publish successor");
|
||||
|
||||
let reclaimed = reclaim_committed_snapshot_predecessor(&[first.clone(), second.clone()], owner, 2, 4096)
|
||||
.await
|
||||
.expect("reclaim predecessor");
|
||||
|
||||
assert_eq!(reclaimed.reclaimed_slot, 0);
|
||||
assert_eq!(reclaimed.manifest_replicas, 2);
|
||||
assert_eq!(reclaimed.payload_replicas, 2);
|
||||
let recovered = read_committed(&[first.clone(), second.clone()], 4096)
|
||||
.await
|
||||
.expect("read current successor")
|
||||
.expect("successor remains committed");
|
||||
assert_eq!(recovered.sequence(), 2);
|
||||
assert_eq!(recovered.payload(), next.as_slice());
|
||||
for disk in [&first, &second] {
|
||||
assert!(
|
||||
matches!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]).await,
|
||||
Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound)
|
||||
),
|
||||
"old manifest should be reclaimed"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]).await,
|
||||
Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound)
|
||||
),
|
||||
"old payload should be reclaimed"
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1])
|
||||
.await
|
||||
.expect("successor manifest retained")
|
||||
.as_ref(),
|
||||
manifest(owner, 2, &next)
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1])
|
||||
.await
|
||||
.expect("successor payload retained")
|
||||
.as_ref(),
|
||||
next
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_snapshot_reclaim_requires_the_successor_to_be_committed() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let store = disk(&root, "disk").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let old = payload("old");
|
||||
let next = payload("next");
|
||||
commit(&store, 0, owner, 1, &old).await;
|
||||
install(&store, PAYLOAD_PATHS[1], &next).await;
|
||||
|
||||
assert!(matches!(
|
||||
reclaim_committed_snapshot_predecessor(std::slice::from_ref(&store), owner, 2, 4096).await,
|
||||
Err(SnapshotError::Conflict)
|
||||
));
|
||||
|
||||
let reopened = disk(&root, "disk").await;
|
||||
let recovered = read_committed(std::slice::from_ref(&reopened), 4096)
|
||||
.await
|
||||
.expect("read old committed snapshot")
|
||||
.expect("old anchor remains committed");
|
||||
assert_eq!(recovered.sequence(), 1);
|
||||
assert_eq!(recovered.payload(), old.as_slice());
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
|
||||
.await
|
||||
.expect("old manifest retained")
|
||||
.as_ref(),
|
||||
manifest(owner, 1, &old)
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
|
||||
.await
|
||||
.expect("old payload retained")
|
||||
.as_ref(),
|
||||
old
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_snapshot_writer_manifest_failure_preserves_previous_anchor() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let store = disk(&root, "disk").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let old = payload("old");
|
||||
let next = payload("next");
|
||||
commit(&store, 0, owner, 1, &old).await;
|
||||
std::fs::create_dir(root.path().join("disk").join(RUSTFS_META_BUCKET).join(MANIFEST_PATHS[1]))
|
||||
.expect("manifest path blocks successor commit");
|
||||
|
||||
let result = publish_committed_snapshot(std::slice::from_ref(&store), owner, 2, &next, 4096).await;
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(SnapshotError::Disk(_) | SnapshotError::Read(_) | SnapshotError::NoWritableReplica)
|
||||
),
|
||||
"manifest failure must be visible: {result:?}"
|
||||
);
|
||||
let reopened = disk(&root, "disk").await;
|
||||
let recovered = read_committed(std::slice::from_ref(&reopened), 4096)
|
||||
.await
|
||||
.expect("read previous committed snapshot")
|
||||
.expect("old anchor remains committed");
|
||||
assert_eq!(recovered.sequence(), 1);
|
||||
assert_eq!(recovered.slot(), 0);
|
||||
assert_eq!(recovered.payload(), old.as_slice());
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
|
||||
.await
|
||||
.expect("old payload retained")
|
||||
.as_ref(),
|
||||
old.as_slice()
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
|
||||
.await
|
||||
.expect("old manifest retained")
|
||||
.as_ref(),
|
||||
manifest(owner, 1, &old).as_slice()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_snapshot_writer_does_not_overwrite_damaged_inactive_manifest() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let store = disk(&root, "disk").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let old = payload("old");
|
||||
let next = payload("next");
|
||||
let damaged = b"damaged successor manifest".to_vec();
|
||||
commit(&store, 0, owner, 1, &old).await;
|
||||
EcstoreDiskAPI::write_all(
|
||||
store.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
MANIFEST_PATHS[1],
|
||||
EcstoreDiskBytes::copy_from_slice(&damaged),
|
||||
)
|
||||
.await
|
||||
.expect("damaged inactive manifest fixture");
|
||||
|
||||
let result = publish_committed_snapshot(std::slice::from_ref(&store), owner, 2, &next, 4096).await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(SnapshotError::Corrupt)),
|
||||
"damaged manifest must fail closed: {result:?}"
|
||||
);
|
||||
let reopened = disk(&root, "disk").await;
|
||||
let recovered = read_committed(std::slice::from_ref(&reopened), 4096)
|
||||
.await
|
||||
.expect("read previous committed snapshot")
|
||||
.expect("old anchor remains committed");
|
||||
assert_eq!(recovered.sequence(), 1);
|
||||
assert_eq!(recovered.payload(), old.as_slice());
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1])
|
||||
.await
|
||||
.expect("damaged manifest retained")
|
||||
.as_ref(),
|
||||
damaged.as_slice()
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1]).await,
|
||||
Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound)
|
||||
),
|
||||
"successor payload must not be written before manifest slot is reusable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_snapshot_writer_publishes_100k_records_with_bounded_readback() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let first = disk(&root, "first").await;
|
||||
let second = disk(&root, "second").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let records = 100_000usize;
|
||||
let bytes = many_record_payload(records);
|
||||
let limit = rustfs_config::DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES;
|
||||
assert!(bytes.len() < limit, "100k compact MRF records must fit the configured journal limit");
|
||||
|
||||
let publication = publish_committed_snapshot(&[first.clone(), second.clone()], owner, 1, &bytes, limit)
|
||||
.await
|
||||
.expect("publish 100k-record successor");
|
||||
|
||||
assert_eq!(publication.payload_replicas, 2);
|
||||
assert_eq!(publication.manifest_replicas, 2);
|
||||
assert_eq!(publication.payload_len, bytes.len());
|
||||
let mut stats = SnapshotReadStats::default();
|
||||
let recovered = read_committed_with_stats(&[first, second], limit, Some(&mut stats))
|
||||
.await
|
||||
.expect("read committed large snapshot")
|
||||
.expect("large snapshot committed");
|
||||
assert_eq!(recovered.sequence(), 1);
|
||||
assert_eq!(recovered.payload().len(), bytes.len());
|
||||
let (decoded, truncated) = decode_journal(recovered.payload());
|
||||
assert_eq!(truncated, 0);
|
||||
assert_eq!(decoded.len(), records);
|
||||
assert_eq!(stats.file_reads, 4, "two manifest and two payload files should be read");
|
||||
assert_eq!(stats.bytes_read, (MANIFEST_LEN * 2) + (bytes.len() * 2));
|
||||
assert_eq!(stats.peak_file_bytes, bytes.len().max(MANIFEST_LEN));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_manifest_cas_cannot_replace_committed_anchor() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
@@ -609,6 +1200,116 @@ mod tests {
|
||||
assert_eq!(recovered.manifest.sequence, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manifest_cas_publication_transitions_from_legacy_without_losing_anchor() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let store = disk(&root, "disk").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let legacy = payload("legacy");
|
||||
let committed = payload("committed");
|
||||
let successor = payload("successor");
|
||||
|
||||
EcstoreDiskAPI::write_all(store.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, legacy.clone().into())
|
||||
.await
|
||||
.expect("legacy fixture");
|
||||
assert!(
|
||||
matches!(read_recovery_snapshot(std::slice::from_ref(&store), 4096).await.expect("legacy read"), Some(RecoverySnapshot::Legacy(data)) if data == legacy),
|
||||
"complete legacy journal remains the fallback before committed publication"
|
||||
);
|
||||
|
||||
install(&store, PAYLOAD_PATHS[0], &committed).await;
|
||||
assert!(
|
||||
matches!(read_recovery_snapshot(std::slice::from_ref(&store), 4096).await.expect("payload-only read"), Some(RecoverySnapshot::Legacy(data)) if data == legacy),
|
||||
"payload-only successor is not a committed snapshot"
|
||||
);
|
||||
|
||||
install(&store, MANIFEST_PATHS[0], &manifest(owner, 1, &committed)).await;
|
||||
let recovered = read_recovery_snapshot(std::slice::from_ref(&store), 4096)
|
||||
.await
|
||||
.expect("committed read")
|
||||
.expect("committed snapshot");
|
||||
assert!(
|
||||
matches!(recovered, RecoverySnapshot::Committed(snapshot) if snapshot.sequence() == 1 && snapshot.payload() == committed),
|
||||
"manifest CAS completion promotes the committed snapshot above legacy"
|
||||
);
|
||||
|
||||
let stale_manifest = manifest(owner, 2, &successor);
|
||||
let result = EcstoreDiskAPI::compare_and_update_file(
|
||||
store.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
MANIFEST_PATHS[0],
|
||||
None,
|
||||
Some(EcstoreDiskBytes::copy_from_slice(&stale_manifest)),
|
||||
)
|
||||
.await
|
||||
.expect("stale CAS call");
|
||||
assert_eq!(result, EcstoreConditionalFileUpdate::Mismatch);
|
||||
install(&store, PAYLOAD_PATHS[1], &successor).await;
|
||||
install(&store, MANIFEST_PATHS[1], &stale_manifest[..20]).await;
|
||||
|
||||
let reopened = disk(&root, "disk").await;
|
||||
let recovered = read_recovery_snapshot(std::slice::from_ref(&reopened), 4096)
|
||||
.await
|
||||
.expect("committed anchor after stale successor")
|
||||
.expect("committed snapshot");
|
||||
assert!(
|
||||
matches!(recovered, RecoverySnapshot::Committed(snapshot) if snapshot.sequence() == 1 && snapshot.payload() == committed),
|
||||
"failed or torn successor publication must not fall back to legacy"
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
|
||||
.await
|
||||
.expect("old manifest retained")
|
||||
.as_ref(),
|
||||
manifest(owner, 1, &committed)
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
|
||||
.await
|
||||
.expect("legacy bytes retained")
|
||||
.as_ref(),
|
||||
legacy
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_cleanup_removes_only_manifests_at_or_below_sequence() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let disk = disk(&root, "disk").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let older = payload("older");
|
||||
let newer = payload("newer");
|
||||
commit(&disk, 0, owner, 3, &older).await;
|
||||
commit(&disk, 1, owner, 4, &newer).await;
|
||||
|
||||
assert!(
|
||||
delete_committed_snapshots_through_on(std::slice::from_ref(&disk), 3, 4096)
|
||||
.await
|
||||
.expect("delete old manifest"),
|
||||
"old committed manifest should be removed"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]).await,
|
||||
Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound)
|
||||
),
|
||||
"old manifest is gone, so the old payload cannot become visible again"
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
|
||||
.await
|
||||
.expect("old payload orphan may remain")
|
||||
.as_ref(),
|
||||
older
|
||||
);
|
||||
let recovered = read_committed(std::slice::from_ref(&disk), 4096)
|
||||
.await
|
||||
.expect("read newer commit")
|
||||
.expect("newer commit remains visible");
|
||||
assert_eq!(recovered.sequence(), 4);
|
||||
assert_eq!(recovered.payload(), newer);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_import_requires_complete_consistent_replicas() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
|
||||
@@ -110,7 +110,8 @@ impl HealObjectReceipt {
|
||||
&& self.identity.version_id == expected.version_id
|
||||
&& self.identity.pool_index == expected.pool_index
|
||||
&& self.identity.set_index == expected.set_index
|
||||
&& self.identity.bucket_incarnation_id.is_some()
|
||||
&& self.identity.bucket_incarnation_id == expected.bucket_incarnation_id
|
||||
&& expected.bucket_incarnation_id.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -496,21 +497,31 @@ mod canonical_outcome_tests {
|
||||
|
||||
#[test]
|
||||
fn positive_receipt_requires_exact_identity_and_bucket_incarnation() {
|
||||
let expected = item(HealObjectDisposition::Unknown).identity;
|
||||
let incarnation = Uuid::new_v4();
|
||||
let expected = HealObjectIdentity {
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
..item(HealObjectDisposition::Unknown).identity
|
||||
};
|
||||
let mut receipt = HealObjectReceipt {
|
||||
identity: expected.clone(),
|
||||
disposition: HealObjectDisposition::Repaired,
|
||||
};
|
||||
|
||||
receipt.identity.bucket_incarnation_id = None;
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a positive storage receipt without bucket incarnation must remain untrusted"
|
||||
);
|
||||
|
||||
let incarnation = Uuid::new_v4();
|
||||
receipt.identity.bucket_incarnation_id = Some(incarnation);
|
||||
assert!(receipt.verified_for(&expected));
|
||||
|
||||
receipt.identity.bucket_incarnation_id = Some(Uuid::new_v4());
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a storage receipt for a different bucket incarnation must not clear the requested responsibility"
|
||||
);
|
||||
|
||||
receipt.identity.version_id = Some("older-version".to_string());
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
|
||||
@@ -20,6 +20,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::outcome::{HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectReceipt};
|
||||
use super::progress::stable_generation;
|
||||
@@ -355,6 +356,14 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
/// Get bucket info
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
|
||||
|
||||
/// Return the current bucket incarnation for exact MRF durable proof
|
||||
/// matching. Alternate backends that cannot expose this must return
|
||||
/// `None`, leaving replay anchors retained instead of acknowledged with an
|
||||
/// incomplete identity.
|
||||
async fn mrf_bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Aggregate usage-cache baselines for the requested buckets.
|
||||
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
Ok(None)
|
||||
@@ -383,6 +392,11 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
/// Check object exists
|
||||
async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool>;
|
||||
|
||||
/// Stable bucket incarnation observed before an object heal starts.
|
||||
async fn bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Heal object using ecstore
|
||||
async fn heal_object(
|
||||
&self,
|
||||
@@ -422,6 +436,14 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Err(Error::other("target-scoped replacement format is unsupported"))
|
||||
}
|
||||
|
||||
/// Whether the selected replacement set owns the pool metadata replica.
|
||||
///
|
||||
/// Only a topology-aware backend may exempt a valid non-owner set. The
|
||||
/// conservative default requires the existing repair and readback checks.
|
||||
async fn replacement_pool_metadata_applies(&self, _opts: &HealOpts) -> Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Read target-specific physical evidence for one replacement version.
|
||||
///
|
||||
/// This is only used by automatic replacement healing after the normal
|
||||
@@ -809,6 +831,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn mrf_bucket_incarnation_id(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||
self.ecstore
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
if buckets.is_empty() {
|
||||
return Ok(None);
|
||||
@@ -1028,6 +1058,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||
self.ecstore
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1238,6 +1276,18 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn replacement_pool_metadata_applies(&self, opts: &HealOpts) -> Result<bool> {
|
||||
let pool_index = opts
|
||||
.pool
|
||||
.ok_or_else(|| Error::other("replacement pool metadata is missing pool scope"))?;
|
||||
let set_index = opts
|
||||
.set
|
||||
.ok_or_else(|| Error::other("replacement pool metadata is missing set scope"))?;
|
||||
self.ecstore
|
||||
.replacement_pool_metadata_applies(pool_index, set_index)
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -29,6 +29,7 @@ pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageEr
|
||||
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext,
|
||||
POOL_META_NAME as ECSTORE_POOL_META_NAME,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
@@ -36,10 +37,11 @@ pub(crate) mod owner {
|
||||
pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
|
||||
|
||||
pub(crate) use super::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext,
|
||||
EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME,
|
||||
ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes,
|
||||
EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
|
||||
EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached,
|
||||
ecstore_local_disk_map_read,
|
||||
};
|
||||
|
||||
pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};
|
||||
|
||||
@@ -106,7 +106,7 @@ const EVENT_HEAL_ERASURE_SET_STAGE: &str = "heal_erasure_set_stage";
|
||||
const EVENT_HEAL_ERASURE_SET_RESULT: &str = "heal_erasure_set_result";
|
||||
|
||||
/// Heal type
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum HealType {
|
||||
/// Cluster heal
|
||||
Cluster,
|
||||
@@ -209,7 +209,7 @@ impl HealPriority {
|
||||
}
|
||||
|
||||
/// Heal options
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct HealOptions {
|
||||
/// Scan mode
|
||||
pub scan_mode: HealScanMode,
|
||||
@@ -574,6 +574,18 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn outcome_bucket_incarnation_id(&self, bucket: &str, dry_run: bool) -> Result<Option<Uuid>> {
|
||||
if dry_run {
|
||||
return Ok(None);
|
||||
}
|
||||
match self.await_with_control(self.storage.bucket_incarnation_id(bucket)).await {
|
||||
Ok(incarnation_id) => Ok(incarnation_id),
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn single_object_identity(&self) -> Option<HealObjectIdentity> {
|
||||
let (bucket, object, version) = match &self.heal_type {
|
||||
HealType::Object {
|
||||
@@ -597,6 +609,9 @@ impl HealTask {
|
||||
expected: HealObjectIdentity,
|
||||
receipt: Option<HealObjectReceipt>,
|
||||
) -> bool {
|
||||
if self.options.dry_run || self.cancel_token.is_cancelled() {
|
||||
return false;
|
||||
}
|
||||
let Some(receipt) = receipt else {
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -403,6 +403,7 @@ impl HealTask {
|
||||
};
|
||||
|
||||
for (set_disk_id, heal_opts) in listing_scopes {
|
||||
let bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, heal_opts.dry_run).await?;
|
||||
let mut continuation_token: Option<String> = None;
|
||||
let mut deferred = DeferredWindow::default();
|
||||
let mut inline_retry: Option<DeferredObject> = None;
|
||||
@@ -501,13 +502,15 @@ impl HealTask {
|
||||
let retry_attempt = item.attempt;
|
||||
let mut telemetry_unknown = false;
|
||||
let object = item.name.as_str();
|
||||
let identity =
|
||||
let mut identity =
|
||||
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
identity.bucket_incarnation_id = bucket_incarnation_id;
|
||||
let mut disposition = if heal_opts.dry_run {
|
||||
HealObjectDisposition::DryRunObserved
|
||||
} else {
|
||||
HealObjectDisposition::Unknown
|
||||
};
|
||||
let mut recorded_authoritative_outcome = false;
|
||||
let mut detail = None;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
@@ -520,23 +523,37 @@ impl HealTask {
|
||||
Some(Error::other("heal object retry age exhausted"))
|
||||
} else {
|
||||
match self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
|
||||
)
|
||||
.await_with_control(self.storage.heal_object_with_receipt(
|
||||
bucket,
|
||||
object,
|
||||
item.version_id.as_deref(),
|
||||
&heal_opts,
|
||||
))
|
||||
.await
|
||||
{
|
||||
Ok((result, None)) => match unavailable_recreate_error(&result, &heal_opts) {
|
||||
Some(error) => Some(error),
|
||||
None => {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |=
|
||||
!add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
|
||||
self.record_result_item(result).await;
|
||||
None
|
||||
Ok(storage_result) if storage_result.error.is_none() => {
|
||||
match unavailable_recreate_error(&storage_result.item, &heal_opts) {
|
||||
Some(error) => Some(error),
|
||||
None => {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |= !add_bytes(
|
||||
&mut bytes,
|
||||
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
|
||||
);
|
||||
recorded_authoritative_outcome = self
|
||||
.record_verified_storage_receipt(identity.clone(), storage_result.receipt)
|
||||
.await;
|
||||
self.record_result_item(storage_result.item).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
|
||||
}
|
||||
Ok(storage_result)
|
||||
if storage_result
|
||||
.error
|
||||
.as_ref()
|
||||
.is_some_and(|err| is_missing_object_dir_heal_result(object, err)) =>
|
||||
{
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -551,7 +568,8 @@ impl HealTask {
|
||||
);
|
||||
None
|
||||
}
|
||||
Ok((_, Some(err))) | Err(err) => Some(err),
|
||||
Ok(storage_result) => storage_result.error,
|
||||
Err(err) => Some(err),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -674,11 +692,13 @@ impl HealTask {
|
||||
continue;
|
||||
}
|
||||
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
if !recorded_authoritative_outcome {
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
}
|
||||
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(
|
||||
|
||||
@@ -162,6 +162,9 @@ impl HealTask {
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
let mut expected_identity =
|
||||
self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
||||
expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?;
|
||||
|
||||
let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
|
||||
let heal_result = if self.source == HealRequestSource::ReadRepair {
|
||||
@@ -266,8 +269,6 @@ impl HealTask {
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(1, 1, 0, 0, object_size);
|
||||
}
|
||||
let expected_identity =
|
||||
self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
||||
self.record_verified_storage_receipt(expected_identity, storage_result.receipt)
|
||||
.await;
|
||||
self.record_result_item(result).await;
|
||||
|
||||
@@ -154,6 +154,126 @@ mod canonical_outcome {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_records_matching_positive_storage_receipts() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([
|
||||
(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||
),
|
||||
(
|
||||
"object-b".to_string(),
|
||||
VecDeque::from([object_receipt("object-b", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||
),
|
||||
])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("bucket heal should record verified object receipts");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.healed, 2);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.objects.len(), 2);
|
||||
assert!(outcome.objects.iter().all(|item| {
|
||||
item.identity.bucket_incarnation_id == Some(incarnation) && item.disposition == HealObjectDisposition::Repaired
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_keeps_repairing_when_bucket_incarnation_is_unavailable() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_unavailable: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage.clone());
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("bucket heal should continue when only proof ownership is unavailable");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unknown, 2);
|
||||
assert!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.all(|item| item.disposition == HealObjectDisposition::Unknown)
|
||||
);
|
||||
assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_rejects_stale_receipts_without_double_recording() {
|
||||
let expected_incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([
|
||||
(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
),
|
||||
(
|
||||
"object-b".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-b",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
expected_incarnation,
|
||||
)]),
|
||||
),
|
||||
])),
|
||||
bucket_incarnation_id: Mutex::new(Some(expected_incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("stale bucket receipt should not fail the legacy heal");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.healed, 1);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.objects.len(), 2);
|
||||
let object_a = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-a")
|
||||
.expect("stale receipt object outcome");
|
||||
assert_eq!(object_a.disposition, HealObjectDisposition::Unknown);
|
||||
let object_b = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-b")
|
||||
.expect("matching receipt object outcome");
|
||||
assert_eq!(object_b.disposition, HealObjectDisposition::Repaired);
|
||||
assert_eq!(object_b.identity.bucket_incarnation_id, Some(expected_incarnation));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
@@ -313,6 +433,96 @@ mod canonical_outcome {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mixed_grace_and_repaired_receipt_transfer_only_repaired_responsibility() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_outcomes: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred]),
|
||||
)])),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-b".to_string(),
|
||||
VecDeque::from([object_receipt("object-b", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("mixed grace and repaired receipt should complete");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.healed, 1);
|
||||
assert_eq!(outcome.counters.skipped, 1);
|
||||
let deferred = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-a")
|
||||
.expect("grace object should remain recorded");
|
||||
assert!(matches!(
|
||||
deferred.disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_ne!(deferred.disposition, HealObjectDisposition::Repaired);
|
||||
let repaired = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-b")
|
||||
.expect("receipt-backed object should be recorded");
|
||||
assert_eq!(repaired.identity.bucket_incarnation_id, Some(incarnation));
|
||||
assert_eq!(repaired.disposition, HealObjectDisposition::Repaired);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_records_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
|
||||
task.execute().await.expect("bucket heal should complete");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.healed, 1);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|item| item.identity.object == "object-a")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
let repaired = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-a")
|
||||
.expect("receipt-backed bucket object should be recorded");
|
||||
assert_eq!(repaired.identity.bucket_incarnation_id, Some(incarnation));
|
||||
assert_eq!(repaired.disposition, HealObjectDisposition::Repaired);
|
||||
let legacy = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-b")
|
||||
.expect("legacy bucket object should still be recorded");
|
||||
assert_eq!(legacy.disposition, HealObjectDisposition::Unknown);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn grace_single_object_is_completed_but_deferred() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
@@ -344,6 +554,18 @@ mod canonical_outcome {
|
||||
.lock()
|
||||
.expect("existence fixture")
|
||||
.insert("object".to_string(), MockObjectExists::TransientSkip("retry later"));
|
||||
} else {
|
||||
let incarnation = Uuid::new_v4();
|
||||
*storage.bucket_incarnation_id.lock().expect("bucket incarnation") = Some(incarnation);
|
||||
storage.heal_object_receipts.lock().expect("receipt fixture").insert(
|
||||
"object".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object",
|
||||
None,
|
||||
HealObjectDisposition::VerifiedHealthy,
|
||||
incarnation,
|
||||
)]),
|
||||
);
|
||||
}
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
|
||||
request.options.dry_run = !transient;
|
||||
@@ -1116,6 +1338,13 @@ struct MockStorage {
|
||||
heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
|
||||
heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
|
||||
heal_object_receipts: Mutex<HashMap<String, VecDeque<HealObjectReceipt>>>,
|
||||
bucket_incarnation_id: Mutex<Option<Uuid>>,
|
||||
bucket_incarnation_calls: AtomicU64,
|
||||
bucket_incarnation_error: Mutex<Option<Error>>,
|
||||
block_bucket_incarnation: bool,
|
||||
bucket_incarnation_started: tokio::sync::Notify,
|
||||
bucket_incarnation_after_object_heal: Mutex<Option<Uuid>>,
|
||||
bucket_incarnation_unavailable: Mutex<bool>,
|
||||
format_no_heal_required: Mutex<bool>,
|
||||
format_error: Mutex<Option<Error>>,
|
||||
global_format_calls: Mutex<u32>,
|
||||
@@ -1219,14 +1448,19 @@ async fn execute_emits_heal_trace_task_state() {
|
||||
assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed"));
|
||||
}
|
||||
|
||||
fn object_receipt(object: &str, version_id: Option<&str>, disposition: HealObjectDisposition) -> HealObjectReceipt {
|
||||
fn object_receipt(
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
disposition: HealObjectDisposition,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> HealObjectReceipt {
|
||||
HealObjectReceipt {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket-a".to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: version_id.map(ToOwned::to_owned),
|
||||
bucket_incarnation_id: Some(Uuid::new_v4()),
|
||||
bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
@@ -1236,16 +1470,23 @@ fn object_receipt(object: &str, version_id: Option<&str>, disposition: HealObjec
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_records_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt("object-a", Some("version-a"), HealObjectDisposition::Repaired)]),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::Repaired,
|
||||
incarnation,
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
storage.clone(),
|
||||
);
|
||||
|
||||
task.execute().await.expect("mock object heal should complete");
|
||||
@@ -1258,19 +1499,276 @@ async fn object_heal_records_matching_positive_storage_receipt() {
|
||||
assert_eq!(object.identity.version_id.as_deref(), Some("version-a"));
|
||||
assert!(object.identity.bucket_incarnation_id.is_some());
|
||||
assert_eq!(object.disposition, HealObjectDisposition::Repaired);
|
||||
assert_eq!(
|
||||
storage.bucket_incarnation_calls.load(Ordering::Relaxed),
|
||||
1,
|
||||
"latch the owner exactly once before repair"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_rejects_mismatched_or_legacy_storage_receipts() {
|
||||
async fn object_heal_owner_lookup_failure_preserves_unverified_repair() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_error: Mutex::new(Some(Error::other("owner metadata unavailable"))),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(HealRequest::object("bucket-a".to_string(), "object-a".to_string(), None), storage.clone());
|
||||
|
||||
task.execute().await.expect("missing receipt owner must not prevent repair");
|
||||
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("heal calls").as_slice(), ["object-a"]);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(
|
||||
outcome.objects.front().expect("unverified outcome").disposition,
|
||||
HealObjectDisposition::Unknown
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_dry_run_skips_owner_lookup_and_positive_receipts() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_error: Mutex::new(Some(Error::other("dry-run must not query the receipt owner"))),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object-a".to_string(), None);
|
||||
request.options.dry_run = true;
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
task.execute().await.expect("dry-run should complete without owner metadata");
|
||||
|
||||
assert!(storage.object_heal_opts.lock().expect("heal options")[0].dry_run);
|
||||
assert_eq!(storage.bucket_incarnation_calls.load(Ordering::Relaxed), 0);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.counters.skipped, 1);
|
||||
assert_eq!(
|
||||
outcome.objects.front().expect("dry-run outcome").disposition,
|
||||
HealObjectDisposition::DryRunObserved
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn object_heal_owner_lookup_obeys_task_timeout() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
block_bucket_incarnation: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object-a".to_string(), None);
|
||||
request.options.timeout = Some(Duration::from_secs(5));
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let result = tokio::time::timeout(Duration::from_secs(60), task.execute())
|
||||
.await
|
||||
.expect("owner lookup must honor the task deadline");
|
||||
|
||||
assert!(matches!(result, Err(Error::TaskTimeout)));
|
||||
assert!(storage.heal_object_calls.lock().expect("heal calls").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_owner_lookup_obeys_cancellation() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
block_bucket_incarnation: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object-a".to_string(), None);
|
||||
request.options.timeout = None;
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
tokio::join!(task.execute(), async {
|
||||
storage.bucket_incarnation_started.notified().await;
|
||||
task.cancel().await.expect("cancel pending owner lookup");
|
||||
})
|
||||
})
|
||||
.await
|
||||
.expect("cancellation must interrupt owner lookup");
|
||||
|
||||
assert!(matches!(result, Err(Error::TaskCancelled)));
|
||||
assert!(storage.heal_object_calls.lock().expect("heal calls").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_object_heal_rejects_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
task.cancel().await.expect("task cancellation should succeed");
|
||||
|
||||
let expected = HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket-a".to_string(),
|
||||
object: "object-a".to_string(),
|
||||
version_id: Some("version-a".to_string()),
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
};
|
||||
let accepted = task
|
||||
.record_verified_storage_receipt(
|
||||
expected,
|
||||
Some(object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::Repaired,
|
||||
incarnation,
|
||||
)),
|
||||
)
|
||||
.await;
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert!(!accepted);
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert!(outcome.objects.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_object_heal_rejects_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithOtherError("commit failed"))),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::Repaired,
|
||||
incarnation,
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
let result = task.execute().await;
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unchanged, 0);
|
||||
assert!(outcome.objects.iter().all(|object| {
|
||||
!matches!(
|
||||
object.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_quorum_object_heal_rejects_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithReadQuorum)),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::Repaired,
|
||||
incarnation,
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
let result = task.execute().await;
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unchanged, 0);
|
||||
assert!(outcome.objects.iter().all(|object| {
|
||||
!matches!(
|
||||
object.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_latches_expected_incarnation_before_repair() {
|
||||
let original_incarnation = Uuid::new_v4();
|
||||
let successor_incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("old-version"),
|
||||
HealObjectDisposition::Repaired,
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::VerifiedHealthy,
|
||||
original_incarnation,
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(original_incarnation)),
|
||||
bucket_incarnation_after_object_heal: Mutex::new(Some(successor_incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
task.execute().await.expect("mock object heal should complete");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.unchanged, 1);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
let object = outcome.objects.front().expect("latched receipt should be recorded");
|
||||
assert_eq!(object.identity.bucket_incarnation_id, Some(original_incarnation));
|
||||
assert_eq!(object.disposition, HealObjectDisposition::VerifiedHealthy);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_rejects_mismatched_or_legacy_storage_receipts() {
|
||||
let expected_incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(expected_incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
@@ -1361,6 +1859,7 @@ enum MockHealObjectOutcome {
|
||||
RetryableLock,
|
||||
RetryableLockTimeout,
|
||||
OkWithOtherError(&'static str),
|
||||
OkWithReadQuorum,
|
||||
ErrOther(&'static str),
|
||||
DanglingGraceDeferred,
|
||||
UnavailableDrive(DriveState),
|
||||
@@ -1456,6 +1955,21 @@ impl HealStorageAPI for MockStorage {
|
||||
Ok(self.object_exists.lock().unwrap().unwrap_or(true))
|
||||
}
|
||||
|
||||
async fn bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||
self.bucket_incarnation_calls.fetch_add(1, Ordering::Relaxed);
|
||||
self.bucket_incarnation_started.notify_one();
|
||||
if self.block_bucket_incarnation {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
if let Some(error) = self.bucket_incarnation_error.lock().expect("owner lookup error").take() {
|
||||
return Err(error);
|
||||
}
|
||||
if *self.bucket_incarnation_unavailable.lock().unwrap() {
|
||||
return Err(Error::Other("bucket incarnation unavailable".to_string()));
|
||||
}
|
||||
Ok(*self.bucket_incarnation_id.lock().unwrap())
|
||||
}
|
||||
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1513,6 +2027,13 @@ impl HealStorageAPI for MockStorage {
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
MockHealObjectOutcome::OkWithReadQuorum => Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
)),
|
||||
MockHealObjectOutcome::PermanentOther(message) => Err(Error::other(message)),
|
||||
MockHealObjectOutcome::OkWithOtherError(message) => Ok((HealResultItem::default(), Some(Error::other(message)))),
|
||||
MockHealObjectOutcome::ErrOther(message) => Err(Error::other(message)),
|
||||
@@ -1552,6 +2073,13 @@ impl HealStorageAPI for MockStorage {
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
MockHealObjectOutcome::OkWithReadQuorum => Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
)),
|
||||
};
|
||||
}
|
||||
if bucket == RUSTFS_META_BUCKET && object == format!("{BUCKET_META_PREFIX}/{DATA_USAGE_CACHE_NAME}") {
|
||||
@@ -1566,6 +2094,9 @@ impl HealStorageAPI for MockStorage {
|
||||
return Ok((HealResultItem::default(), Some(Error::Disk(DiskError::FileNotFound))));
|
||||
}
|
||||
self.healed_objects.lock().unwrap().push(object.to_string());
|
||||
if let Some(bucket_incarnation_id) = self.bucket_incarnation_after_object_heal.lock().unwrap().take() {
|
||||
*self.bucket_incarnation_id.lock().unwrap() = Some(bucket_incarnation_id);
|
||||
}
|
||||
Ok((
|
||||
HealResultItem {
|
||||
object_size: 1,
|
||||
|
||||
@@ -29,6 +29,7 @@ use rustfs_heal::heal::{
|
||||
storage::{ECStoreHealStorage, HealStorageAPI},
|
||||
};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
#[cfg(unix)]
|
||||
use std::{
|
||||
fs::{File, OpenOptions},
|
||||
@@ -48,6 +49,10 @@ use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, Pool
|
||||
const META_BUCKET: &str = ".rustfs.sys";
|
||||
const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
|
||||
const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin";
|
||||
const COMMITTED_PAYLOAD_REL: &str = ".heal-mrf-snapshot.0.bin";
|
||||
const COMMITTED_MANIFEST_REL: &str = ".heal-mrf-commit.0.bin";
|
||||
const COMMITTED_MAGIC: &[u8; 8] = b"RFMRFC01";
|
||||
const COMMITTED_MANIFEST_LEN: usize = 8 + 1 + 16 + 8 + 8 + 32 + 32;
|
||||
|
||||
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
|
||||
heal_env_at(None).await
|
||||
@@ -195,6 +200,29 @@ fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
|
||||
write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
|
||||
}
|
||||
|
||||
fn committed_manifest(owner: uuid::Uuid, sequence: u64, payload: &[u8]) -> Vec<u8> {
|
||||
let mut manifest = Vec::with_capacity(COMMITTED_MANIFEST_LEN);
|
||||
manifest.extend_from_slice(COMMITTED_MAGIC);
|
||||
manifest.push(1);
|
||||
manifest.extend_from_slice(owner.as_bytes());
|
||||
manifest.extend_from_slice(&sequence.to_le_bytes());
|
||||
manifest.extend_from_slice(
|
||||
&u64::try_from(payload.len())
|
||||
.expect("fixture payload length fits")
|
||||
.to_le_bytes(),
|
||||
);
|
||||
manifest.extend_from_slice(&Sha256::digest(payload));
|
||||
manifest.extend_from_slice(&Sha256::digest(&manifest));
|
||||
assert_eq!(manifest.len(), COMMITTED_MANIFEST_LEN, "committed fixture manifest length");
|
||||
manifest
|
||||
}
|
||||
|
||||
fn write_committed_snapshot_to_disks(disk_paths: &[std::path::PathBuf], sequence: u64, payload: &[u8]) {
|
||||
let manifest = committed_manifest(uuid::Uuid::new_v4(), sequence, payload);
|
||||
write_journal_path_to_disks(disk_paths, COMMITTED_PAYLOAD_REL, payload);
|
||||
write_journal_path_to_disks(disk_paths, COMMITTED_MANIFEST_REL, &manifest);
|
||||
}
|
||||
|
||||
fn journal_exists_on_all_disks(disk_paths: &[std::path::PathBuf], relative_path: &str) -> bool {
|
||||
disk_paths
|
||||
.iter()
|
||||
@@ -255,11 +283,12 @@ async fn decode_failure_intent_maps_to_urgent_mrf_heal_request() {
|
||||
}
|
||||
|
||||
/// A journal left behind by a previous process must be replayed into the
|
||||
/// manager queue and then removed, and a torn tail must not block replay of
|
||||
/// the intact records.
|
||||
/// manager queue, and a torn tail must not block replay of the intact records.
|
||||
/// The partial-write record keeps the legacy journal as the durable anchor
|
||||
/// until an exact verified repair proof can discharge it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn journal_replay_arms_intents_and_deletes_the_file() {
|
||||
async fn journal_replay_arms_intents_and_retains_unproven_partial_write_anchor() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
|
||||
// The journal reader resolves disks through the process-local disk map;
|
||||
@@ -285,14 +314,14 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
|
||||
assert!(
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
|
||||
"the journal file must be removed after a successful replay"
|
||||
.all(|path| Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
|
||||
"partial-write replay must retain the legacy journal until durable proof"
|
||||
);
|
||||
assert!(
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
|
||||
"the authoritative journal file must also be removed after replay"
|
||||
"missing authoritative journal remains absent"
|
||||
);
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
@@ -300,6 +329,76 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
|
||||
assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
|
||||
}
|
||||
|
||||
/// A committed checkpoint published by the new two-slot writer is the
|
||||
/// authoritative startup snapshot. Legacy mirrors are fallback-only and must
|
||||
/// not be merged with or preferred over the committed epoch.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn committed_snapshot_replay_takes_precedence_over_stale_legacy_mirror() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
register_local_disks(&disk_paths, "mrf-committed-replay-test").await;
|
||||
|
||||
let committed = scoped_journal_record(3, "committed-bucket", "committed-object", Some([9u8; 16]), 0, 0, 0);
|
||||
let stale_legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0);
|
||||
write_committed_snapshot_to_disks(&disk_paths, 7, &committed);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy);
|
||||
|
||||
let manager = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||
assert_eq!(replayed, 1, "only the committed snapshot epoch may replay");
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
assert_eq!(snapshot.queued_by_source.mrf, 1);
|
||||
assert_eq!(
|
||||
snapshot.queued_by_priority.normal, 1,
|
||||
"the committed partial-write record must replay instead of the stale legacy decode-failure"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.queued_by_priority.urgent, 0,
|
||||
"stale legacy decode-failure records must not be mixed into committed replay"
|
||||
);
|
||||
assert!(
|
||||
journal_exists_on_all_disks(&disk_paths, COMMITTED_MANIFEST_REL),
|
||||
"the committed checkpoint remains until the accepted partial-write has proof"
|
||||
);
|
||||
}
|
||||
|
||||
/// A damaged committed checkpoint is ambiguous: replay must not fall back to
|
||||
/// older legacy bytes or delete any recovery anchor until another process can
|
||||
/// publish a valid successor.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn damaged_committed_snapshot_blocks_legacy_fallback_and_retains_anchors() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
register_local_disks(&disk_paths, "mrf-damaged-committed-replay-test").await;
|
||||
|
||||
let committed = scoped_journal_record(3, "damaged-committed-bucket", "committed-object", Some([8u8; 16]), 0, 0, 0);
|
||||
let stale_legacy = journal_record(1, "damaged-legacy-bucket", "legacy-object", None, 0);
|
||||
write_journal_path_to_disks(&disk_paths, COMMITTED_PAYLOAD_REL, &committed);
|
||||
let mut manifest = committed_manifest(uuid::Uuid::new_v4(), 9, &committed);
|
||||
manifest[25] ^= 1;
|
||||
write_journal_path_to_disks(&disk_paths, COMMITTED_MANIFEST_REL, &manifest);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy);
|
||||
|
||||
let manager = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||
assert_eq!(replayed, 0, "damaged committed state must fail closed");
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
0,
|
||||
"stale legacy bytes must not be replayed when committed state is ambiguous"
|
||||
);
|
||||
assert!(
|
||||
journal_exists_on_all_disks(&disk_paths, COMMITTED_MANIFEST_REL)
|
||||
&& journal_exists_on_all_disks(&disk_paths, COMMITTED_PAYLOAD_REL)
|
||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy)
|
||||
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &stale_legacy),
|
||||
"all recovery anchors must remain after a fail-closed committed read"
|
||||
);
|
||||
}
|
||||
|
||||
/// A canonical snapshot and its compatibility mirror may differ after a
|
||||
/// partial flush. Replay must choose the complete canonical epoch instead of
|
||||
/// combining records that never coexisted in memory.
|
||||
@@ -322,21 +421,24 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
||||
assert_eq!(snapshot.queued_by_source.mrf, 1);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"replay cleanup must remove both journal paths"
|
||||
"accepted replay responsibilities remain anchored until a verified repair proof"
|
||||
);
|
||||
|
||||
// A scoped-only snapshot is valid during a rollout where no legacy
|
||||
// compatibility mirror was written. Missing legacy files must not leave
|
||||
// the runtime in a permanent cleanup-retry state.
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
register_local_disks(&disk_paths, "mrf-scoped-authoritative-test").await;
|
||||
let manager = make_manager(storage);
|
||||
let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only);
|
||||
assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1);
|
||||
assert!(disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
|
||||
let scoped_v2 = scoped_journal_record(1, "scoped-v2-bucket", "scoped-v2-object", None, 0, 3, 7);
|
||||
@@ -350,12 +452,12 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
||||
);
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
3,
|
||||
"only the three authoritative/scoped-only epochs should have reached the manager"
|
||||
2,
|
||||
"only the scoped-only and scoped-v2 authoritative epochs should have reached the manager"
|
||||
);
|
||||
assert!(disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -400,10 +502,13 @@ async fn authoritative_journal_replay_preserves_kind_and_scope_identity() {
|
||||
snapshot.queued_by_priority.urgent, 1,
|
||||
"decode-failure repair must not merge with object repair responsibility"
|
||||
);
|
||||
assert!(disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"partial-write responsibilities keep both replay anchors until proof"
|
||||
);
|
||||
}
|
||||
|
||||
/// If replay reaches a full heal-manager queue, the old journal remains the
|
||||
@@ -521,7 +626,8 @@ fn mrf_successor_flush_child_process_fixture() {
|
||||
}),
|
||||
));
|
||||
mrf_queue::spawn_mrf_consumer(manager.clone());
|
||||
let expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
let mut expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "successor-bucket", "first-object", None, 0));
|
||||
let flushed = wait_until(Duration::from_secs(10), || async {
|
||||
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
||||
@@ -568,7 +674,8 @@ fn mrf_successor_flush_waiting_child_process_fixture() {
|
||||
}),
|
||||
));
|
||||
mrf_queue::spawn_mrf_consumer(manager.clone());
|
||||
let expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
let mut expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "service-kill-bucket", "first-object", None, 0));
|
||||
let flushed = wait_until(Duration::from_secs(10), || async {
|
||||
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
||||
@@ -608,7 +715,8 @@ fn mrf_authoritative_fsync_waiting_child_process_fixture() {
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &startup);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &startup);
|
||||
|
||||
let successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
let mut successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
successor.extend(journal_record(1, "fsync-kill-bucket", "first-object", None, 0));
|
||||
write_journal_path_to_disks_synced(&disk_paths, SCOPED_JOURNAL_REL, &successor);
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &successor)
|
||||
@@ -665,9 +773,8 @@ async fn journal_replay_retains_child_process_anchor_when_manager_is_full() {
|
||||
);
|
||||
}
|
||||
|
||||
/// If a process crashes after flushing a smaller successor snapshot but before
|
||||
/// deleting the startup anchor, the restarted process must replay the
|
||||
/// successor tail rather than losing it or merging it with stale records.
|
||||
/// A successor flush must preserve both pending work and accepted work whose
|
||||
/// repair has not been proven when the process restarts.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn journal_replay_survives_successor_flush_before_delete() {
|
||||
@@ -682,7 +789,8 @@ async fn journal_replay_survives_successor_flush_before_delete() {
|
||||
assert_eq!(status.code(), Some(78), "child process did not reach the successor flush boundary");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
let expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
let mut expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "successor-bucket", "first-object", None, 0));
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor),
|
||||
"restarted process must see the pending successor snapshot"
|
||||
@@ -690,24 +798,24 @@ async fn journal_replay_survives_successor_flush_before_delete() {
|
||||
|
||||
let restarted = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 1, "restart after successor flush must replay only the still-pending tail");
|
||||
assert_eq!(replayed, 2, "restart must replay both the admitted and pending responsibilities");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"the successor tail must be accepted after restart"
|
||||
2,
|
||||
"both unproven successor responsibilities must be accepted after restart"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"a fully consumed successor snapshot may be deleted after restart replay"
|
||||
"the accepted successor remains anchored until a verified repair proof"
|
||||
);
|
||||
}
|
||||
|
||||
/// A service-style hard kill after successor flush must be equivalent to a
|
||||
/// crash at the flush-before-delete boundary: restart may replay the smaller
|
||||
/// successor snapshot, but must not lose or merge stale startup records.
|
||||
/// crash at the flush-before-delete boundary: restart must recover every
|
||||
/// unproven responsibility from the successor snapshot.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
#[cfg(unix)]
|
||||
@@ -735,7 +843,8 @@ async fn journal_replay_survives_service_kill_after_successor_flush() {
|
||||
assert!(!status.success(), "child fixture must be terminated instead of exiting cleanly");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
let expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
let mut expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "service-kill-bucket", "first-object", None, 0));
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor),
|
||||
"restarted process must see the successor snapshot produced before the kill"
|
||||
@@ -743,25 +852,25 @@ async fn journal_replay_survives_service_kill_after_successor_flush() {
|
||||
|
||||
let restarted = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 1, "restart after service kill must replay only the still-pending tail");
|
||||
assert_eq!(replayed, 2, "service-kill restart must preserve every unproven responsibility");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"the successor tail must be accepted after service kill restart"
|
||||
2,
|
||||
"both unproven responsibilities must be accepted after service kill restart"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"a fully consumed successor snapshot may be deleted after service-kill restart replay"
|
||||
"the accepted successor remains anchored until a verified repair proof after service-kill restart"
|
||||
);
|
||||
}
|
||||
|
||||
/// A hard kill between the authoritative successor fsync and the legacy mirror
|
||||
/// rewrite must prefer the canonical successor tail over the stale legacy
|
||||
/// startup epoch. This models the mixed-version boundary conservatively: new
|
||||
/// readers must not merge epochs, while the old mirror remains crash-visible.
|
||||
/// rewrite must prefer the canonical successor over the stale legacy startup
|
||||
/// epoch while retaining every unproven responsibility. New readers must not
|
||||
/// merge epochs, while the old mirror remains crash-visible.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
#[cfg(unix)]
|
||||
@@ -789,7 +898,8 @@ async fn journal_replay_survives_sigkill_after_authoritative_successor_fsync_bef
|
||||
assert!(!status.success(), "child fixture must be terminated instead of exiting cleanly");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
let expected_successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
let mut expected_successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "fsync-kill-bucket", "first-object", None, 0));
|
||||
let stale_startup = {
|
||||
let mut startup = journal_record(1, "fsync-kill-bucket", "first-object", None, 0);
|
||||
startup.extend(journal_record(1, "fsync-kill-bucket", "second-object", None, 0));
|
||||
@@ -806,17 +916,17 @@ async fn journal_replay_survives_sigkill_after_authoritative_successor_fsync_bef
|
||||
|
||||
let restarted = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 1, "new reader must replay only the authoritative successor tail");
|
||||
assert_eq!(replayed, 2, "new reader must recover every responsibility in the authoritative successor");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"the successor tail must be accepted after the fsync-boundary restart"
|
||||
2,
|
||||
"both responsibilities must be accepted after the fsync-boundary restart"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"a fully consumed authoritative successor may clean both epochs after restart replay"
|
||||
"the accepted authoritative successor remains anchored until a verified repair proof"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1233,8 +1233,7 @@ impl LocalKmsClient {
|
||||
async fn decode_stored_key(&self, key_id: &str) -> Result<(StoredMasterKey, Vec<u8>)> {
|
||||
let key_path = self.master_key_path(key_id)?;
|
||||
if !fs::try_exists(&key_path).await? {
|
||||
// Only an accessible key store can establish that a single key is
|
||||
// missing; a directory outage must retain its filesystem error.
|
||||
// A missing key is a caller error only while its storage directory is available.
|
||||
let _ = fs::read_dir(&self.config.key_dir).await?;
|
||||
return Err(KmsError::key_not_found(key_id));
|
||||
}
|
||||
@@ -2098,11 +2097,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
let _write_guard = self.client.lock_key_for_write(key_id).await;
|
||||
|
||||
// First, load the key from disk to get the master key
|
||||
let mut master_key = self
|
||||
.client
|
||||
.load_master_key(key_id)
|
||||
.await
|
||||
.map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
let mut master_key = self.client.load_master_key(key_id).await?;
|
||||
|
||||
let (deletion_date_str, deletion_date_dt) = if request.force_immediate.unwrap_or(false) {
|
||||
// Tombstone first: mark the record Deleted before removing the
|
||||
@@ -2205,11 +2200,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
let _write_guard = self.client.lock_key_for_write(key_id).await;
|
||||
|
||||
// Load the key from disk to get the master key
|
||||
let mut master_key = self
|
||||
.client
|
||||
.load_master_key(key_id)
|
||||
.await
|
||||
.map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
let mut master_key = self.client.load_master_key(key_id).await?;
|
||||
|
||||
if master_key.status != KeyStatus::PendingDeletion {
|
||||
return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion")));
|
||||
@@ -2390,76 +2381,6 @@ mod tests {
|
||||
(client, temp_dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_key_directory_outage_is_io_error_and_recovers_original_key() {
|
||||
let root = TempDir::new().expect("create isolated key store");
|
||||
let key_dir = root.path().join("keys");
|
||||
let unavailable_dir = root.path().join("keys-unavailable");
|
||||
let config = KmsConfig::local(key_dir.clone()).with_insecure_development_defaults();
|
||||
let backend = LocalKmsBackend::new(config).await.expect("start Local KMS");
|
||||
let key_id = "directory-outage-key";
|
||||
backend
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some(key_id.to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("create the original key");
|
||||
let request = |key_id: &str| GenerateDataKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
key_spec: KeySpec::Aes256,
|
||||
encryption_context: HashMap::new(),
|
||||
};
|
||||
let before = backend
|
||||
.generate_data_key(request(key_id))
|
||||
.await
|
||||
.expect("generate a data key before the outage");
|
||||
let missing_key = backend.generate_data_key(request("no-such-key")).await;
|
||||
let key_path = key_dir.join(format!("{key_id}.key"));
|
||||
let original_record = fs::read(&key_path).await.expect("read the original key record");
|
||||
|
||||
fs::rename(&key_dir, &unavailable_dir)
|
||||
.await
|
||||
.expect("make the key directory unavailable");
|
||||
let unavailable = backend.generate_data_key(request(key_id)).await;
|
||||
// Restore before checking the error so the failing regression leaves no
|
||||
// orphaned key store; both paths also belong to the same temporary root.
|
||||
fs::rename(&unavailable_dir, &key_dir)
|
||||
.await
|
||||
.expect("restore the original key directory");
|
||||
|
||||
let after = backend
|
||||
.generate_data_key(request(key_id))
|
||||
.await
|
||||
.expect("generate a data key after directory restoration");
|
||||
for data_key in [&before, &after] {
|
||||
let decrypted = backend
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: data_key.ciphertext_blob.clone(),
|
||||
encryption_context: HashMap::new(),
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
.await
|
||||
.expect("the original master key must decrypt both data keys");
|
||||
assert!(
|
||||
decrypted.plaintext == data_key.plaintext_key,
|
||||
"directory restoration must preserve the original key material"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
fs::read(&key_path).await.expect("read the restored key record") == original_record,
|
||||
"reads and recovery must not rewrite the key record"
|
||||
);
|
||||
assert!(
|
||||
matches!(missing_key, Err(KmsError::KeyNotFound { key_id }) if key_id == "no-such-key"),
|
||||
"a missing key in a readable directory must remain KeyNotFound"
|
||||
);
|
||||
assert!(
|
||||
matches!(unavailable, Err(KmsError::IoError { .. })),
|
||||
"an unavailable key directory must remain an I/O error, not KeyNotFound"
|
||||
);
|
||||
}
|
||||
|
||||
/// With the AAD write switch on, the Local backend seals the stored
|
||||
/// encryption context into the wrap exactly like KV2: the bound envelope
|
||||
/// round-trips, a rewritten stored context fails authentication even with
|
||||
@@ -3038,6 +2959,146 @@ mod tests {
|
||||
assert!(matches!(error, KmsError::InvalidKey { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_key_preserves_directory_io_error() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("existing-key", "AES_256", None).await.expect("create key");
|
||||
let backend = LocalKmsBackend { client };
|
||||
let offline_dir = TempDir::new().expect("create offline directory");
|
||||
let offline_key_dir = offline_dir.path().join("keys");
|
||||
fs::rename(temp_dir.path(), &offline_key_dir)
|
||||
.await
|
||||
.expect("move key directory offline");
|
||||
fs::write(temp_dir.path(), b"not a directory")
|
||||
.await
|
||||
.expect("replace key directory with a file");
|
||||
|
||||
let error = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "existing-key".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("unreadable storage must prevent scheduling deletion");
|
||||
|
||||
fs::remove_file(temp_dir.path()).await.expect("remove replacement file");
|
||||
fs::rename(&offline_key_dir, temp_dir.path())
|
||||
.await
|
||||
.expect("restore key directory");
|
||||
assert!(matches!(error, KmsError::IoError { .. }), "got {error:?}");
|
||||
let key = backend
|
||||
.client
|
||||
.load_master_key("existing-key")
|
||||
.await
|
||||
.expect("read retained key");
|
||||
assert_eq!(key.status, KeyStatus::Active, "failed deletion must not mutate key state");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_key_deletion_preserves_directory_io_error() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("existing-key", "AES_256", None).await.expect("create key");
|
||||
let backend = LocalKmsBackend { client };
|
||||
backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "existing-key".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("schedule key deletion");
|
||||
let offline_dir = TempDir::new().expect("create offline directory");
|
||||
let offline_key_dir = offline_dir.path().join("keys");
|
||||
fs::rename(temp_dir.path(), &offline_key_dir)
|
||||
.await
|
||||
.expect("move key directory offline");
|
||||
fs::write(temp_dir.path(), b"not a directory")
|
||||
.await
|
||||
.expect("replace key directory with a file");
|
||||
|
||||
let error = backend
|
||||
.cancel_key_deletion(CancelKeyDeletionRequest {
|
||||
key_id: "existing-key".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect_err("unreadable storage must prevent cancelling deletion");
|
||||
|
||||
fs::remove_file(temp_dir.path()).await.expect("remove replacement file");
|
||||
fs::rename(&offline_key_dir, temp_dir.path())
|
||||
.await
|
||||
.expect("restore key directory");
|
||||
assert!(matches!(error, KmsError::IoError { .. }), "got {error:?}");
|
||||
let key = backend
|
||||
.client
|
||||
.load_master_key("existing-key")
|
||||
.await
|
||||
.expect("read retained key");
|
||||
assert_eq!(
|
||||
key.status,
|
||||
KeyStatus::PendingDeletion,
|
||||
"failed cancellation must retain the deletion state"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_missing_key_remains_not_found() {
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
|
||||
let error = client
|
||||
.load_master_key("missing-key")
|
||||
.await
|
||||
.expect_err("missing key must fail");
|
||||
|
||||
assert!(matches!(error, KmsError::KeyNotFound { key_id } if key_id == "missing-key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_unavailable_directory_is_io_error() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("existing-key", "AES_256", None).await.expect("create key");
|
||||
let offline_dir = TempDir::new().expect("create offline directory");
|
||||
let offline_key_dir = offline_dir.path().join("keys");
|
||||
fs::rename(temp_dir.path(), &offline_key_dir)
|
||||
.await
|
||||
.expect("move key directory offline");
|
||||
|
||||
let error = client
|
||||
.load_master_key("existing-key")
|
||||
.await
|
||||
.expect_err("unavailable key directory must fail");
|
||||
|
||||
fs::rename(&offline_key_dir, temp_dir.path())
|
||||
.await
|
||||
.expect("restore key directory");
|
||||
assert!(matches!(error, KmsError::IoError { .. }), "got {error:?}");
|
||||
let key = client.load_master_key("existing-key").await.expect("read restored key");
|
||||
assert_eq!(key.key_id, "existing-key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_directory_replaced_by_file_is_io_error() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("existing-key", "AES_256", None).await.expect("create key");
|
||||
let offline_dir = TempDir::new().expect("create offline directory");
|
||||
let offline_key_dir = offline_dir.path().join("keys");
|
||||
fs::rename(temp_dir.path(), &offline_key_dir)
|
||||
.await
|
||||
.expect("move key directory offline");
|
||||
fs::write(temp_dir.path(), b"not a directory")
|
||||
.await
|
||||
.expect("replace key directory with a file");
|
||||
|
||||
let error = client
|
||||
.load_master_key("existing-key")
|
||||
.await
|
||||
.expect_err("a file in place of the key directory must fail");
|
||||
|
||||
fs::remove_file(temp_dir.path()).await.expect("remove replacement file");
|
||||
fs::rename(&offline_key_dir, temp_dir.path())
|
||||
.await
|
||||
.expect("restore key directory");
|
||||
assert!(matches!(error, KmsError::IoError { .. }), "got {error:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_accepts_legacy_rfc3339_timestamp() {
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
|
||||
@@ -263,6 +263,14 @@ pub struct ScannerUsageRecoveryIntentResponse {
|
||||
pub mode: String,
|
||||
pub intent_id: String,
|
||||
pub state: String,
|
||||
#[serde(default)]
|
||||
pub actor_sha256: Option<String>,
|
||||
#[serde(default)]
|
||||
pub idempotency_key_sha256: Option<String>,
|
||||
#[serde(default)]
|
||||
pub request_sha256: Option<String>,
|
||||
#[serde(default)]
|
||||
pub accepted_at_unix_secs: Option<u64>,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
@@ -869,6 +877,10 @@ mod tests {
|
||||
"mode": "full-rebuild",
|
||||
"intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"state": "accepted",
|
||||
"actor_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"idempotency_key_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
|
||||
"request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
|
||||
"accepted_at_unix_secs": 7,
|
||||
"future": {"worker": "pending"}
|
||||
}))
|
||||
.unwrap();
|
||||
@@ -877,7 +889,33 @@ mod tests {
|
||||
assert_eq!(intent.mode, "full-rebuild");
|
||||
assert_eq!(intent.intent_id, "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef");
|
||||
assert_eq!(intent.state, "accepted");
|
||||
assert_eq!(
|
||||
intent.actor_sha256.as_deref(),
|
||||
Some("1111111111111111111111111111111111111111111111111111111111111111")
|
||||
);
|
||||
assert_eq!(
|
||||
intent.idempotency_key_sha256.as_deref(),
|
||||
Some("2222222222222222222222222222222222222222222222222222222222222222")
|
||||
);
|
||||
assert_eq!(
|
||||
intent.request_sha256.as_deref(),
|
||||
Some("3333333333333333333333333333333333333333333333333333333333333333")
|
||||
);
|
||||
assert_eq!(intent.accepted_at_unix_secs, Some(7));
|
||||
assert_eq!(intent.extra["future"]["worker"], "pending");
|
||||
|
||||
let legacy_intent: ScannerUsageRecoveryIntentResponse = serde_json::from_value(json!({
|
||||
"status": "accepted",
|
||||
"action": "usage-full-rebuild",
|
||||
"mode": "full-rebuild",
|
||||
"intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"state": "accepted"
|
||||
}))
|
||||
.unwrap();
|
||||
assert!(legacy_intent.actor_sha256.is_none());
|
||||
assert!(legacy_intent.idempotency_key_sha256.is_none());
|
||||
assert!(legacy_intent.request_sha256.is_none());
|
||||
assert!(legacy_intent.accepted_at_unix_secs.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -971,7 +1009,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_async_reset_posts_explicit_intent_contract() {
|
||||
let server = TestServer::spawn(
|
||||
r#"{"status":"accepted","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"accepted"}"#,
|
||||
r#"{"status":"accepted","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"accepted","actor_sha256":"1111111111111111111111111111111111111111111111111111111111111111","idempotency_key_sha256":"2222222222222222222222222222222222222222222222222222222222222222","request_sha256":"3333333333333333333333333333333333333333333333333333333333333333","accepted_at_unix_secs":7}"#,
|
||||
202,
|
||||
)
|
||||
.await;
|
||||
@@ -985,6 +1023,7 @@ mod tests {
|
||||
assert_eq!(accepted.status, "accepted");
|
||||
assert_eq!(accepted.mode, "full-rebuild");
|
||||
assert_eq!(accepted.state, "accepted");
|
||||
assert_eq!(accepted.accepted_at_unix_secs, Some(7));
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/scanner/usage-state/reset");
|
||||
|
||||
@@ -97,9 +97,11 @@ pub use scanner::{
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot,
|
||||
scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
record_dirty_usage_bucket_from_producer, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
pub use segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
use super::*;
|
||||
use crate::segment_invalidation::{
|
||||
MAX_SEGMENT_INVALIDATION_BYTES, MAX_SEGMENT_INVALIDATION_ENTRIES, SegmentInvalidationDomain, SegmentInvalidationEnvelope,
|
||||
SegmentInvalidationError, SegmentInvalidationProducer, SegmentInvalidationProof, admit_segment_invalidation,
|
||||
SegmentInvalidationError, SegmentInvalidationProducer, SegmentInvalidationProducerIdentity, SegmentInvalidationProof,
|
||||
admit_segment_invalidation, complete_segment_invalidation_producers,
|
||||
};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
@@ -11,7 +12,8 @@ const MAX_WALK_SAMPLES: usize = 32;
|
||||
const MAX_WALK_BYTES: usize = 1024;
|
||||
|
||||
fn segment_producers() -> BTreeSet<SegmentInvalidationProducer> {
|
||||
SegmentInvalidationProducer::REQUIRED.into_iter().collect()
|
||||
complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION)
|
||||
.expect("fixture should enumerate the complete production producer matrix")
|
||||
}
|
||||
|
||||
fn segment_envelope() -> SegmentInvalidationEnvelope {
|
||||
|
||||
@@ -39,7 +39,7 @@ use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration, VersioningConfiguration,
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
@@ -1224,8 +1224,9 @@ pub(crate) use cache::{
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot,
|
||||
scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
record_dirty_usage_bucket_from_producer, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||
|
||||
@@ -22,6 +22,11 @@ pub(super) static DIRTY_USAGE_BUCKETS: LazyLock<StdMutex<DirtyUsageBuckets>> = L
|
||||
// matching scope.
|
||||
pub(super) static DIRTY_USAGE_BUCKET_SCOPES: LazyLock<StdMutex<DirtyUsageBucketScopes>> =
|
||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
// Non-authoritative process-local producer coverage. Any future segment reuse
|
||||
// activation must bind this to the exact generation window and durable proof.
|
||||
pub(super) static DIRTY_USAGE_PRODUCER_IDENTITIES: LazyLock<
|
||||
StdMutex<BTreeSet<crate::segment_invalidation::SegmentInvalidationProducerIdentity>>,
|
||||
> = LazyLock::new(|| StdMutex::new(BTreeSet::new()));
|
||||
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
pub(super) static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
@@ -153,6 +158,7 @@ fn apply_scoped_dirty_usage_ack(
|
||||
#[cfg(test)]
|
||||
mod scoped_dirty_usage_tests {
|
||||
use super::*;
|
||||
use crate::segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
|
||||
@@ -213,6 +219,30 @@ mod scoped_dirty_usage_tests {
|
||||
assert_eq!(scopes, original_scopes);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dirty_usage_tracks_known_segment_producer_identities_without_authorizing_unknown_sources() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_object_from_producer("photos", "hot/object", SegmentInvalidationProducerIdentity::PutObject);
|
||||
record_dirty_usage_object_from_producer("photos", "archive/object", SegmentInvalidationProducerIdentity::DeleteObject);
|
||||
record_dirty_usage_bucket_from_producer("photos", SegmentInvalidationProducerIdentity::Unknown);
|
||||
|
||||
assert_eq!(
|
||||
dirty_usage_producer_identities_for_tests(),
|
||||
BTreeSet::from([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
dirty_usage_bucket_scopes_for_tests().get("photos"),
|
||||
Some(&DirtyUsageBucketScope::WholeBucket),
|
||||
"an unknown producer keeps the bucket dirty but must not count as producer coverage"
|
||||
);
|
||||
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
assert!(dirty_usage_producer_identities_for_tests().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||
@@ -225,6 +255,13 @@ fn dirty_usage_bucket_scopes() -> MutexGuard<'static, DirtyUsageBucketScopes> {
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
fn dirty_usage_producer_identities()
|
||||
-> MutexGuard<'static, BTreeSet<crate::segment_invalidation::SegmentInvalidationProducerIdentity>> {
|
||||
DIRTY_USAGE_PRODUCER_IDENTITIES
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
pub(super) fn usize_to_u64_saturated(value: usize) -> u64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX)
|
||||
}
|
||||
@@ -240,6 +277,22 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
return;
|
||||
}
|
||||
|
||||
record_dirty_usage_bucket_inner(bucket);
|
||||
}
|
||||
|
||||
pub fn record_dirty_usage_bucket_from_producer(
|
||||
bucket: &str,
|
||||
producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity,
|
||||
) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
record_segment_invalidation_producer_identity(producer);
|
||||
record_dirty_usage_bucket_inner(bucket);
|
||||
}
|
||||
|
||||
fn record_dirty_usage_bucket_inner(bucket: &str) {
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
@@ -263,6 +316,23 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
/// local: after restart or any unverified distributed path the scanner falls
|
||||
/// back to its ordinary bucket scan.
|
||||
pub fn record_dirty_usage_object(bucket: &str, object: &str) {
|
||||
record_dirty_usage_object_inner(bucket, object);
|
||||
}
|
||||
|
||||
pub fn record_dirty_usage_object_from_producer(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity,
|
||||
) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
record_segment_invalidation_producer_identity(producer);
|
||||
record_dirty_usage_object_inner(bucket, object);
|
||||
}
|
||||
|
||||
fn record_dirty_usage_object_inner(bucket: &str, object: &str) {
|
||||
let Some(top_level_entry) = dirty_usage_top_level_entry(object) else {
|
||||
record_dirty_usage_bucket(bucket);
|
||||
return;
|
||||
@@ -296,6 +366,17 @@ pub fn record_dirty_usage_object(bucket: &str, object: &str) {
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
fn record_segment_invalidation_producer_identity(producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity) {
|
||||
if producer.producer().is_some() {
|
||||
dirty_usage_producer_identities().insert(producer);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn dirty_usage_producer_identities_for_tests() -> BTreeSet<crate::segment_invalidation::SegmentInvalidationProducerIdentity> {
|
||||
dirty_usage_producer_identities().clone()
|
||||
}
|
||||
|
||||
fn dirty_usage_top_level_entry(object: &str) -> Option<String> {
|
||||
let (top_level_entry, _) = object.split_once('/').unwrap_or((object, ""));
|
||||
(!top_level_entry.is_empty()
|
||||
@@ -577,6 +658,7 @@ pub(super) fn dirty_usage_bucket_count() -> usize {
|
||||
pub(crate) fn clear_dirty_usage_buckets_for_tests() {
|
||||
dirty_usage_buckets().clear();
|
||||
dirty_usage_bucket_scopes().clear();
|
||||
dirty_usage_producer_identities().clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -16,23 +16,29 @@ use super::*;
|
||||
use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision};
|
||||
|
||||
async fn create_cohort_bucket(store: &ECStore, bucket: &str) {
|
||||
create_cohort_bucket_objects(store, bucket, 1).await;
|
||||
}
|
||||
|
||||
async fn create_cohort_bucket_objects(store: &ECStore, bucket: &str, objects: usize) {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket");
|
||||
for set in store.all_set_disks() {
|
||||
let mut reader = ScannerPutObjReader::from_vec(b"cohort".to_vec());
|
||||
set.put_object(
|
||||
bucket,
|
||||
"initial",
|
||||
&mut reader,
|
||||
&ScannerObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("fixture object and all rename tails should persist");
|
||||
for index in 0..objects {
|
||||
let mut reader = ScannerPutObjReader::from_vec(b"cohort".to_vec());
|
||||
set.put_object(
|
||||
bucket,
|
||||
&format!("object-{index:04}"),
|
||||
&mut reader,
|
||||
&ScannerObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("fixture object and all rename tails should persist");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,6 +174,92 @@ async fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let flat = format!("a-flat-{}", Uuid::new_v4().simple());
|
||||
let small = format!("z-small-{}", Uuid::new_v4().simple());
|
||||
create_cohort_bucket_objects(&store, &flat, 6).await;
|
||||
create_cohort_bucket(&store, &small).await;
|
||||
let cohort = Arc::new(StdMutex::new(ScannerServiceCohort::default()));
|
||||
|
||||
let expected_flat = store
|
||||
.all_set_disks()
|
||||
.iter()
|
||||
.map(|set| (DataUsageCacheSource::new(set.pool_index, set.set_index), flat.clone()))
|
||||
.collect::<HashSet<_>>();
|
||||
let expected_small = store
|
||||
.all_set_disks()
|
||||
.iter()
|
||||
.map(|set| (DataUsageCacheSource::new(set.pool_index, set.set_index), small.clone()))
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (result, usage) = run_cohort_cycle(&store, cohort.clone(), 1, budget.clone()).await;
|
||||
assert_eq!(result.status, ScannerCycleStatus::Incomplete);
|
||||
assert!(usage.is_none(), "wide-bucket budget exhaustion must not publish a partial aggregate");
|
||||
assert!(budget.budget_elapsed());
|
||||
let first_round_admitted = cohort
|
||||
.lock()
|
||||
.expect("cohort lock")
|
||||
.admitted_members()
|
||||
.into_iter()
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(
|
||||
!first_round_admitted.is_disjoint(&expected_flat),
|
||||
"the first fixed budget round should exercise the wide flat bucket"
|
||||
);
|
||||
assert!(
|
||||
first_round_admitted.is_disjoint(&expected_small),
|
||||
"a small bucket not yet reached by the real scanner must not be marked admitted"
|
||||
);
|
||||
|
||||
for cycle in 2..=4 {
|
||||
let ctx = CancellationToken::new();
|
||||
let (result, usage) = run_cohort_cycle(
|
||||
&store,
|
||||
cohort.clone(),
|
||||
cycle,
|
||||
ScannerCycleBudget::new_with_progress_tracking(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.status, ScannerCycleStatus::Incomplete);
|
||||
assert!(usage.is_none(), "mixed partial coverage still cannot publish the set root");
|
||||
}
|
||||
let admitted_after_budgeted_rounds = cohort
|
||||
.lock()
|
||||
.expect("cohort lock")
|
||||
.admitted_members()
|
||||
.into_iter()
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(
|
||||
expected_small.is_subset(&admitted_after_budgeted_rounds),
|
||||
"tracked small buckets must receive real execution opportunities within their fixed service-round bound"
|
||||
);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let (result, usage) =
|
||||
run_cohort_cycle(&store, cohort, 5, ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default())).await;
|
||||
assert_eq!(result.status, ScannerCycleStatus::Complete);
|
||||
assert_eq!(usage.expect("final complete aggregate").objects_total_count, 14);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||
|
||||
@@ -25,6 +25,7 @@ pub enum SegmentInvalidationError {
|
||||
ByteLimit,
|
||||
InvalidProof,
|
||||
InvalidKey,
|
||||
UnknownProducer,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
@@ -50,6 +51,74 @@ impl SegmentInvalidationProducer {
|
||||
];
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum SegmentInvalidationProducerIdentity {
|
||||
PutObject,
|
||||
DeleteObject,
|
||||
DeleteMarker,
|
||||
CompleteMultipartUpload,
|
||||
Replication,
|
||||
TierTransition,
|
||||
TierExpiration,
|
||||
DirectoryObject,
|
||||
Unknown,
|
||||
TestFixture,
|
||||
}
|
||||
|
||||
impl SegmentInvalidationProducerIdentity {
|
||||
pub const REQUIRED_PRODUCTION: [Self; 8] = [
|
||||
Self::PutObject,
|
||||
Self::DeleteObject,
|
||||
Self::DeleteMarker,
|
||||
Self::CompleteMultipartUpload,
|
||||
Self::Replication,
|
||||
Self::TierTransition,
|
||||
Self::TierExpiration,
|
||||
Self::DirectoryObject,
|
||||
];
|
||||
|
||||
pub fn producer(self) -> Option<SegmentInvalidationProducer> {
|
||||
match self {
|
||||
Self::PutObject => Some(SegmentInvalidationProducer::Put),
|
||||
Self::DeleteObject => Some(SegmentInvalidationProducer::Delete),
|
||||
Self::DeleteMarker => Some(SegmentInvalidationProducer::DeleteMarker),
|
||||
Self::CompleteMultipartUpload => Some(SegmentInvalidationProducer::Multipart),
|
||||
Self::Replication => Some(SegmentInvalidationProducer::Replication),
|
||||
Self::TierTransition | Self::TierExpiration => Some(SegmentInvalidationProducer::Tier),
|
||||
Self::DirectoryObject => Some(SegmentInvalidationProducer::DirectoryObject),
|
||||
Self::Unknown | Self::TestFixture => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn complete_segment_invalidation_producers<I>(
|
||||
identities: I,
|
||||
) -> Result<BTreeSet<SegmentInvalidationProducer>, SegmentInvalidationError>
|
||||
where
|
||||
I: IntoIterator<Item = SegmentInvalidationProducerIdentity>,
|
||||
{
|
||||
let mut covered_identities = BTreeSet::new();
|
||||
let mut producers = BTreeSet::new();
|
||||
for identity in identities {
|
||||
let Some(producer) = identity.producer() else {
|
||||
return Err(SegmentInvalidationError::UnknownProducer);
|
||||
};
|
||||
covered_identities.insert(identity);
|
||||
producers.insert(producer);
|
||||
}
|
||||
if SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION
|
||||
.iter()
|
||||
.all(|identity| covered_identities.contains(identity))
|
||||
&& SegmentInvalidationProducer::REQUIRED
|
||||
.iter()
|
||||
.all(|producer| producers.contains(producer))
|
||||
{
|
||||
Ok(producers)
|
||||
} else {
|
||||
Err(SegmentInvalidationError::InvalidProof)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum SegmentInvalidationDomain {
|
||||
LocalSingleSet,
|
||||
@@ -170,7 +239,8 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
fn producers() -> BTreeSet<SegmentInvalidationProducer> {
|
||||
SegmentInvalidationProducer::REQUIRED.into_iter().collect()
|
||||
complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION)
|
||||
.expect("production producer matrix should be complete")
|
||||
}
|
||||
|
||||
fn envelope() -> SegmentInvalidationEnvelope {
|
||||
@@ -320,6 +390,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segment_invalidation_producer_identities_must_be_known_and_complete() {
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION),
|
||||
Ok(SegmentInvalidationProducer::REQUIRED.into_iter().collect())
|
||||
);
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||
SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||
SegmentInvalidationProducerIdentity::Replication,
|
||||
SegmentInvalidationProducerIdentity::TierTransition,
|
||||
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||
SegmentInvalidationProducerIdentity::Unknown,
|
||||
]),
|
||||
Err(SegmentInvalidationError::UnknownProducer)
|
||||
);
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||
SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||
SegmentInvalidationProducerIdentity::Replication,
|
||||
SegmentInvalidationProducerIdentity::TierTransition,
|
||||
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||
SegmentInvalidationProducerIdentity::TestFixture,
|
||||
]),
|
||||
Err(SegmentInvalidationError::UnknownProducer)
|
||||
);
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||
SegmentInvalidationProducerIdentity::Replication,
|
||||
SegmentInvalidationProducerIdentity::TierTransition,
|
||||
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||
]),
|
||||
Err(SegmentInvalidationError::InvalidProof)
|
||||
);
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||
SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||
SegmentInvalidationProducerIdentity::Replication,
|
||||
SegmentInvalidationProducerIdentity::TierTransition,
|
||||
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||
]),
|
||||
Err(SegmentInvalidationError::InvalidProof)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segment_invalidation_entries_are_bounded_and_key_checked() {
|
||||
let envelope = envelope();
|
||||
|
||||
@@ -66,6 +66,7 @@ The manifest has the following JSON contract (all fields are required):
|
||||
| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 900..86400 seconds for measured runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. |
|
||||
| `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. |
|
||||
| `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. |
|
||||
| `release_evidence` | Required for `measured` runs. It binds the 3x4 EC8+4 topology, multi-pool/multi-set coverage, per-node metrics endpoints, same-window distributed sampling, process restart and crash-restart fault modes, mixed-version reader/writer/rollback participation, and allocation/flamegraph/RSS/save-frequency profile artifact requirements. Synthetic runs do not need this field and still cannot approve release evidence. |
|
||||
| `oracles` | A map with all five scenario names. Each value contains positive integer `objects`, `versions`, `bytes`, and `sha256` of the independently prepared canonical object/version/content manifest. |
|
||||
| `expected_healed_objects` | A map with all five scenario names and independently seeded repair counts. Running-heal and MRF-replay require a positive count. |
|
||||
|
||||
@@ -75,6 +76,14 @@ object/version/content result. Fix the foreground arrival rate (offered load),
|
||||
cache preparation procedure, configuration, and hardware across every leg.
|
||||
Do not include credentials in the manifest, adapter output, or saved commands;
|
||||
the collector reads `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` from its environment.
|
||||
The adapter must echo the measured run's `release_evidence` object in every
|
||||
measurement response. A mismatch fails the cell because it means the deployment,
|
||||
mixed-version set, crash mode, or profiler contract no longer matches the
|
||||
operator-reviewed manifest. This echo is provenance binding only; it does not
|
||||
replace the independent correctness oracle, distributed metrics samples, profile
|
||||
artifacts, or ABBA comparison thresholds. The summary tool revalidates the same
|
||||
manifest contract before it can print a measured PASS result, so hand-built or
|
||||
trimmed reports without this provenance fail closed.
|
||||
|
||||
#### Deployment Adapter Contract
|
||||
|
||||
@@ -204,6 +213,14 @@ The command prints only `PASS scanner_heal_perf ...` for measured passing ABBA
|
||||
evidence, otherwise `FAIL scanner_heal_perf ...`. The JSON and Markdown outputs
|
||||
carry the key p99/throughput/P1/P2/cache-cost fields and artifact provenance
|
||||
hashes; raw per-cell logs remain in the original artifact tree for audit.
|
||||
Failed or interrupted ABBA reports that contain only `status`, `performance`,
|
||||
`completed_cells`, and `error` also summarize as `FAIL`; they do not become
|
||||
performance evidence, and a missing comparison matrix is accepted only for a
|
||||
non-passing report.
|
||||
Measured passing reports must also retain the W10/W11 foreground-pressure,
|
||||
heal-lock-wait, and heal-attempt-cost fields emitted by the ABBA evaluator. If
|
||||
those fields are removed, empty, malformed, or length-mismatched, the quiet
|
||||
summary fails closed instead of treating the report as performance evidence.
|
||||
|
||||
They cover the complete 120-cell schedule, data isolation, missing builds and
|
||||
oracles, zero samples/requests, swallowed request errors, offered-load drift,
|
||||
|
||||
+74
-17
@@ -43,17 +43,18 @@ Promotion rule: never promote a report-only lane to required from one green run.
|
||||
| PR, non-doc change | `End-to-End Tests` | `ci.yml` `e2e-tests` | Report-only | `cargo nextest run --profile e2e-smoke -p e2e_test`, then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>`; membership guards `scripts/check_test_wiring.py --check-profile e2e-smoke <listing.json>` and `scripts/check_security_smoke_count.sh check <listing.json>` |
|
||||
| PR, non-doc change | `S3 Implemented Tests` | `ci.yml` `s3-implemented-tests` | Report-only | build `rustfs`, then `scripts/s3-tests/run.sh` with the job's `DEPLOY_MODE` / `TEST_MODE` / `MAXFAIL` env |
|
||||
| PR, non-doc change | `S3 Lifecycle Behavior Tests` | `ci.yml` `s3-lifecycle-behavior-tests` | Report-only | `scripts/s3-tests/run.sh` with the job's accelerated-scanner env |
|
||||
| PR touching `paths` in `audit.yml` | `Cargo Deny`, `Workflow Pin Report`, `Dependency Review` | `audit.yml` `cargo-deny`, `workflow-pin-report`, `dependency-review` | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| PR to `main` or `release` touching `paths` in `audit.yml` | `Cargo Deny`, `Workflow Pin Report`, `Dependency Review` | `audit.yml` `cargo-deny`, `workflow-pin-report`, `dependency-review` | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| Push to `main` or `release` touching `paths` in `audit.yml` | `Cargo Deny`, `Workflow Pin Report` | `audit.yml` `cargo-deny`, `workflow-pin-report` | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| PR touching `paths` in `architecture-migration-rules.yml` | `Architecture Migration Rules` | `architecture-migration-rules.yml` `architecture-migration-rules` | Report-only | `scripts/check_architecture_migration_rules.sh` |
|
||||
| PR touching `paths` in `nix.yml` | `Nix Build & Check` | `nix.yml` `nix-validation` | Report-only | `nix flake check` |
|
||||
| PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / <target>` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
|
||||
| PR touching `paths` in `windows-filesystem.yml` | `Rename Safety` | `windows-filesystem.yml` `rename-safety` | Report-only | the `cargo test -p rustfs-ecstore --lib <filter>` commands in the job, on Windows |
|
||||
| PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` |
|
||||
| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
|
||||
| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata`, `ODM configuration recovery after rc.5 rollback`, `Multipart layouts survive the rc.5 upgrade`, `rc.5 multipart replication baseline` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
|
||||
| PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` |
|
||||
| PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test <name> -- --ignored --test-threads=1` commands |
|
||||
| PR limited to main-CI-excluded paths | `Quick Checks`, `Test and Lint` | `ci-docs-only.yml` `quick-checks`, `test-and-lint` | Required | `git diff --check`; `make doc-paths-check`; `scripts/check_no_planning_docs.sh` |
|
||||
| `merge_group`; push to `main` | `End-to-End Tests (full merge gate)` | `ci.yml` `e2e-full` | Report-only | `cargo nextest run --profile e2e-full -p e2e_test` |
|
||||
| `merge_group`; push to `main` or `release` | `End-to-End Tests (full merge gate)` | `ci.yml` `e2e-full` | Report-only | `cargo nextest run --profile e2e-full -p e2e_test` |
|
||||
|
||||
e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e-<profile>-selection.txt`, so a silent test drop fails closed.
|
||||
|
||||
@@ -62,7 +63,7 @@ cost. `data_usage_test` runs in the PR `e2e-smoke` lane so changes that affect
|
||||
authoritative scanner usage publication, quota-visible usage, or admin usage
|
||||
snapshots get an end-to-end signal before merge review. `heal_erasure_disk_rebuild_test`
|
||||
runs in `e2e-full` so core erasure heal rebuild regressions are caught no later
|
||||
than the merge queue or `main` push lane; it also remains in `e2e-nightly` with
|
||||
than the merge queue or `main`/`release` push lane; it also remains in `e2e-nightly` with
|
||||
the serialized cluster fault-domain suites for scheduled soak signal.
|
||||
|
||||
## Scheduled validation
|
||||
@@ -85,7 +86,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
|
||||
| `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow |
|
||||
| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` |
|
||||
| `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
|
||||
| `e2e-upgrade.yml` (weekly) | `upgrade` (4-case matrix) | upgrade and rollback gate; server logs | no | see the PR row |
|
||||
| `e2e-upgrade.yml` (weekly) | `upgrade` (7-case matrix) | upgrade and rollback gate; server logs | no | see the PR row |
|
||||
| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row |
|
||||
| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row |
|
||||
| `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on a never-created or stale schedule | n/a | dispatch |
|
||||
@@ -148,11 +149,21 @@ and exact S3 content.
|
||||
This case is a **four-node, one-drive-per-node process-restart test**. It is not
|
||||
power-loss validation, a 3x4 EC8+4 experiment, an all-version inventory, or proof
|
||||
of scanner enumeration, exact MRF disposition, legacy migration, or rollback.
|
||||
The registry keeps all G01-G14/P1-P4 and R-E/R-D/R-L release requirements pending
|
||||
until their actual feature-specific oracles and required topologies exist.
|
||||
Missing cases cannot be supplied by synthetic W20 results. W20's bounded JSON
|
||||
and file-hash helpers are reused; its ABBA performance contracts remain in
|
||||
The schema 2 registry separates the implemented single-set restart lane from
|
||||
structured release lanes for authority coverage, checkpoint/crash, status and
|
||||
outcome, MRF responsibility, mixed-version rollback, scheduler pressure,
|
||||
maintenance producers, and EC8+4 multi-set coverage. All G01-G14/P1-P4 and
|
||||
R-E/R-D/R-L release requirements stay `pending` until their actual
|
||||
feature-specific oracles, measurements and required topologies exist. Missing
|
||||
cases cannot be supplied by synthetic W20 results. W20's bounded JSON and
|
||||
file-hash helpers are reused; its ABBA performance contracts remain in
|
||||
`docs/operations/scanner-benchmark-runbook.md`.
|
||||
Measured ABBA manifests must also carry the runbook's `release_evidence`
|
||||
contract. The runner rejects reports that cannot bind the exact 3x4 EC8+4
|
||||
topology, multi-pool/multi-set shape, distributed same-window metrics endpoints,
|
||||
restart/crash modes, mixed-version reader/writer/rollback participation, and
|
||||
allocation/flamegraph/RSS/save-frequency profile artifact plan. Synthetic runs
|
||||
and manifests missing that contract remain harness-only evidence.
|
||||
|
||||
### Recording One Case
|
||||
|
||||
@@ -232,14 +243,60 @@ For automation, `--check-scanner-heal-release "$RUN_DIR"` emits one compact
|
||||
JSON decision and exits nonzero while blocked. `verified_cases` contains only
|
||||
cases that pass the complete receipt, build provenance, nextest/JUnit and real
|
||||
oracle checks; `rejected_cases` names registered cases that do not, and
|
||||
`pending_gates` names the unimplemented release requirements. Approval requires
|
||||
every registered case to verify, `pending_gates` to be empty, and a future
|
||||
registry schema capable of representing the complete release matrix. Schema 1
|
||||
is deliberately marked `release_schema_capable: false`: it models only the
|
||||
single-version, unversioned-object restart/crash cases and cannot represent
|
||||
mixed-version, rollback, EC8+4 or performance evidence. A focused run,
|
||||
synthetic harness, compile-only result, skipped/retried test, ordinary CI
|
||||
success, or removal of pending text therefore cannot become a release approval.
|
||||
`pending_gates` names the unimplemented release requirements and
|
||||
`pending_lanes` names the structured release lanes that still need real
|
||||
evidence. Schema 1 is deliberately marked `release_schema_capable: false`
|
||||
because it models only the single-version, unversioned-object restart/crash
|
||||
cases. Schema 2 can describe the wider release matrix, but approval still
|
||||
requires every registered case to verify and every required gate to leave
|
||||
`pending` only after a future checker can bind it to real feature-specific
|
||||
evidence. The current checker hard-rejects missing structured requirements and
|
||||
pending gates mapped to an implemented lane, so clearing pending text cannot
|
||||
become approval. A focused run, synthetic harness, compile-only result,
|
||||
skipped/retried test, ordinary CI success, or unregistered mixed-version,
|
||||
rollback, EC8+4 or performance claim therefore cannot become a release approval.
|
||||
For high-risk rollback gates, `evidence_fields` records the specific proof
|
||||
fields that a future real-evidence checker must bind before a pending gate can
|
||||
move out of the blocked set. G03 keeps scoped ACK tied to durable root
|
||||
publication, ACK request identity, participating peer capability snapshots, and
|
||||
mixed-peer fallback oracles; G09 keeps mixed-version reader, writer, and rollback
|
||||
payload evidence explicit. These fields are part of the release contract, not
|
||||
evidence by themselves.
|
||||
|
||||
When the real release lanes have produced their dedicated artifacts, validate
|
||||
the complete hard-gate bundle with:
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/check_test_wiring.py \
|
||||
--check-scanner-heal-release-bundle /path/to/release-evidence.json
|
||||
```
|
||||
|
||||
The bundle checker is intentionally stricter than the case checker. It requires
|
||||
schema 2 registry metadata, `evidence: measured`, the current checkout revision,
|
||||
all G01-G14/P1-P4/R-E/R-D/R-L gates, per-gate `status: pass`, lane identity,
|
||||
relative artifact paths, matching SHA256 hashes, and non-empty summaries. It
|
||||
also binds each evidence field to its own run provenance: `source_revision`,
|
||||
`run_id`, `measurement_window_id`, timezone-qualified `started_at` and
|
||||
`finished_at`, command arguments, and artifact format. The field
|
||||
`source_revision` must match the bundle revision, and measured performance
|
||||
duration cannot exceed the recorded run window.
|
||||
|
||||
The hard evidence shape remains claim-specific: mixed-version gates must name at
|
||||
least two participating versions, crash/durable replay gates must include
|
||||
crash-boundary evidence, G14 must record EC8+4 with at least three nodes and four
|
||||
drives per node plus multi-set and multi-pool evidence, performance gates need
|
||||
measured durations, P3's pressure run needs at least two hours, and P1 needs a
|
||||
symbolized profile summary with resolved samples. Every G14 field and every
|
||||
performance gate's fields must also share one `measurement_window_id`, so EC8+4,
|
||||
multi-set/multi-pool, ABBA, throughput, and profiling artifacts cannot be
|
||||
stitched together from unrelated runs. Missing, synthetic, stale, tampered,
|
||||
undersized, or topology-mismatched evidence returns a compact blocked or invalid
|
||||
JSON result and a nonzero exit.
|
||||
|
||||
This command validates the evidence package; it does not create evidence. A
|
||||
handwritten JSON file, a synthetic harness pass, a single focused case, or a
|
||||
local unit fixture still cannot satisfy the distributed, mixed-version,
|
||||
crash-restart, durable MRF replay, EC8+4, ABBA, or profiling gates.
|
||||
|
||||
Run parser/receipt regressions with
|
||||
`scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures
|
||||
|
||||
@@ -75,6 +75,10 @@ struct ScannerRecoveryIntentResponse {
|
||||
mode: String,
|
||||
intent_id: String,
|
||||
state: String,
|
||||
actor_sha256: String,
|
||||
idempotency_key_sha256: String,
|
||||
request_sha256: String,
|
||||
accepted_at_unix_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -332,6 +336,10 @@ fn scanner_recovery_intent_record_response(
|
||||
mode: record.mode,
|
||||
intent_id: record.intent_id,
|
||||
state: record.state,
|
||||
actor_sha256: record.actor_sha256,
|
||||
idempotency_key_sha256: record.idempotency_key_sha256,
|
||||
request_sha256: record.request_sha256,
|
||||
accepted_at_unix_secs: record.accepted_at_unix_secs,
|
||||
};
|
||||
let body = serde_json::to_vec(&response).map_err(|err| {
|
||||
S3Error::with_message(
|
||||
@@ -371,10 +379,7 @@ fn scanner_recovery_intent_accept_response(
|
||||
|
||||
fn scanner_recovery_intent_executor_id(result: &rustfs_scanner::ScannerRecoveryIntentAcceptResult) -> Option<String> {
|
||||
match result {
|
||||
rustfs_scanner::ScannerRecoveryIntentAcceptResult::Accepted { record }
|
||||
| rustfs_scanner::ScannerRecoveryIntentAcceptResult::Replayed { record }
|
||||
if matches!(record.state.as_str(), "accepted" | "running") =>
|
||||
{
|
||||
rustfs_scanner::ScannerRecoveryIntentAcceptResult::Accepted { record } if record.state == "accepted" => {
|
||||
Some(record.intent_id.clone())
|
||||
}
|
||||
_ => None,
|
||||
@@ -673,7 +678,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_recovery_intent_executor_only_starts_non_terminal_work() {
|
||||
fn scanner_recovery_intent_executor_starts_only_newly_accepted_work() {
|
||||
let mut record = rustfs_scanner::ScannerRecoveryIntentRecord {
|
||||
schema_version: 1,
|
||||
intent_id: "0".repeat(64),
|
||||
@@ -699,7 +704,16 @@ mod tests {
|
||||
record: record.clone(),
|
||||
})
|
||||
.as_deref(),
|
||||
Some(record.intent_id.as_str())
|
||||
None,
|
||||
"a lost-response retry must not start a duplicate executor"
|
||||
);
|
||||
record.state = "accepted".to_string();
|
||||
assert!(
|
||||
scanner_recovery_intent_executor_id(&rustfs_scanner::ScannerRecoveryIntentAcceptResult::Replayed {
|
||||
record: record.clone(),
|
||||
})
|
||||
.is_none(),
|
||||
"replayed accepted records remain durable for startup/control recovery instead of duplicating work"
|
||||
);
|
||||
record.state = "completed".to_string();
|
||||
assert!(
|
||||
|
||||
@@ -814,7 +814,11 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject,
|
||||
);
|
||||
Ok::<_, S3Error>((oi, dest_versioned))
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1175,7 +1175,12 @@ impl DefaultObjectUsecase {
|
||||
let manager = get_capacity_manager();
|
||||
manager.record_write_operation().await;
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
||||
let producer = if delete_marker && version_id_clone.is_none() {
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteMarker
|
||||
} else {
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteObject
|
||||
};
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(&bucket, &key, producer);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
@@ -689,7 +689,11 @@ impl DefaultObjectUsecase {
|
||||
schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await;
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||
);
|
||||
Ok::<_, ApiError>(obj_info)
|
||||
}
|
||||
});
|
||||
|
||||
@@ -2054,7 +2054,11 @@ impl DefaultObjectUsecase {
|
||||
schedule_object_replication(obj_info.clone(), store, dsc).await;
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start);
|
||||
|
||||
let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
|
||||
@@ -3475,6 +3475,9 @@ mod tests {
|
||||
1,
|
||||
"post-admission response loss must leave exactly one canonical task"
|
||||
);
|
||||
if let Some(cache) = super::HEAL_CONTROL_REPLAY_CACHE.get() {
|
||||
cache.lock().await.clear();
|
||||
}
|
||||
|
||||
let mut retry = connect_faulty_heal_control_client(
|
||||
Arc::clone(&manager),
|
||||
|
||||
+624
-28
@@ -13,7 +13,7 @@ import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest import mock
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
@@ -27,6 +27,47 @@ from scanner_abba import MAX_JSON_BYTES, digest, number, read_json, require, sha
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCANNER_HEAL_REGISTRY_SCHEMA_MAX = 2
|
||||
SCANNER_HEAL_RELEASE_REQUIRED_GATES = (
|
||||
"G01", "G02", "G03", "G04", "G05", "G06", "G07", "G08", "G09", "G10", "G11", "G12", "G13", "G14",
|
||||
"P1", "P2", "P3", "P4", "R-E", "R-D", "R-L",
|
||||
)
|
||||
SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS = {
|
||||
"G03": (
|
||||
"durable_root_publication_proof",
|
||||
"scoped_ack_request_identity",
|
||||
"participating_peer_capability_snapshot",
|
||||
"mixed_peer_ack_fallback_oracle",
|
||||
),
|
||||
"G09": (
|
||||
"mixed_version_reader_evidence",
|
||||
"mixed_version_writer_evidence",
|
||||
"rollback_payload_evidence",
|
||||
),
|
||||
}
|
||||
SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS = {
|
||||
"G01": ("root_authority_evidence", "quota_authority_evidence"),
|
||||
"G02": ("bounded_checkpoint_oracle", "independent_version_inventory"),
|
||||
"G03": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G03"],
|
||||
"G04": ("cache_boundary_crash_evidence", "root_floor_intent_crash_evidence"),
|
||||
"G05": ("per_object_outcome_oracle", "terminal_retention_bounds"),
|
||||
"G06": ("concurrent_status_evidence", "legacy_client_compatibility", "truncation_behavior"),
|
||||
"G07": ("mrf_responsibility_oracle", "commit_boundary_crash_matrix"),
|
||||
"G08": ("mrf_capacity_evidence", "disk_full_matrix", "replica_loss_matrix"),
|
||||
"G09": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G09"],
|
||||
"G10": ("scheduler_bound_evidence", "pressure_recovery_evidence"),
|
||||
"G11": ("maintenance_producer_matrix", "complete_producer_inventory"),
|
||||
"G12": ("reset_quota_path_evidence", "settlement_quota_path_evidence"),
|
||||
"G13": ("quorum_minus_one_matrix", "unknown_disk_remount_matrix", "object_lock_dry_run_grace_evidence"),
|
||||
"G14": ("same_window_field_evidence", "ec8_4_evidence", "multi_set_evidence", "multi_pool_evidence"),
|
||||
"P1": ("cold_walk_share_measurement", "foreground_latency_throughput_measurement", "profile_evidence"),
|
||||
"P2": ("post_stop_convergence_measurement", "cold_segment_reuse_measurement"),
|
||||
"P3": ("two_hour_pressure_measurement", "heal_capacity_measurement", "recovery_window_measurement"),
|
||||
"P4": ("mrf_scale_measurement", "mrf_replay_cost_measurement", "retained_responsibility_evidence"),
|
||||
"R-E": ("fixed_budget_restart_evidence", "enumeration_evidence", "classification_evidence"),
|
||||
"R-D": ("manager_disposition_evidence", "event_disposition_evidence", "ledger_disposition_evidence", "grace_handling"),
|
||||
"R-L": ("legacy_source_conflict_evidence", "migration_gap_evidence", "crash_safe_source_retirement_evidence"),
|
||||
}
|
||||
SCHEDULED_ALERT_WORKFLOWS = tuple(
|
||||
item["workflow"]
|
||||
for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text())
|
||||
@@ -886,9 +927,30 @@ def evidence_integer(value: object, name: str, minimum: int, maximum: int) -> in
|
||||
return value
|
||||
|
||||
|
||||
def evidence_string(value: object, name: str, pattern: str | None = None) -> str:
|
||||
require(isinstance(value, str) and value.strip(), f"invalid string {name}")
|
||||
if pattern is not None:
|
||||
require(re.fullmatch(pattern, value) is not None, f"invalid string {name}")
|
||||
return value
|
||||
|
||||
|
||||
def evidence_timestamp(value: object, name: str) -> datetime:
|
||||
text = evidence_string(value, name)
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError as error:
|
||||
raise ValueError(f"invalid timestamp {name}") from error
|
||||
require(parsed.tzinfo is not None, f"{name} must include timezone")
|
||||
return parsed
|
||||
|
||||
|
||||
def scanner_heal_registry_schema(registry: dict[str, object]) -> int:
|
||||
return evidence_integer(registry.get("schema"), "registry schema", 1, SCANNER_HEAL_REGISTRY_SCHEMA_MAX)
|
||||
|
||||
|
||||
def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]:
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
evidence_integer(registry.get("schema"), "registry schema", 1, 1)
|
||||
scanner_heal_registry_schema(registry)
|
||||
cases = registry.get("cases")
|
||||
require(isinstance(cases, dict) and cases, "invalid scanner/heal registry")
|
||||
names = set()
|
||||
@@ -902,10 +964,103 @@ def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]:
|
||||
f"invalid evidence for {case_id}")
|
||||
require(type(requirement.get("unclean_shutdown_marker")) is bool,
|
||||
f"invalid unclean-shutdown marker expectation for {case_id}")
|
||||
if "erasure" in requirement:
|
||||
erasure = requirement["erasure"]
|
||||
require(isinstance(erasure, dict), f"invalid erasure expectation for {case_id}")
|
||||
data_blocks = evidence_integer(erasure.get("data_blocks"), f"{case_id} data_blocks", 1, 16)
|
||||
parity_blocks = evidence_integer(erasure.get("parity_blocks"), f"{case_id} parity_blocks", 1, 16)
|
||||
require(data_blocks >= parity_blocks, f"invalid erasure geometry for {case_id}")
|
||||
require(data_blocks + parity_blocks == requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"],
|
||||
f"erasure geometry differs from topology for {case_id}")
|
||||
if "erasure_set_drive_count" in requirement:
|
||||
erasure_set_drive_count = evidence_integer(requirement.get("erasure_set_drive_count"),
|
||||
f"{case_id} erasure_set_drive_count", 1, 64)
|
||||
require(erasure_set_drive_count == requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"],
|
||||
f"erasure set drive count differs from topology for {case_id}")
|
||||
names.add(oracle)
|
||||
return tuple(sorted(names))
|
||||
|
||||
|
||||
def scanner_heal_release_requirements(registry: dict[str, object]) -> tuple[dict[str, dict[str, object]], bool, list[str]]:
|
||||
schema = scanner_heal_registry_schema(registry)
|
||||
if schema == 1:
|
||||
pending = registry.get("release_pending")
|
||||
require(isinstance(pending, dict), "invalid scanner/heal release requirements")
|
||||
requirements = {}
|
||||
for gate, reason in pending.items():
|
||||
require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None,
|
||||
"invalid scanner/heal release gate")
|
||||
require(isinstance(reason, str) and reason.strip(), f"missing release requirement for {gate}")
|
||||
requirements[gate] = {"gate": gate, "status": "pending", "lane": "schema-1-pending",
|
||||
"description": reason, "requires": [reason]}
|
||||
return requirements, False, ["schema-1-pending"] if requirements else []
|
||||
|
||||
lanes = registry.get("release_lanes")
|
||||
cases = registry.get("cases")
|
||||
require(isinstance(cases, dict) and cases, "invalid scanner/heal registry")
|
||||
require(isinstance(lanes, dict) and lanes, "invalid scanner/heal release lanes")
|
||||
lane_statuses = {}
|
||||
lane_gates = {}
|
||||
for lane_id, lane in lanes.items():
|
||||
require(isinstance(lane_id, str) and re.fullmatch(r"[a-z0-9-]+", lane_id) is not None,
|
||||
"invalid scanner/heal release lane")
|
||||
require(isinstance(lane, dict), f"invalid release lane {lane_id}")
|
||||
status = lane.get("status")
|
||||
require(status in ("implemented", "pending"), f"invalid release lane status for {lane_id}")
|
||||
lane_statuses[lane_id] = status
|
||||
if status == "implemented":
|
||||
lane_cases = lane.get("cases")
|
||||
require(isinstance(lane_cases, list) and lane_cases and all(isinstance(case, str) and case for case in lane_cases),
|
||||
f"implemented release lane {lane_id} has no cases")
|
||||
require(all(case in cases for case in lane_cases), f"implemented release lane {lane_id} has unknown cases")
|
||||
else:
|
||||
gates = lane.get("gates")
|
||||
require(isinstance(gates, list) and gates and all(isinstance(gate, str) and gate for gate in gates),
|
||||
f"pending release lane {lane_id} has no gates")
|
||||
lane_gates[lane_id] = set(gates)
|
||||
|
||||
raw_requirements = registry.get("release_requirements")
|
||||
require(isinstance(raw_requirements, list) and raw_requirements, "invalid scanner/heal release requirements")
|
||||
requirements: dict[str, dict[str, object]] = {}
|
||||
for item in raw_requirements:
|
||||
require(isinstance(item, dict), "invalid scanner/heal release requirement")
|
||||
gate = item.get("gate")
|
||||
require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None,
|
||||
"invalid scanner/heal release gate")
|
||||
require(gate not in requirements, f"duplicate scanner/heal release gate {gate}")
|
||||
status = item.get("status")
|
||||
require(status == "pending", f"release gate {gate} must stay pending until real evidence is registered")
|
||||
lane = item.get("lane")
|
||||
require(isinstance(lane, str) and lane in lane_statuses, f"unknown release lane for {gate}")
|
||||
require(lane_statuses[lane] == "pending", f"pending release gate {gate} mapped to non-pending lane {lane}")
|
||||
description = item.get("description")
|
||||
require(isinstance(description, str) and description.strip(), f"missing release requirement for {gate}")
|
||||
requires = item.get("requires")
|
||||
require(isinstance(requires, list) and requires and
|
||||
all(isinstance(requirement, str) and requirement.strip() for requirement in requires),
|
||||
f"missing concrete evidence requirements for {gate}")
|
||||
evidence_fields = item.get("evidence_fields", [])
|
||||
require(isinstance(evidence_fields, list) and
|
||||
all(isinstance(field, str) and re.fullmatch(r"[a-z0-9][a-z0-9_]*", field) is not None
|
||||
for field in evidence_fields),
|
||||
f"invalid evidence fields for release gate {gate}")
|
||||
required_fields = set(SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS.get(gate, ()))
|
||||
missing_fields = sorted(required_fields - set(evidence_fields))
|
||||
require(not missing_fields,
|
||||
f"release gate {gate} missing required evidence fields: {', '.join(missing_fields)}")
|
||||
requirements[gate] = item
|
||||
|
||||
missing = sorted(set(SCANNER_HEAL_RELEASE_REQUIRED_GATES) - set(requirements))
|
||||
require(not missing, f"missing scanner/heal release requirements: {', '.join(missing)}")
|
||||
for lane, gates in lane_gates.items():
|
||||
unknown = sorted(gates - set(requirements))
|
||||
require(not unknown, f"pending release lane {lane} has unknown gates: {', '.join(unknown)}")
|
||||
mapped = {gate for gate, requirement in requirements.items() if requirement["lane"] == lane}
|
||||
require(gates == mapped, f"pending release lane {lane} gates do not match release requirements")
|
||||
pending_lanes = sorted(lane for lane, status in lane_statuses.items() if status == "pending")
|
||||
return requirements, True, pending_lanes
|
||||
|
||||
|
||||
def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_binary: Path) -> None:
|
||||
"""Record an existing build; this command never builds or runs a test."""
|
||||
require(not directory.exists(), "scanner/heal run directory must be new")
|
||||
@@ -969,7 +1124,7 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
"""Validate one actual case, or fail the release while required lanes are pending."""
|
||||
try:
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
evidence_integer(registry.get("schema"), "registry schema", 1, 1)
|
||||
scanner_heal_registry_schema(registry)
|
||||
require(registry.get("cases"), "invalid scanner/heal registry")
|
||||
selected = registry["cases"] if case_id == "release" else {case_id: registry["cases"][case_id]}
|
||||
run = read_json(directory / "run.json")
|
||||
@@ -1050,6 +1205,19 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
require(oracle.get("topology") == requirement["topology"], "oracle topology mismatch")
|
||||
for key in ("nodes", "drives_per_node"):
|
||||
evidence_integer(oracle["topology"][key], f"observed {key}", 1, 16)
|
||||
expected_erasure = requirement.get("erasure")
|
||||
if expected_erasure is not None:
|
||||
require(isinstance(expected_erasure, dict), "invalid erasure expectation")
|
||||
expected_data_blocks = evidence_integer(expected_erasure.get("data_blocks"), "expected EC data blocks", 1, 16)
|
||||
expected_parity_blocks = evidence_integer(expected_erasure.get("parity_blocks"), "expected EC parity blocks", 1, 16)
|
||||
require(
|
||||
expected_data_blocks + expected_parity_blocks
|
||||
== oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"],
|
||||
"expected EC geometry differs from topology",
|
||||
)
|
||||
else:
|
||||
expected_data_blocks = None
|
||||
expected_parity_blocks = None
|
||||
evidence_integer(oracle.get("pid_before"), "pid_before", 1, 2**32 - 1)
|
||||
evidence_integer(oracle.get("pid_after"), "pid_after", 1, 2**32 - 1)
|
||||
require(oracle["pid_before"] != oracle["pid_after"], "no process restart witnessed")
|
||||
@@ -1073,6 +1241,9 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16)
|
||||
require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"],
|
||||
"EC geometry differs from this case's single set")
|
||||
if expected_data_blocks is not None:
|
||||
require(data == expected_data_blocks and parity == expected_parity_blocks,
|
||||
"EC data/parity geometry differs from the required case")
|
||||
evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity)
|
||||
require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata")
|
||||
parts = physical["expected_part_numbers"]
|
||||
@@ -1088,7 +1259,10 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
require(all(keys == sorted(obj["key"] for obj in objects) for keys in node_listings),
|
||||
"S3 listing differs from object oracle")
|
||||
if case_id == "release":
|
||||
errors.extend(f"pending {gate}: {reason}" for gate, reason in registry["release_pending"].items())
|
||||
release_requirements, _, _ = scanner_heal_release_requirements(registry)
|
||||
errors.extend(
|
||||
f"pending {gate}: {requirement['description']}" for gate, requirement in release_requirements.items()
|
||||
)
|
||||
return errors
|
||||
except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error:
|
||||
return [f"scanner/heal evidence rejected: {error}"]
|
||||
@@ -1097,15 +1271,10 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
def scanner_heal_release_status(root: Path, directory: Path) -> dict[str, object]:
|
||||
"""Return a compact release decision without weakening case validation."""
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
evidence_integer(registry.get("schema"), "registry schema", 1, 1)
|
||||
scanner_heal_registry_schema(registry)
|
||||
cases = registry.get("cases")
|
||||
require(isinstance(cases, dict) and cases, "invalid scanner/heal registry")
|
||||
pending = registry.get("release_pending")
|
||||
require(isinstance(pending, dict), "invalid scanner/heal release requirements")
|
||||
for gate, reason in pending.items():
|
||||
require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None,
|
||||
"invalid scanner/heal release gate")
|
||||
require(isinstance(reason, str) and reason.strip(), f"missing release requirement for {gate}")
|
||||
release_requirements, release_schema_capable, pending_lanes = scanner_heal_release_requirements(registry)
|
||||
|
||||
verified_cases = []
|
||||
rejected_cases = []
|
||||
@@ -1119,10 +1288,160 @@ def scanner_heal_release_status(root: Path, directory: Path) -> dict[str, object
|
||||
"schema": 1,
|
||||
"decision": "blocked",
|
||||
"release_approved": False,
|
||||
"release_schema_capable": False,
|
||||
"release_schema_capable": release_schema_capable,
|
||||
"verified_cases": verified_cases,
|
||||
"rejected_cases": rejected_cases,
|
||||
"pending_gates": sorted(pending),
|
||||
"pending_gates": sorted(release_requirements),
|
||||
"pending_lanes": pending_lanes,
|
||||
}
|
||||
|
||||
|
||||
def release_bundle_artifact_path(bundle_path: Path, raw_path: object, gate: str, field: str) -> Path:
|
||||
require(isinstance(raw_path, str) and raw_path.strip(), f"{gate}.{field} missing artifact")
|
||||
path = Path(raw_path)
|
||||
require(not path.is_absolute() and ".." not in path.parts, f"{gate}.{field} artifact path escapes bundle directory")
|
||||
resolved = (bundle_path.parent / path).resolve()
|
||||
require(resolved.is_relative_to(bundle_path.parent.resolve()), f"{gate}.{field} artifact path escapes bundle directory")
|
||||
require(resolved.is_file(), f"{gate}.{field} artifact is missing")
|
||||
return resolved
|
||||
|
||||
|
||||
def validate_release_bundle_artifact(bundle_path: Path, source_revision: str, gate: str, field: str,
|
||||
evidence: dict[str, object]) -> str:
|
||||
require(evidence.get("evidence_type") == "measured", f"{gate}.{field} must be measured evidence")
|
||||
require(evidence.get("source_revision") == source_revision, f"{gate}.{field} source revision mismatch")
|
||||
run_id = evidence_string(evidence.get("run_id"), f"{gate}.{field}.run_id", r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}")
|
||||
window_id = evidence_string(evidence.get("measurement_window_id"), f"{gate}.{field}.measurement_window_id",
|
||||
r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}")
|
||||
require(window_id != run_id, f"{gate}.{field} must separate run and measurement-window identities")
|
||||
started = evidence_timestamp(evidence.get("started_at"), f"{gate}.{field}.started_at")
|
||||
finished = evidence_timestamp(evidence.get("finished_at"), f"{gate}.{field}.finished_at")
|
||||
require(started <= finished, f"{gate}.{field} evidence timestamps are inverted")
|
||||
command = evidence.get("command")
|
||||
require(isinstance(command, list) and command and
|
||||
all(isinstance(part, str) and part.strip() for part in command),
|
||||
f"{gate}.{field} missing command provenance")
|
||||
evidence_string(evidence.get("artifact_format"), f"{gate}.{field}.artifact_format",
|
||||
r"[A-Za-z0-9][A-Za-z0-9._+:-]{1,63}")
|
||||
artifact = release_bundle_artifact_path(bundle_path, evidence.get("artifact"), gate, field)
|
||||
require(sha(evidence.get("sha256")) and digest(artifact) == evidence["sha256"], f"{gate}.{field} artifact hash mismatch")
|
||||
summary = evidence.get("summary")
|
||||
require(isinstance(summary, str) and summary.strip(), f"{gate}.{field} missing human summary")
|
||||
if gate.startswith("P"):
|
||||
duration = evidence_integer(evidence.get("duration_seconds"), f"{gate}.{field}.duration_seconds", 1, 86400)
|
||||
require((finished - started).total_seconds() + 1 >= duration, f"{gate}.{field} duration exceeds run window")
|
||||
require(duration >= 900, f"{gate}.{field} requires at least 900 seconds")
|
||||
if gate == "P3" and field == "two_hour_pressure_measurement":
|
||||
require(duration >= 7200, f"{gate}.{field} requires at least two hours")
|
||||
elif "duration_seconds" in evidence:
|
||||
evidence_integer(evidence.get("duration_seconds"), f"{gate}.{field}.duration_seconds", 1, 86400)
|
||||
if gate in ("G03", "G09", "R-L"):
|
||||
versions = evidence.get("versions")
|
||||
require(isinstance(versions, list) and
|
||||
len({version for version in versions if isinstance(version, str) and version.strip()}) >= 2,
|
||||
f"{gate}.{field} requires mixed-version evidence")
|
||||
if gate in ("G04", "G07", "R-E", "R-L"):
|
||||
crash_points = evidence.get("crash_points")
|
||||
require(isinstance(crash_points, list) and crash_points,
|
||||
f"{gate}.{field} requires crash-boundary evidence")
|
||||
if gate == "G14":
|
||||
if field == "ec8_4_evidence":
|
||||
topology = evidence.get("topology")
|
||||
require(isinstance(topology, dict), "G14.ec8_4_evidence missing topology")
|
||||
require(topology.get("erasure") == "EC8+4", "G14.ec8_4_evidence must record EC8+4")
|
||||
require(evidence_integer(topology.get("nodes"), "G14 topology nodes", 3, 64) >= 3,
|
||||
"G14.ec8_4_evidence requires at least three nodes")
|
||||
require(evidence_integer(topology.get("drives_per_node"), "G14 topology drives", 4, 64) >= 4,
|
||||
"G14.ec8_4_evidence requires at least four drives per node")
|
||||
if field == "multi_set_evidence":
|
||||
evidence_integer(evidence.get("sets"), "G14 multi_set_evidence.sets", 2, 1024)
|
||||
if field == "multi_pool_evidence":
|
||||
evidence_integer(evidence.get("pools"), "G14 multi_pool_evidence.pools", 2, 1024)
|
||||
if field == "profile_evidence":
|
||||
evidence_integer(evidence.get("resolved_samples"), f"{gate}.{field}.resolved_samples", 1, 2**63 - 1)
|
||||
return window_id
|
||||
|
||||
|
||||
def validate_release_bundle_gate_windows(gate: str, field_windows: dict[str, str]) -> None:
|
||||
if gate == "G14" or gate.startswith("P"):
|
||||
windows = sorted(set(field_windows.values()))
|
||||
require(len(windows) == 1, f"{gate} evidence fields must share one measurement window")
|
||||
|
||||
|
||||
def scanner_heal_release_bundle_status(root: Path, bundle_path: Path) -> dict[str, object]:
|
||||
"""Validate a complete hard-gate evidence bundle without accepting synthetic claims."""
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
requirements, release_schema_capable, pending_lanes = scanner_heal_release_requirements(registry)
|
||||
require(release_schema_capable, "scanner/heal release bundle requires schema 2 registry")
|
||||
bundle_path = bundle_path.resolve()
|
||||
bundle = read_json(bundle_path)
|
||||
require(bundle.get("schema") == 1, "unsupported scanner/heal release evidence bundle schema")
|
||||
require(bundle.get("evidence") == "measured", "scanner/heal release evidence bundle must be measured")
|
||||
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip()
|
||||
source_revision = bundle.get("source_revision")
|
||||
require(isinstance(source_revision, str) and re.fullmatch(r"[0-9a-f]{40}", source_revision) is not None and
|
||||
source_revision == revision,
|
||||
"scanner/heal release evidence source revision mismatch")
|
||||
raw_gates = bundle.get("gates")
|
||||
require(isinstance(raw_gates, dict), "scanner/heal release evidence bundle missing gates")
|
||||
|
||||
verified: list[str] = []
|
||||
rejected: dict[str, list[str]] = {}
|
||||
for gate, requirement in requirements.items():
|
||||
gate_errors: list[str] = []
|
||||
gate_evidence = raw_gates.get(gate)
|
||||
if not isinstance(gate_evidence, dict):
|
||||
rejected[gate] = ["missing gate evidence"]
|
||||
continue
|
||||
if gate_evidence.get("status") != "pass":
|
||||
gate_errors.append("gate status must be pass")
|
||||
if gate_evidence.get("lane") != requirement["lane"]:
|
||||
gate_errors.append("gate lane mismatch")
|
||||
if gate_evidence.get("evidence_type") != "measured":
|
||||
gate_errors.append("gate evidence type must be measured")
|
||||
fields = gate_evidence.get("evidence_fields")
|
||||
if not isinstance(fields, dict):
|
||||
gate_errors.append("missing gate evidence fields")
|
||||
fields = {}
|
||||
required_fields = tuple(SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS[gate])
|
||||
missing_fields = [field for field in required_fields if field not in fields]
|
||||
if missing_fields:
|
||||
gate_errors.append(f"missing required fields: {', '.join(missing_fields)}")
|
||||
field_windows: dict[str, str] = {}
|
||||
for field in required_fields:
|
||||
if field not in fields:
|
||||
continue
|
||||
evidence = fields[field]
|
||||
if not isinstance(evidence, dict):
|
||||
gate_errors.append(f"{field} must be an object")
|
||||
continue
|
||||
try:
|
||||
field_windows[field] = validate_release_bundle_artifact(bundle_path, source_revision, gate, field, evidence)
|
||||
except (OSError, KeyError, TypeError, ValueError) as error:
|
||||
gate_errors.append(str(error))
|
||||
if not gate_errors:
|
||||
try:
|
||||
validate_release_bundle_gate_windows(gate, field_windows)
|
||||
except ValueError as error:
|
||||
gate_errors.append(str(error))
|
||||
if gate_errors:
|
||||
rejected[gate] = gate_errors
|
||||
else:
|
||||
verified.append(gate)
|
||||
|
||||
unknown = sorted(set(raw_gates) - set(requirements))
|
||||
if unknown:
|
||||
rejected["unknown"] = [f"unknown gates: {', '.join(unknown)}"]
|
||||
approved = not rejected and sorted(verified) == sorted(requirements)
|
||||
return {
|
||||
"schema": 1,
|
||||
"decision": "approved" if approved else "blocked",
|
||||
"release_approved": approved,
|
||||
"release_schema_capable": release_schema_capable,
|
||||
"verified_gates": sorted(verified),
|
||||
"rejected_gates": rejected,
|
||||
"pending_gates": [] if approved else sorted(gate for gate in requirements if gate not in verified),
|
||||
"pending_lanes": [] if approved else pending_lanes,
|
||||
}
|
||||
|
||||
|
||||
@@ -1310,16 +1629,31 @@ class SelfTests(unittest.TestCase):
|
||||
)
|
||||
+ "</testsuite></testsuites>"
|
||||
)
|
||||
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
|
||||
"erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1],
|
||||
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
|
||||
"inline_data_fingerprint": None}
|
||||
obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
|
||||
"expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
|
||||
"expected_physical": physical, "physical": physical}
|
||||
objects = [dict(obj, key=f"object-{index}") for index in range(9)]
|
||||
objects[-1] = dict(objects[-1], expected_physical=None)
|
||||
def oracle_objects(requirement: dict[str, object]) -> list[dict[str, object]]:
|
||||
topology = requirement["topology"]
|
||||
total_blocks = topology["nodes"] * topology["drives_per_node"]
|
||||
erasure = requirement.get("erasure")
|
||||
if erasure is None:
|
||||
parity_blocks = 4 if total_blocks == 12 else total_blocks // 2
|
||||
data_blocks = total_blocks - parity_blocks
|
||||
else:
|
||||
data_blocks = erasure["data_blocks"]
|
||||
parity_blocks = erasure["parity_blocks"]
|
||||
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
|
||||
"erasure_index": 1, "data_blocks": data_blocks, "parity_blocks": parity_blocks,
|
||||
"expected_part_numbers": [1],
|
||||
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
|
||||
"inline_data_fingerprint": None}
|
||||
obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
|
||||
"expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
|
||||
"expected_physical": physical, "physical": physical}
|
||||
count = requirement.get("min_objects", 9)
|
||||
objects = [dict(obj, key=f"object-{index}") for index in range(count)]
|
||||
objects[-1] = dict(objects[-1], expected_physical=None)
|
||||
return objects
|
||||
|
||||
for case_id, requirement in requirements.items():
|
||||
objects = oracle_objects(requirement)
|
||||
write_json(run_dir / requirement["oracle"], {
|
||||
"schema": 1, "evidence": requirement["evidence"], "case": case_id,
|
||||
"run_id": "a" * 32, "source_revision": "b" * 40,
|
||||
@@ -1328,11 +1662,163 @@ class SelfTests(unittest.TestCase):
|
||||
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
|
||||
"topology": requirement["topology"], "pid_before": 10, "pid_after": 11,
|
||||
"unclean_shutdown_marker": requirement["unclean_shutdown_marker"],
|
||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
|
||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * requirement["topology"]["nodes"],
|
||||
})
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
return root, run_dir
|
||||
|
||||
def scanner_heal_release_bundle_fixture(self, directory: Path) -> tuple[Path, Path]:
|
||||
"""Parser fixtures only; the bundle is not runtime evidence."""
|
||||
root, _ = self.scanner_heal_fixture(directory)
|
||||
bundle_dir = directory / "bundle"
|
||||
artifact_dir = bundle_dir / "artifacts"
|
||||
artifact_dir.mkdir(parents=True)
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
requirements, _, _ = scanner_heal_release_requirements(registry)
|
||||
gates = {}
|
||||
source_revision = "b" * 40
|
||||
started = datetime(2026, 9, 8, 0, 0, tzinfo=timezone.utc)
|
||||
for gate, requirement in requirements.items():
|
||||
fields = {}
|
||||
for field in SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS[gate]:
|
||||
artifact = artifact_dir / f"{gate}-{field}.json"
|
||||
write_json(artifact, {"gate": gate, "field": field, "fixture": True})
|
||||
duration = 60
|
||||
evidence = {
|
||||
"artifact": artifact.relative_to(bundle_dir).as_posix(),
|
||||
"sha256": digest(artifact),
|
||||
"evidence_type": "measured",
|
||||
"source_revision": source_revision,
|
||||
"run_id": f"{gate.lower()}-{field.replace('_', '-')}-run",
|
||||
"measurement_window_id": f"{gate.lower()}-window",
|
||||
"started_at": started.isoformat().replace("+00:00", "Z"),
|
||||
"command": ["cargo", "nextest", "run", requirement["description"]],
|
||||
"artifact_format": "json",
|
||||
"summary": f"parser fixture for {gate}.{field}",
|
||||
}
|
||||
if gate.startswith("P"):
|
||||
duration = 900
|
||||
evidence["duration_seconds"] = duration
|
||||
if gate == "P3" and field == "two_hour_pressure_measurement":
|
||||
duration = 7200
|
||||
evidence["duration_seconds"] = duration
|
||||
evidence["finished_at"] = (started + timedelta(seconds=duration)).isoformat().replace("+00:00", "Z")
|
||||
if gate in ("G03", "G09", "R-L"):
|
||||
evidence["versions"] = ["previous", "candidate"]
|
||||
if gate in ("G04", "G07", "R-E", "R-L"):
|
||||
evidence["crash_points"] = ["before-commit"]
|
||||
if gate == "G14" and field == "ec8_4_evidence":
|
||||
evidence["topology"] = {"erasure": "EC8+4", "nodes": 3, "drives_per_node": 4}
|
||||
if gate == "G14" and field == "multi_set_evidence":
|
||||
evidence["sets"] = 2
|
||||
if gate == "G14" and field == "multi_pool_evidence":
|
||||
evidence["pools"] = 2
|
||||
if field == "profile_evidence":
|
||||
evidence["resolved_samples"] = 1
|
||||
fields[field] = evidence
|
||||
gates[gate] = {
|
||||
"status": "pass",
|
||||
"lane": requirement["lane"],
|
||||
"evidence_type": "measured",
|
||||
"evidence_fields": fields,
|
||||
}
|
||||
bundle = bundle_dir / "release-evidence.json"
|
||||
write_json(bundle, {"schema": 1, "evidence": "measured", "source_revision": source_revision, "gates": gates})
|
||||
return root, bundle
|
||||
|
||||
def test_scanner_heal_release_bundle_accepts_complete_measured_evidence(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||
status = scanner_heal_release_bundle_status(root, bundle)
|
||||
self.assertEqual(status["decision"], "approved")
|
||||
self.assertTrue(status["release_approved"])
|
||||
self.assertEqual(len(status["verified_gates"]), len(SCANNER_HEAL_RELEASE_REQUIRED_GATES))
|
||||
self.assertEqual(status["pending_gates"], [])
|
||||
self.assertEqual(status["pending_lanes"], [])
|
||||
|
||||
def test_scanner_heal_release_bundle_rejects_synthetic_or_missing_fields(self) -> None:
|
||||
for fault in ("synthetic", "missing-field", "hash"):
|
||||
with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
|
||||
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||
data = read_json(bundle)
|
||||
if fault == "synthetic":
|
||||
data["evidence"] = "synthetic"
|
||||
elif fault == "missing-field":
|
||||
del data["gates"]["G09"]["evidence_fields"]["rollback_payload_evidence"]
|
||||
else:
|
||||
artifact = bundle.parent / data["gates"]["G01"]["evidence_fields"]["root_authority_evidence"]["artifact"]
|
||||
artifact.write_text(artifact.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
||||
write_json(bundle, data)
|
||||
|
||||
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||
if fault == "synthetic":
|
||||
with self.assertRaisesRegex(ValueError, "must be measured"):
|
||||
scanner_heal_release_bundle_status(root, bundle)
|
||||
else:
|
||||
status = scanner_heal_release_bundle_status(root, bundle)
|
||||
if fault != "synthetic":
|
||||
self.assertEqual(status["decision"], "blocked")
|
||||
self.assertFalse(status["release_approved"])
|
||||
self.assertTrue(status["rejected_gates"])
|
||||
|
||||
def test_scanner_heal_release_bundle_enforces_topology_duration_profile_and_versions(self) -> None:
|
||||
for fault, gate, field, mutation, expected in (
|
||||
("topology", "G14", "ec8_4_evidence", lambda item: item.update({"topology": {"erasure": "EC4+2", "nodes": 2, "drives_per_node": 3}}), "EC8+4"),
|
||||
("missing-duration", "P1", "cold_walk_share_measurement", lambda item: item.pop("duration_seconds"), "duration_seconds"),
|
||||
("duration", "P3", "two_hour_pressure_measurement", lambda item: item.update({"duration_seconds": 7199}), "two hours"),
|
||||
("profile", "P1", "profile_evidence", lambda item: item.pop("resolved_samples"), "resolved_samples"),
|
||||
("versions", "G09", "mixed_version_reader_evidence", lambda item: item.update({"versions": [1, 2]}), "mixed-version"),
|
||||
):
|
||||
with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
|
||||
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||
data = read_json(bundle)
|
||||
mutation(data["gates"][gate]["evidence_fields"][field])
|
||||
write_json(bundle, data)
|
||||
|
||||
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||
status = scanner_heal_release_bundle_status(root, bundle)
|
||||
self.assertEqual(status["decision"], "blocked")
|
||||
self.assertFalse(status["release_approved"])
|
||||
self.assertTrue(any(expected in error for error in status["rejected_gates"][gate]))
|
||||
|
||||
def test_scanner_heal_release_bundle_requires_field_provenance(self) -> None:
|
||||
for fault, mutation, expected in (
|
||||
("source", lambda item: item.update({"source_revision": "c" * 40}), "source revision mismatch"),
|
||||
("run-id", lambda item: item.pop("run_id"), "run_id"),
|
||||
("window-id", lambda item: item.update({"measurement_window_id": item["run_id"]}), "separate run"),
|
||||
("started-at", lambda item: item.update({"started_at": "not-a-time"}), "timestamp"),
|
||||
("finished-at", lambda item: item.update({"finished_at": "2026-09-07T00:00:00Z"}), "timestamps are inverted"),
|
||||
("command", lambda item: item.update({"command": []}), "command provenance"),
|
||||
("artifact-format", lambda item: item.pop("artifact_format"), "artifact_format"),
|
||||
):
|
||||
with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
|
||||
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||
data = read_json(bundle)
|
||||
mutation(data["gates"]["G01"]["evidence_fields"]["root_authority_evidence"])
|
||||
write_json(bundle, data)
|
||||
|
||||
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||
status = scanner_heal_release_bundle_status(root, bundle)
|
||||
self.assertEqual(status["decision"], "blocked")
|
||||
self.assertFalse(status["release_approved"])
|
||||
self.assertTrue(any(expected in error for error in status["rejected_gates"]["G01"]), fault)
|
||||
|
||||
def test_scanner_heal_release_bundle_requires_same_gate_measurement_window(self) -> None:
|
||||
for gate, field in (("G14", "multi_pool_evidence"), ("P1", "profile_evidence"), ("P3", "heal_capacity_measurement")):
|
||||
with self.subTest(gate=gate), tempfile.TemporaryDirectory() as tmp:
|
||||
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||
data = read_json(bundle)
|
||||
data["gates"][gate]["evidence_fields"][field]["measurement_window_id"] = f"{gate.lower()}-different-window"
|
||||
write_json(bundle, data)
|
||||
|
||||
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||
status = scanner_heal_release_bundle_status(root, bundle)
|
||||
self.assertEqual(status["decision"], "blocked")
|
||||
self.assertFalse(status["release_approved"])
|
||||
self.assertTrue(any("must share one measurement window" in error
|
||||
for error in status["rejected_gates"][gate]))
|
||||
|
||||
def test_scanner_heal_case_does_not_approve_pending_release(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
@@ -1346,14 +1832,21 @@ class SelfTests(unittest.TestCase):
|
||||
status = scanner_heal_release_status(root, run_dir)
|
||||
self.assertEqual(status["decision"], "blocked")
|
||||
self.assertFalse(status["release_approved"])
|
||||
self.assertTrue(status["release_schema_capable"])
|
||||
self.assertEqual(status["rejected_cases"], [])
|
||||
self.assertEqual(len(status["pending_gates"]), 21)
|
||||
self.assertIn("mixed-version-rollback", status["pending_lanes"])
|
||||
self.assertIn("ec8-4-multiset", status["pending_lanes"])
|
||||
self.assertIn("scheduler-pressure", status["pending_lanes"])
|
||||
|
||||
def test_scanner_heal_case_only_schema_cannot_approve_release(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
registry["schema"] = 1
|
||||
registry["release_pending"] = {}
|
||||
registry.pop("release_lanes")
|
||||
registry.pop("release_requirements")
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
|
||||
status = scanner_heal_release_status(root, run_dir)
|
||||
@@ -1363,11 +1856,99 @@ class SelfTests(unittest.TestCase):
|
||||
self.assertEqual(status["rejected_cases"], [])
|
||||
self.assertEqual(status["pending_gates"], [])
|
||||
|
||||
def test_scanner_heal_release_requirements_cannot_be_cleared(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
registry["release_requirements"] = []
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "invalid scanner/heal release requirements"):
|
||||
scanner_heal_release_status(root, run_dir)
|
||||
|
||||
def test_scanner_heal_release_matrix_lanes_remain_pending_without_real_evidence(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
|
||||
errors = check_scanner_heal_evidence(root, run_dir, "release")
|
||||
for gate, text in (
|
||||
("G03", "Exact scoped ACK"),
|
||||
("G09", "mixed-version reader/writer"),
|
||||
("G14", "3x4 EC8+4"),
|
||||
("P3", "two-hour pressure/heal capacity"),
|
||||
("R-L", "Legacy source conflicts"),
|
||||
):
|
||||
self.assertTrue(any(error.startswith(f"pending {gate}:") and text in error for error in errors), gate)
|
||||
|
||||
status = scanner_heal_release_status(root, run_dir)
|
||||
self.assertFalse(status["release_approved"])
|
||||
self.assertIn("mixed-version-rollback", status["pending_lanes"])
|
||||
self.assertIn("ec8-4-multiset", status["pending_lanes"])
|
||||
self.assertIn("scheduler-pressure", status["pending_lanes"])
|
||||
requirements, _, _ = scanner_heal_release_requirements(read_json(root / ".config/scanner-heal-required-tests.json"))
|
||||
self.assertIn("durable_root_publication_proof", requirements["G03"]["evidence_fields"])
|
||||
self.assertIn("mixed_version_writer_evidence", requirements["G09"]["evidence_fields"])
|
||||
|
||||
def test_scanner_heal_required_evidence_fields_cannot_be_removed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
for gate in ("G03", "G09"):
|
||||
for requirement in registry["release_requirements"]:
|
||||
if requirement["gate"] == gate:
|
||||
requirement["evidence_fields"] = []
|
||||
break
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
|
||||
with self.subTest(gate=gate):
|
||||
with self.assertRaisesRegex(ValueError, f"release gate {gate} missing required evidence fields"):
|
||||
scanner_heal_release_status(root, run_dir)
|
||||
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
for requirement in registry["release_requirements"]:
|
||||
if requirement["gate"] == gate:
|
||||
requirement["evidence_fields"] = list(SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS[gate])
|
||||
break
|
||||
|
||||
def test_scanner_heal_ec84_case_rejects_wrong_erasure_geometry(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4"), [])
|
||||
|
||||
path = run_dir / "background-target-crash-ec8-4.json"
|
||||
oracle = read_json(path)
|
||||
oracle["objects"][0]["physical"]["data_blocks"] = 10
|
||||
oracle["objects"][0]["physical"]["parity_blocks"] = 2
|
||||
oracle["objects"][0]["expected_physical"]["data_blocks"] = 10
|
||||
oracle["objects"][0]["expected_physical"]["parity_blocks"] = 2
|
||||
write_json(path, oracle)
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
|
||||
errors = check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4")
|
||||
self.assertTrue(
|
||||
any("EC data/parity geometry differs from the required case" in error for error in errors),
|
||||
errors,
|
||||
)
|
||||
|
||||
def test_scanner_heal_pending_gate_cannot_map_to_implemented_lane(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
registry["release_requirements"][0]["lane"] = "single-set-restart"
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "mapped to non-pending lane"):
|
||||
scanner_heal_release_status(root, run_dir)
|
||||
|
||||
def test_scanner_heal_release_status_rejects_synthetic_case(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
registry["schema"] = 1
|
||||
registry["release_pending"] = {}
|
||||
registry.pop("release_lanes")
|
||||
registry.pop("release_requirements")
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
path = run_dir / "background-target-crash.json"
|
||||
oracle = read_json(path)
|
||||
@@ -1379,6 +1960,7 @@ class SelfTests(unittest.TestCase):
|
||||
status = scanner_heal_release_status(root, run_dir)
|
||||
self.assertEqual(status["decision"], "blocked")
|
||||
self.assertFalse(status["release_approved"])
|
||||
self.assertFalse(status["release_schema_capable"])
|
||||
self.assertEqual(status["rejected_cases"], ["background-target-crash"])
|
||||
self.assertEqual(status["pending_gates"], [])
|
||||
|
||||
@@ -1386,9 +1968,14 @@ class SelfTests(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
registry["schema"] = 1
|
||||
registry["release_pending"] = {}
|
||||
registry.pop("release_lanes")
|
||||
registry.pop("release_requirements")
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
(run_dir / "background-target-crash.json").unlink()
|
||||
for case_id, requirement in registry["cases"].items():
|
||||
if case_id != "background-target-restart":
|
||||
(run_dir / requirement["oracle"]).unlink()
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
|
||||
@@ -1396,7 +1983,7 @@ class SelfTests(unittest.TestCase):
|
||||
self.assertEqual(status["decision"], "blocked")
|
||||
self.assertFalse(status["release_approved"])
|
||||
self.assertEqual(status["verified_cases"], ["background-target-restart"])
|
||||
self.assertEqual(status["rejected_cases"], ["background-target-crash"])
|
||||
self.assertEqual(status["rejected_cases"], sorted(set(registry["cases"]) - {"background-target-restart"}))
|
||||
|
||||
def test_scanner_heal_finish_collects_oracles_from_registry(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
@@ -2246,7 +2833,7 @@ def main() -> int:
|
||||
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
|
||||
return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
|
||||
if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"],
|
||||
["--check-scanner-heal-release"]):
|
||||
["--check-scanner-heal-release"], ["--check-scanner-heal-release-bundle"]):
|
||||
try:
|
||||
if len(sys.argv) == 5 and sys.argv[1] == "--begin-scanner-heal":
|
||||
begin_scanner_heal_receipt(ROOT, Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4]))
|
||||
@@ -2270,7 +2857,16 @@ def main() -> int:
|
||||
return 2
|
||||
print(json.dumps(status, sort_keys=True, separators=(",", ":")))
|
||||
return 0 if status["release_approved"] else 1
|
||||
raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, --check-scanner-heal DIR CASE|release, or --check-scanner-heal-release DIR")
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--check-scanner-heal-release-bundle":
|
||||
try:
|
||||
status = scanner_heal_release_bundle_status(ROOT, Path(sys.argv[2]))
|
||||
except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error:
|
||||
print(json.dumps({"schema": 1, "decision": "invalid", "release_approved": False,
|
||||
"error": str(error)}, sort_keys=True, separators=(",", ":")))
|
||||
return 2
|
||||
print(json.dumps(status, sort_keys=True, separators=(",", ":")))
|
||||
return 0 if status["release_approved"] else 1
|
||||
raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, --check-scanner-heal DIR CASE|release, --check-scanner-heal-release DIR, or --check-scanner-heal-release-bundle FILE")
|
||||
except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
@@ -2297,7 +2893,7 @@ def main() -> int:
|
||||
if sys.argv[1:]:
|
||||
print(
|
||||
"usage: check_test_wiring.py [--self-test | --check-core LISTING | --check-profile PROFILE LISTING | "
|
||||
"--update-profile PROFILE LISTING PLATFORM]",
|
||||
"--update-profile PROFILE LISTING PLATFORM | --check-scanner-heal-release-bundle FILE]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
@@ -103,8 +103,8 @@ import sys
|
||||
status = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||
if status.get("decision") != "blocked" or status.get("release_approved") is not False:
|
||||
raise SystemExit("release status did not record a blocked decision")
|
||||
if status.get("release_schema_capable") is not False:
|
||||
raise SystemExit("case-only evidence schema unexpectedly became release-capable")
|
||||
if not status.get("pending_gates"):
|
||||
raise SystemExit("release status did not retain pending gates")
|
||||
PY
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,16 @@ METRICS = (
|
||||
)
|
||||
REPEATABILITY_LIMIT = Decimal("0.05")
|
||||
P2_WORK_MULTIPLE_LIMIT = Decimal("1.2")
|
||||
RELEASE_PROFILE_ARTIFACTS = (
|
||||
"allocation-profile",
|
||||
"flamegraph",
|
||||
"rss-samples",
|
||||
"save-frequency",
|
||||
)
|
||||
RELEASE_FAULT_MODES = (
|
||||
"process-restart",
|
||||
"process-crash-restart",
|
||||
)
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
@@ -140,6 +150,101 @@ def validate_manifest(manifest):
|
||||
number(manifest["expected_healed_objects"].get(scenario), f"{scenario} expected repairs")
|
||||
if scenario in ("running-heal", "mrf-replay"):
|
||||
require(manifest["expected_healed_objects"][scenario] > 0, f"{scenario} requires repairs")
|
||||
validate_release_evidence_manifest(manifest)
|
||||
|
||||
|
||||
def release_evidence_integer(value, name, minimum=1, maximum=1024):
|
||||
require(type(value) is int and minimum <= value <= maximum, f"invalid release_evidence.{name}")
|
||||
return value
|
||||
|
||||
|
||||
def release_evidence_string(value, name):
|
||||
require(isinstance(value, str) and value.strip(), f"missing release_evidence.{name}")
|
||||
return value
|
||||
|
||||
|
||||
def release_evidence_bool(value, name):
|
||||
require(type(value) is bool, f"invalid release_evidence.{name}")
|
||||
return value
|
||||
|
||||
|
||||
def release_evidence_true(value, name):
|
||||
release_evidence_bool(value, name)
|
||||
require(value is True, f"missing release_evidence.{name}")
|
||||
|
||||
|
||||
def validate_release_evidence_manifest(manifest):
|
||||
if manifest["evidence"] != "measured":
|
||||
return
|
||||
|
||||
evidence = manifest.get("release_evidence")
|
||||
require(isinstance(evidence, dict), "missing release_evidence for measured ABBA")
|
||||
|
||||
topology = evidence.get("topology")
|
||||
require(isinstance(topology, dict), "missing release_evidence.topology")
|
||||
nodes = release_evidence_integer(topology.get("nodes"), "topology.nodes", 3, 64)
|
||||
drives = release_evidence_integer(topology.get("drives_per_node"), "topology.drives_per_node", 1, 64)
|
||||
set_size = release_evidence_integer(topology.get("erasure_set_size"), "topology.erasure_set_size", 12, 12)
|
||||
data = release_evidence_integer(topology.get("erasure_data_blocks"), "topology.erasure_data_blocks", 8, 8)
|
||||
parity = release_evidence_integer(topology.get("erasure_parity_blocks"), "topology.erasure_parity_blocks", 4, 4)
|
||||
require(data + parity == set_size, "release_evidence.topology must be EC8+4")
|
||||
require(nodes * drives >= set_size, "release_evidence.topology cannot host one EC8+4 set")
|
||||
pools = release_evidence_integer(topology.get("pools"), "topology.pools", 1)
|
||||
sets_total = release_evidence_integer(topology.get("sets_total"), "topology.sets_total", 1)
|
||||
sampled_pools = release_evidence_integer(topology.get("sampled_pools"), "topology.sampled_pools", 2)
|
||||
sampled_sets = release_evidence_integer(topology.get("sampled_sets"), "topology.sampled_sets", 2)
|
||||
require(sampled_pools <= pools, "release_evidence.topology sampled pools exceed total pools")
|
||||
require(sampled_sets <= sets_total, "release_evidence.topology sampled sets exceed total sets")
|
||||
|
||||
distributed = evidence.get("distributed")
|
||||
require(isinstance(distributed, dict), "missing release_evidence.distributed")
|
||||
endpoints = distributed.get("metrics_endpoints")
|
||||
require(isinstance(endpoints, list) and len(endpoints) >= nodes, "missing release_evidence.distributed.metrics_endpoints")
|
||||
require(
|
||||
all(isinstance(endpoint, str) and endpoint.strip() for endpoint in endpoints)
|
||||
and len(set(endpoints)) == len(endpoints),
|
||||
"invalid release_evidence.distributed.metrics_endpoints",
|
||||
)
|
||||
release_evidence_string(distributed.get("failure_domain"), "distributed.failure_domain")
|
||||
release_evidence_true(distributed.get("same_window_sampling"), "distributed.same_window_sampling")
|
||||
|
||||
crash = evidence.get("crash_restart")
|
||||
require(isinstance(crash, dict), "missing release_evidence.crash_restart")
|
||||
fault_modes = crash.get("fault_modes")
|
||||
require(
|
||||
isinstance(fault_modes, list)
|
||||
and all(mode in fault_modes for mode in RELEASE_FAULT_MODES)
|
||||
and all(isinstance(mode, str) and mode.strip() for mode in fault_modes),
|
||||
"missing release_evidence.crash_restart.fault_modes",
|
||||
)
|
||||
release_evidence_true(crash.get("unclean_shutdown_marker"), "crash_restart.unclean_shutdown_marker")
|
||||
|
||||
mixed = evidence.get("mixed_version")
|
||||
require(isinstance(mixed, dict), "missing release_evidence.mixed_version")
|
||||
revisions = mixed.get("participating_revisions")
|
||||
require(
|
||||
isinstance(revisions, list)
|
||||
and len(set(revisions)) >= 2
|
||||
and all(isinstance(revision, str) and len(revision) == 40 and all(c in "0123456789abcdef" for c in revision)
|
||||
for revision in revisions),
|
||||
"invalid release_evidence.mixed_version.participating_revisions",
|
||||
)
|
||||
for revision in (manifest["baseline"]["revision"], manifest["candidate"]["revision"]):
|
||||
require(revision in revisions, "release_evidence.mixed_version omits tested build revision")
|
||||
for key in ("reader", "writer", "rollback_payload"):
|
||||
require(mixed.get(key) is True, f"missing release_evidence.mixed_version.{key}")
|
||||
|
||||
profile = evidence.get("profile")
|
||||
require(isinstance(profile, dict), "missing release_evidence.profile")
|
||||
artifacts = profile.get("required_artifacts")
|
||||
require(
|
||||
isinstance(artifacts, list)
|
||||
and all(item in artifacts for item in RELEASE_PROFILE_ARTIFACTS)
|
||||
and all(isinstance(item, str) and item.strip() for item in artifacts),
|
||||
"missing release_evidence.profile.required_artifacts",
|
||||
)
|
||||
for key in ("collector_config_sha256", "profiler_config_sha256"):
|
||||
require(sha(profile.get(key)), f"invalid release_evidence.profile.{key}")
|
||||
|
||||
|
||||
class OwnedCommand:
|
||||
@@ -254,6 +359,8 @@ def validate_result(result, request, expected):
|
||||
require(result.get("build") == request["build"], "deployed build provenance mismatch")
|
||||
require(result.get("data_dir") == request["data_dir"], "adapter data isolation mismatch")
|
||||
require(result.get("background") == request["background"], "background mode mismatch")
|
||||
if request["evidence"] == "measured":
|
||||
require(result.get("release_evidence") == request["release_evidence"], "release evidence provenance mismatch")
|
||||
require(type(result.get("sample_count")) is int and 1 <= result["sample_count"] <= 3600,
|
||||
"sample_count must be 1..3600")
|
||||
number(result.get("elapsed_seconds"), "elapsed_seconds", request["duration_seconds"])
|
||||
@@ -445,6 +552,9 @@ def collect_live(prepared, request, request_path, adapter):
|
||||
connection = prepared["collector"]
|
||||
require(set(connection) == {"alias", "endpoint", "metrics_endpoints"}, "invalid collector connection")
|
||||
require(all(isinstance(value, str) and value for value in connection.values()), "missing collector endpoint")
|
||||
expected_metrics_endpoints = None
|
||||
if request.get("evidence") == "measured":
|
||||
expected_metrics_endpoints = request["release_evidence"]["distributed"]["metrics_endpoints"]
|
||||
output = request_path.parent / "telemetry"
|
||||
args = ["bash", str(collector), "--alias", connection["alias"], "--endpoint", connection["endpoint"],
|
||||
"--metrics-endpoints", connection["metrics_endpoints"], "--deployment", "distributed",
|
||||
@@ -470,6 +580,9 @@ def collect_live(prepared, request, request_path, adapter):
|
||||
require(isinstance(status.get("healOperations"), dict) and status["healOperations"], "invalid heal status response")
|
||||
metrics = list((output / "metrics").glob("admin-metrics.*.ndjson"))
|
||||
endpoints = [endpoint for endpoint in connection["metrics_endpoints"].split(",") if endpoint]
|
||||
if expected_metrics_endpoints is not None:
|
||||
require(endpoints == expected_metrics_endpoints,
|
||||
"collector metrics endpoints do not match release evidence")
|
||||
require(metrics and len(metrics) == len(endpoints) * len(samples), "missing distributed metrics samples")
|
||||
for sample in metrics:
|
||||
# The collector requests n=1, so each file contains one final JSON record.
|
||||
@@ -518,6 +631,8 @@ def run(manifest, adapter, output, data_root):
|
||||
"duration_seconds": manifest["duration_seconds"], "data_dir": str(data_dir),
|
||||
"expected_healed_objects": manifest["expected_healed_objects"][scenario],
|
||||
"expected_oracle": manifest["oracles"][scenario]}
|
||||
if manifest["evidence"] == "measured":
|
||||
request["release_evidence"] = manifest["release_evidence"]
|
||||
require(digest(Path(request["build"]["binary"])) == request["build"]["sha256"], "binary changed during run")
|
||||
require(digest(adapter) == manifest["adapter_sha256"], "adapter changed during run")
|
||||
require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space")
|
||||
|
||||
@@ -12,6 +12,8 @@ from pathlib import Path
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from scanner_abba import LEGS, SCENARIOS, validate_release_evidence_manifest
|
||||
|
||||
MAX_JSON_BYTES = 1024 * 1024
|
||||
CACHE_COST_PREFIX = "CACHE_COST "
|
||||
PASS_STATES = {"pass"}
|
||||
@@ -77,12 +79,89 @@ def max_decimal(values: list[Decimal | None]) -> Decimal | None:
|
||||
return max(present)
|
||||
|
||||
|
||||
def require_metric_series(value: Any, name: str, minimum: Decimal | None = None,
|
||||
maximum: Decimal | None = None) -> list[Decimal | None]:
|
||||
require(isinstance(value, list) and value, f"missing performance evidence field: {name}")
|
||||
parsed = [maybe_number(item, name) for item in value]
|
||||
for item in parsed:
|
||||
if item is None:
|
||||
continue
|
||||
if minimum is not None:
|
||||
require(item >= minimum, f"{name} below minimum")
|
||||
if maximum is not None:
|
||||
require(item <= maximum, f"{name} above maximum")
|
||||
return parsed
|
||||
|
||||
|
||||
def require_measured_comparison_evidence(comparison: dict[str, Any], index: int) -> None:
|
||||
w10_w11 = comparison.get("w10_w11")
|
||||
require(isinstance(w10_w11, dict), f"comparison {index} missing W10/W11 evidence")
|
||||
pressure = require_metric_series(
|
||||
w10_w11.get("foreground_pressure_high_sample_ratios"),
|
||||
f"comparison {index} foreground_pressure_high_sample_ratios",
|
||||
Decimal("0"),
|
||||
Decimal("1"),
|
||||
)
|
||||
lock_wait = require_metric_series(
|
||||
w10_w11.get("heal_lock_wait_p99_ms"),
|
||||
f"comparison {index} heal_lock_wait_p99_ms",
|
||||
Decimal("0"),
|
||||
)
|
||||
attempt_cost = require_metric_series(
|
||||
w10_w11.get("attempt_cost_per_healed_object"),
|
||||
f"comparison {index} attempt_cost_per_healed_object",
|
||||
Decimal("0"),
|
||||
)
|
||||
require(len(pressure) == len(lock_wait) == len(attempt_cost),
|
||||
f"comparison {index} W10/W11 evidence length mismatch")
|
||||
candidate_attempt_cost = maybe_number(
|
||||
w10_w11.get("candidate_attempt_cost_per_healed_object"),
|
||||
f"comparison {index} candidate_attempt_cost_per_healed_object",
|
||||
)
|
||||
require(candidate_attempt_cost is None or candidate_attempt_cost >= 0,
|
||||
f"comparison {index} candidate attempt cost below minimum")
|
||||
|
||||
|
||||
def require_complete_abba_matrix(manifest: dict[str, Any], report: dict[str, Any], comparisons: list[dict[str, Any]]) -> None:
|
||||
require(report.get("evidence") == manifest.get("evidence"), "manifest/report evidence mismatch")
|
||||
rounds = manifest.get("rounds")
|
||||
require(type(rounds) is int and 3 <= rounds <= 10, "invalid manifest.rounds")
|
||||
expected_cells = len(SCENARIOS) * 2 * rounds * len(LEGS)
|
||||
require(
|
||||
report.get("cells") == expected_cells,
|
||||
f"ABBA matrix cell count mismatch: expected {expected_cells}, got {report.get('cells')}",
|
||||
)
|
||||
expected_keys = {
|
||||
(scenario, comparison, round_id)
|
||||
for scenario in SCENARIOS
|
||||
for comparison in ("build", "background")
|
||||
for round_id in range(1, rounds + 1)
|
||||
}
|
||||
observed_keys = []
|
||||
for index, comparison in enumerate(comparisons):
|
||||
key = (comparison.get("scenario"), comparison.get("comparison"), comparison.get("round"))
|
||||
require(key in expected_keys, f"comparison {index} is outside the ABBA matrix")
|
||||
require(comparison.get("status") in PASS_STATES, f"comparison {index} did not pass")
|
||||
observed_keys.append(key)
|
||||
observed_set = set(observed_keys)
|
||||
require(len(observed_keys) == len(observed_set), "duplicate ABBA matrix comparison")
|
||||
missing = sorted(expected_keys - observed_set)
|
||||
require(not missing, f"missing ABBA matrix comparison: {missing[0] if missing else ''}")
|
||||
|
||||
|
||||
def summarize_abba(abba_dir: Path) -> dict[str, Any]:
|
||||
manifest_path = abba_dir / "manifest.json"
|
||||
report_path = abba_dir / "report.json"
|
||||
manifest = read_json(manifest_path)
|
||||
report = read_json(report_path)
|
||||
report_state = report.get("status")
|
||||
performance_state = report.get("performance")
|
||||
require(isinstance(report_state, str) and report_state, "report.status missing")
|
||||
require(isinstance(performance_state, str) and performance_state, "report.performance missing")
|
||||
comparisons = report.get("comparisons")
|
||||
if comparisons is None:
|
||||
require(report_state not in PASS_STATES, "passing report requires comparisons")
|
||||
comparisons = []
|
||||
require(isinstance(comparisons, list), "report.comparisons must be a list")
|
||||
|
||||
counts = Counter()
|
||||
@@ -115,15 +194,18 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]:
|
||||
if isinstance(p2, list):
|
||||
p2_values.extend(maybe_number(value, "p2_post_stop_work_multiple") for value in p2)
|
||||
|
||||
report_state = report.get("status")
|
||||
performance_state = report.get("performance")
|
||||
require(isinstance(report_state, str) and report_state, "report.status missing")
|
||||
require(isinstance(performance_state, str) and performance_state, "report.performance missing")
|
||||
measured = report.get("evidence") == "measured"
|
||||
passed = report_state in PASS_STATES and performance_state in PASS_STATES and measured
|
||||
if passed:
|
||||
require_complete_abba_matrix(manifest, report, comparisons)
|
||||
validate_release_evidence_manifest({**manifest, "evidence": "measured"})
|
||||
for index, comparison in enumerate(comparisons):
|
||||
require_measured_comparison_evidence(comparison, index)
|
||||
gate_state = "pass" if passed else "fail"
|
||||
if report_state == "synthetic_validated":
|
||||
reason = "synthetic evidence validates the harness only; measured performance remains pending"
|
||||
elif report_state in FAIL_STATES and isinstance(report.get("error"), str) and report["error"]:
|
||||
reason = f"ABBA report status is {report_state}: {report['error']}"
|
||||
elif report_state not in PASS_STATES:
|
||||
reason = f"ABBA report status is {report_state}"
|
||||
elif performance_state not in PASS_STATES:
|
||||
@@ -142,6 +224,8 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]:
|
||||
"performance": performance_state,
|
||||
"evidence": report.get("evidence"),
|
||||
"cells": report.get("cells", 0),
|
||||
"completed_cells": report.get("completed_cells"),
|
||||
"error": report.get("error"),
|
||||
"comparisons_total": len(comparisons),
|
||||
"comparison_status_counts": dict(sorted(counts.items())),
|
||||
"worst_p99_regression": None if not p99_regressions else float(max(p99_regressions)),
|
||||
@@ -164,6 +248,7 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]:
|
||||
"durability": fixed.get("durability"),
|
||||
"topology": fixed.get("topology"),
|
||||
"offered_load_ops": fixed.get("offered_load_ops"),
|
||||
"release_evidence": manifest.get("release_evidence"),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -247,6 +332,10 @@ def markdown(summary: dict[str, Any]) -> str:
|
||||
f"- worst_throughput_loss: {pct(throughput)}",
|
||||
f"- p2_worst_post_stop_work_multiple: {ratio(p2)}",
|
||||
]
|
||||
if abba.get("completed_cells") is not None:
|
||||
lines.append(f"- completed_cells: {abba['completed_cells']}")
|
||||
if abba.get("error"):
|
||||
lines.append(f"- error: {abba['error']}")
|
||||
if summary.get("cache_cost") is not None:
|
||||
cache = summary["cache_cost"]
|
||||
lines.extend([
|
||||
|
||||
@@ -49,6 +49,8 @@ def fake_adapter():
|
||||
if fault == "measure-exit":
|
||||
return 42
|
||||
result = {key: request[key] for key in ("evidence", "fixed", "build", "data_dir", "background")}
|
||||
if request["evidence"] == "measured":
|
||||
result["release_evidence"] = copy.deepcopy(request["release_evidence"])
|
||||
result.update({"sample_count": 10, "elapsed_seconds": request["duration_seconds"],
|
||||
"metrics": dict.fromkeys(harness.METRICS, 10)})
|
||||
baseline = request["comparison"] == "build" and request["leg"].startswith("A")
|
||||
@@ -163,6 +165,45 @@ class ScannerAbbaTest(unittest.TestCase):
|
||||
build = {"binary": str(self.binary), "sha256": harness.digest(self.binary), "revision": "a" * 40}
|
||||
self.manifest.update(baseline=build.copy(), candidate=build.copy())
|
||||
|
||||
def measured_manifest(self):
|
||||
manifest = copy.deepcopy(self.manifest)
|
||||
manifest.update(evidence="measured", duration_seconds=900)
|
||||
manifest["candidate"]["revision"] = "b" * 40
|
||||
manifest["release_evidence"] = {
|
||||
"topology": {
|
||||
"nodes": 3,
|
||||
"drives_per_node": 4,
|
||||
"pools": 2,
|
||||
"sets_total": 2,
|
||||
"sampled_pools": 2,
|
||||
"sampled_sets": 2,
|
||||
"erasure_set_size": 12,
|
||||
"erasure_data_blocks": 8,
|
||||
"erasure_parity_blocks": 4,
|
||||
},
|
||||
"distributed": {
|
||||
"metrics_endpoints": ["https://node-1:9000", "https://node-2:9000", "https://node-3:9000"],
|
||||
"failure_domain": "three-node-localhost-lab",
|
||||
"same_window_sampling": True,
|
||||
},
|
||||
"crash_restart": {
|
||||
"fault_modes": ["process-restart", "process-crash-restart"],
|
||||
"unclean_shutdown_marker": True,
|
||||
},
|
||||
"mixed_version": {
|
||||
"participating_revisions": ["a" * 40, "b" * 40],
|
||||
"reader": True,
|
||||
"writer": True,
|
||||
"rollback_payload": True,
|
||||
},
|
||||
"profile": {
|
||||
"required_artifacts": ["allocation-profile", "flamegraph", "rss-samples", "save-frequency"],
|
||||
"collector_config_sha256": "4" * 64,
|
||||
"profiler_config_sha256": "5" * 64,
|
||||
},
|
||||
}
|
||||
return manifest
|
||||
|
||||
def run_harness(self, fault=""):
|
||||
with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": fault}), contextlib.redirect_stdout(io.StringIO()):
|
||||
return harness.run(copy.deepcopy(self.manifest), self.adapter, self.root / "out", self.root / "data")
|
||||
@@ -440,6 +481,34 @@ class ScannerAbbaTest(unittest.TestCase):
|
||||
|
||||
process.finish.assert_called_once_with(terminate=True)
|
||||
|
||||
def test_live_collector_binds_release_evidence_metrics_endpoints(self):
|
||||
telemetry = self.root / "telemetry"
|
||||
for name in ("status", "heal", "metrics"):
|
||||
(telemetry / name).mkdir(parents=True)
|
||||
(telemetry / "scanner-summary.csv").write_text("timestamp\n")
|
||||
for index in range(16):
|
||||
harness.write_json(telemetry / f"status/scanner-status.{index}.json", {"metrics": {"objects": 10}})
|
||||
for node in ("node-a", "node-b"):
|
||||
harness.write_json(telemetry / f"heal/background-heal-status.{node}.{index}.json",
|
||||
{"healOperations": {"queueLength": 0}})
|
||||
harness.write_json(telemetry / f"metrics/admin-metrics.{node}.{index}.ndjson",
|
||||
{"errors": [], "final": True,
|
||||
"by_host": {f"{node}:9000": {"scanner": {"objects": 10}}}})
|
||||
prepared = {"collector": {"alias": "test", "endpoint": "http://node-a:9000",
|
||||
"metrics_endpoints": "http://node-a:9000,http://node-b:9000"}}
|
||||
request = {
|
||||
"duration_seconds": 900,
|
||||
"evidence": "measured",
|
||||
"release_evidence": self.measured_manifest()["release_evidence"],
|
||||
}
|
||||
process = Mock(pid=123, wait=Mock(return_value=0))
|
||||
with patch.object(harness, "OwnedCommand", return_value=process), \
|
||||
patch.object(harness, "invoke", return_value={"sample_count": 10}), \
|
||||
patch.object(harness.time, "monotonic", side_effect=(0, 900)):
|
||||
with self.assertRaisesRegex(ValueError, "collector metrics endpoints"):
|
||||
harness.collect_live(prepared, request, self.root / "request.json", self.adapter)
|
||||
process.finish.assert_called_once_with(terminate=True)
|
||||
|
||||
def test_unstable_p1_work_control_is_inconclusive(self):
|
||||
with patch.object(harness, "SCENARIOS", ("cold-hot",)):
|
||||
self.assertEqual(self.run_harness("unstable-p1-control"), 3)
|
||||
@@ -463,6 +532,81 @@ class ScannerAbbaTest(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "rounds"):
|
||||
harness.validate_manifest(self.manifest)
|
||||
|
||||
def test_measured_manifest_requires_release_evidence_contract(self):
|
||||
harness.validate_manifest(self.measured_manifest())
|
||||
faults = {
|
||||
"missing root": lambda manifest: manifest.pop("release_evidence"),
|
||||
"single-set": lambda manifest: manifest["release_evidence"]["topology"].update(sets_total=1),
|
||||
"unsampled-set": lambda manifest: manifest["release_evidence"]["topology"].update(sampled_sets=1),
|
||||
"wrong geometry": lambda manifest: manifest["release_evidence"]["topology"].update(erasure_set_size=11),
|
||||
"duplicate endpoint": lambda manifest: manifest["release_evidence"]["distributed"].update(
|
||||
metrics_endpoints=["https://node-1:9000", "https://node-1:9000", "https://node-3:9000"],
|
||||
),
|
||||
"split sampling": lambda manifest: manifest["release_evidence"]["distributed"].update(
|
||||
same_window_sampling=False,
|
||||
),
|
||||
"missing crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update(
|
||||
fault_modes=["process-restart"],
|
||||
),
|
||||
"clean crash marker": lambda manifest: manifest["release_evidence"]["crash_restart"].update(
|
||||
unclean_shutdown_marker=False,
|
||||
),
|
||||
"mixed version false": lambda manifest: manifest["release_evidence"]["mixed_version"].update(writer=False),
|
||||
"missing candidate": lambda manifest: manifest["release_evidence"]["mixed_version"].update(
|
||||
participating_revisions=["a" * 40, "c" * 40],
|
||||
),
|
||||
"missing profile": lambda manifest: manifest["release_evidence"]["profile"].update(
|
||||
required_artifacts=["allocation-profile", "flamegraph", "rss-samples"],
|
||||
),
|
||||
"bad profile hash": lambda manifest: manifest["release_evidence"]["profile"].update(
|
||||
profiler_config_sha256="not-a-sha",
|
||||
),
|
||||
}
|
||||
for name, mutate in faults.items():
|
||||
with self.subTest(fault=name):
|
||||
manifest = self.measured_manifest()
|
||||
mutate(manifest)
|
||||
with self.assertRaisesRegex(ValueError, "release_evidence"):
|
||||
harness.validate_manifest(manifest)
|
||||
|
||||
def test_measured_result_must_echo_release_evidence(self):
|
||||
manifest = self.measured_manifest()
|
||||
request = {
|
||||
"schema": 1,
|
||||
"scenario": "cold-hot",
|
||||
"comparison": "build",
|
||||
"round": 1,
|
||||
"leg": "B1",
|
||||
"background": "on",
|
||||
"build": manifest["candidate"],
|
||||
"evidence": manifest["evidence"],
|
||||
"fixed": manifest["fixed"],
|
||||
"release_evidence": manifest["release_evidence"],
|
||||
"duration_seconds": manifest["duration_seconds"],
|
||||
"data_dir": str(self.root / "data"),
|
||||
"expected_healed_objects": manifest["expected_healed_objects"]["cold-hot"],
|
||||
}
|
||||
metrics = dict.fromkeys(harness.METRICS, 10)
|
||||
metrics.update(p99_ms=10, throughput_ops=100, errors=0, requests=100,
|
||||
walk_objects=100, cold_walk_objects=20, healed_objects=10)
|
||||
result = {
|
||||
"evidence": request["evidence"],
|
||||
"fixed": request["fixed"],
|
||||
"build": request["build"],
|
||||
"data_dir": request["data_dir"],
|
||||
"background": request["background"],
|
||||
"release_evidence": request["release_evidence"],
|
||||
"sample_count": 10,
|
||||
"elapsed_seconds": request["duration_seconds"],
|
||||
"metrics": metrics,
|
||||
"oracle": manifest["oracles"]["cold-hot"],
|
||||
}
|
||||
harness.validate_result(result, request, manifest["oracles"]["cold-hot"])
|
||||
result["release_evidence"] = copy.deepcopy(result["release_evidence"])
|
||||
result["release_evidence"]["profile"]["required_artifacts"].remove("flamegraph")
|
||||
with self.assertRaisesRegex(ValueError, "release evidence provenance mismatch"):
|
||||
harness.validate_result(result, request, manifest["oracles"]["cold-hot"])
|
||||
|
||||
def test_existing_data_preserved(self):
|
||||
(self.root / "data").mkdir()
|
||||
marker = self.root / "data/keep"
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
@@ -33,6 +34,9 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
|
||||
self.abba = self.root / "abba"
|
||||
self.abba.mkdir()
|
||||
self.manifest = {
|
||||
"schema": 1,
|
||||
"evidence": "measured",
|
||||
"rounds": 3,
|
||||
"fixed": {
|
||||
"config_sha256": "1" * 64,
|
||||
"dataset_sha256": "2" * 64,
|
||||
@@ -45,6 +49,39 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
|
||||
"candidate": {"revision": "b" * 40, "sha256": "4" * 64},
|
||||
"adapter_sha256": "5" * 64,
|
||||
"collector_sha256": "6" * 64,
|
||||
"release_evidence": {
|
||||
"topology": {
|
||||
"nodes": 3,
|
||||
"drives_per_node": 4,
|
||||
"pools": 2,
|
||||
"sets_total": 2,
|
||||
"sampled_pools": 2,
|
||||
"sampled_sets": 2,
|
||||
"erasure_set_size": 12,
|
||||
"erasure_data_blocks": 8,
|
||||
"erasure_parity_blocks": 4,
|
||||
},
|
||||
"distributed": {
|
||||
"metrics_endpoints": ["https://node-1:9000", "https://node-2:9000", "https://node-3:9000"],
|
||||
"failure_domain": "three-node-localhost-lab",
|
||||
"same_window_sampling": True,
|
||||
},
|
||||
"crash_restart": {
|
||||
"fault_modes": ["process-restart", "process-crash-restart"],
|
||||
"unclean_shutdown_marker": True,
|
||||
},
|
||||
"mixed_version": {
|
||||
"participating_revisions": ["a" * 40, "b" * 40],
|
||||
"reader": True,
|
||||
"writer": True,
|
||||
"rollback_payload": True,
|
||||
},
|
||||
"profile": {
|
||||
"required_artifacts": ["allocation-profile", "flamegraph", "rss-samples", "save-frequency"],
|
||||
"collector_config_sha256": "7" * 64,
|
||||
"profiler_config_sha256": "8" * 64,
|
||||
},
|
||||
},
|
||||
}
|
||||
self.comparison = {
|
||||
"scenario": "cold-hot",
|
||||
@@ -55,16 +92,32 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
|
||||
"throughput_change": -0.01,
|
||||
"p1": {"required_reduction": 0.8, "observed_reduction": 0.82, "repeatability_drift": 0.01},
|
||||
"p2_post_stop_work_multiples": [None, 1.1, 1.0, None],
|
||||
"w10_w11": {
|
||||
"foreground_pressure_high_sample_ratios": [0.0, 0.25, 0.25, 0.0],
|
||||
"heal_lock_wait_p99_ms": [12.0, 8.0, 9.0, 13.0],
|
||||
"attempt_cost_per_healed_object": [None, 1.2, 1.3, None],
|
||||
"candidate_attempt_cost_per_healed_object": 1.3,
|
||||
},
|
||||
}
|
||||
self.report = {
|
||||
"status": "pass",
|
||||
"performance": "pass",
|
||||
"evidence": "measured",
|
||||
"cells": 120,
|
||||
"comparisons": [self.comparison],
|
||||
"comparisons": self.full_comparisons(),
|
||||
}
|
||||
self.write_inputs()
|
||||
|
||||
def full_comparisons(self):
|
||||
comparisons = []
|
||||
for scenario in summary.SCENARIOS:
|
||||
for comparison in ("build", "background"):
|
||||
for round_id in range(1, 4):
|
||||
row = copy.deepcopy(self.comparison)
|
||||
row.update(scenario=scenario, comparison=comparison, round=round_id)
|
||||
comparisons.append(row)
|
||||
return comparisons
|
||||
|
||||
def write_inputs(self):
|
||||
(self.abba / "manifest.json").write_text(json.dumps(self.manifest), encoding="utf-8")
|
||||
(self.abba / "report.json").write_text(json.dumps(self.report), encoding="utf-8")
|
||||
@@ -112,6 +165,107 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
|
||||
self.assertEqual(result["verdict"], "FAIL")
|
||||
self.assertIn("synthetic evidence", result["reason"])
|
||||
|
||||
def test_failed_abba_report_without_comparisons_writes_fail_closed_summary(self):
|
||||
self.report = {
|
||||
"status": "failed",
|
||||
"performance": "pending",
|
||||
"completed_cells": 7,
|
||||
"error": "collector failed",
|
||||
}
|
||||
self.write_inputs()
|
||||
args = type("Args", (), {
|
||||
"abba_dir": self.abba,
|
||||
"cache_cost_log": None,
|
||||
"require_cache_cost": False,
|
||||
"json_out": None,
|
||||
"markdown_out": None,
|
||||
})
|
||||
result = summary.build_summary(args)
|
||||
self.assertEqual(result["verdict"], "FAIL")
|
||||
self.assertEqual(result["abba"]["completed_cells"], 7)
|
||||
self.assertEqual(result["abba"]["comparisons_total"], 0)
|
||||
self.assertIn("collector failed", result["reason"])
|
||||
self.assertIn("- completed_cells: 7", summary.markdown(result))
|
||||
self.assertIn("- error: collector failed", summary.markdown(result))
|
||||
|
||||
def test_passing_abba_report_requires_comparisons(self):
|
||||
del self.report["comparisons"]
|
||||
self.write_inputs()
|
||||
args = type("Args", (), {
|
||||
"abba_dir": self.abba,
|
||||
"cache_cost_log": None,
|
||||
"require_cache_cost": False,
|
||||
"json_out": None,
|
||||
"markdown_out": None,
|
||||
})
|
||||
with self.assertRaisesRegex(ValueError, "passing report requires comparisons"):
|
||||
summary.build_summary(args)
|
||||
|
||||
def test_passing_abba_report_requires_complete_matrix(self):
|
||||
cases = {
|
||||
"trimmed": lambda: self.report["comparisons"].pop(),
|
||||
"duplicate": lambda: self.report["comparisons"].__setitem__(1, copy.deepcopy(self.report["comparisons"][0])),
|
||||
"bad cells": lambda: self.report.update(cells=119),
|
||||
"bad evidence": lambda: self.manifest.update(evidence="synthetic"),
|
||||
"outside": lambda: self.report["comparisons"][0].update(round=99),
|
||||
"failed comparison": lambda: self.report["comparisons"][0].update(status="inconclusive"),
|
||||
}
|
||||
for name, mutate in cases.items():
|
||||
with self.subTest(fault=name):
|
||||
self.setUp()
|
||||
mutate()
|
||||
self.write_inputs()
|
||||
args = type("Args", (), {
|
||||
"abba_dir": self.abba,
|
||||
"cache_cost_log": None,
|
||||
"require_cache_cost": False,
|
||||
"json_out": None,
|
||||
"markdown_out": None,
|
||||
})
|
||||
with self.assertRaisesRegex(ValueError, "ABBA matrix|manifest/report evidence|comparison"):
|
||||
summary.build_summary(args)
|
||||
|
||||
def test_passing_abba_report_requires_w10_w11_evidence(self):
|
||||
for fault in ("missing", "pressure", "lock", "attempt", "length", "range"):
|
||||
with self.subTest(fault=fault):
|
||||
self.setUp()
|
||||
target = self.report["comparisons"][0]
|
||||
if fault == "missing":
|
||||
del target["w10_w11"]
|
||||
elif fault == "pressure":
|
||||
del target["w10_w11"]["foreground_pressure_high_sample_ratios"]
|
||||
elif fault == "lock":
|
||||
del target["w10_w11"]["heal_lock_wait_p99_ms"]
|
||||
elif fault == "attempt":
|
||||
del target["w10_w11"]["attempt_cost_per_healed_object"]
|
||||
elif fault == "length":
|
||||
target["w10_w11"]["attempt_cost_per_healed_object"] = [None]
|
||||
else:
|
||||
target["w10_w11"]["foreground_pressure_high_sample_ratios"] = [1.5, 0.0, 0.0, 0.0]
|
||||
self.write_inputs()
|
||||
args = type("Args", (), {
|
||||
"abba_dir": self.abba,
|
||||
"cache_cost_log": None,
|
||||
"require_cache_cost": False,
|
||||
"json_out": None,
|
||||
"markdown_out": None,
|
||||
})
|
||||
with self.assertRaisesRegex(ValueError, "W10/W11|performance evidence|length mismatch|above maximum"):
|
||||
summary.build_summary(args)
|
||||
|
||||
def test_passing_measured_report_requires_release_evidence_manifest(self):
|
||||
del self.manifest["release_evidence"]
|
||||
self.write_inputs()
|
||||
args = type("Args", (), {
|
||||
"abba_dir": self.abba,
|
||||
"cache_cost_log": None,
|
||||
"require_cache_cost": False,
|
||||
"json_out": None,
|
||||
"markdown_out": None,
|
||||
})
|
||||
with self.assertRaisesRegex(ValueError, "release_evidence"):
|
||||
summary.build_summary(args)
|
||||
|
||||
def test_requires_cache_profile_when_requested(self):
|
||||
args = type("Args", (), {
|
||||
"abba_dir": self.abba,
|
||||
|
||||
Reference in New Issue
Block a user