mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-08 13:06:00 +00:00
Compare commits
51 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 600d037b25 | |||
| 3fa2b334be | |||
| 1b549d5907 | |||
| 11c4ce96eb | |||
| 4508a0985d | |||
| ed1b9f25d6 | |||
| 274c2bf402 | |||
| 9aebcefa9c | |||
| 1748814bbf | |||
| ee5f76c180 | |||
| 4d68c32b75 | |||
| 8c6689ff13 | |||
| 595f9f662d | |||
| 4a2b15cb82 | |||
| 2ba7f95547 | |||
| 7b3dad6bae | |||
| c0754f5b1c | |||
| 5dc0e3b402 | |||
| cc5487e7de | |||
| 0b05b6c6ff | |||
| c507da8f75 | |||
| 550dabeffd | |||
| 4d7f0344d3 | |||
| 084338add6 | |||
| 8d339da706 | |||
| 817ad0a682 | |||
| 80321e5bb4 | |||
| e7475cfa4d | |||
| 8c15025a5a | |||
| 6f2ec66263 | |||
| 62542ddc57 | |||
| a8bb53218d | |||
| 5355d9f8f8 | |||
| 50d7a049ee | |||
| 3149c87cf2 | |||
| b7fa6a4615 | |||
| 6fe83f87a4 | |||
| df6981d88e | |||
| 590adad5ae | |||
| be5d14c985 | |||
| 703086d71d | |||
| a159f312f0 | |||
| 2f6f095298 | |||
| efd8ef005f | |||
| 60fa33773a | |||
| ee752b0b03 | |||
| 99c1f4418b | |||
| 7ce0ac72cf | |||
| 944e26d432 | |||
| f0b0a99260 | |||
| 33ddc10ffd |
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"schema": 1,
|
"schema": 2,
|
||||||
"cases": {
|
"cases": {
|
||||||
"background-target-restart": {
|
"background-target-restart": {
|
||||||
"gate": "G14",
|
"gate": "G14",
|
||||||
@@ -13,6 +13,7 @@
|
|||||||
"min_objects": 9,
|
"min_objects": 9,
|
||||||
"max_objects": 65,
|
"max_objects": 65,
|
||||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||||
|
"erasure": {"data_blocks": 2, "parity_blocks": 2},
|
||||||
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||||
},
|
},
|
||||||
"background-target-crash": {
|
"background-target-crash": {
|
||||||
@@ -27,30 +28,284 @@
|
|||||||
"min_objects": 9,
|
"min_objects": 9,
|
||||||
"max_objects": 65,
|
"max_objects": 65,
|
||||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||||
|
"erasure": {"data_blocks": 2, "parity_blocks": 2},
|
||||||
"scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
"scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||||
|
},
|
||||||
|
"ec84-target-drive-restart": {
|
||||||
|
"gate": "G14",
|
||||||
|
"task": "W20/W21",
|
||||||
|
"lane": "e2e-distributed",
|
||||||
|
"suite": "e2e_test",
|
||||||
|
"name": "distributed::heal_test::three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart",
|
||||||
|
"oracle": "ec84-target-drive-restart.json",
|
||||||
|
"evidence": "process-restart",
|
||||||
|
"unclean_shutdown_marker": false,
|
||||||
|
"min_objects": 5,
|
||||||
|
"max_objects": 5,
|
||||||
|
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||||
|
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||||
|
"erasure_set_drive_count": 12,
|
||||||
|
"scope": "3-node x 4-drive single-set EC8+4, graceful target restart, preformatted replacement drive, exact unversioned S3 bodies and physical target shards; not mixed-version, multi-pool or long-window ABBA."
|
||||||
|
},
|
||||||
|
"background-target-restart-ec8-4": {
|
||||||
|
"gate": "G14",
|
||||||
|
"task": "W21",
|
||||||
|
"lane": "e2e-nightly",
|
||||||
|
"suite": "e2e_test",
|
||||||
|
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart",
|
||||||
|
"oracle": "background-target-restart-ec8-4.json",
|
||||||
|
"evidence": "process-restart",
|
||||||
|
"unclean_shutdown_marker": false,
|
||||||
|
"min_objects": 9,
|
||||||
|
"max_objects": 65,
|
||||||
|
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||||
|
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||||
|
"erasure_set_drive_count": 12,
|
||||||
|
"scope": "Target process restart during partial background rebuild on a single 3x4 EC8+4 set; exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
|
||||||
|
},
|
||||||
|
"background-target-crash-ec8-4": {
|
||||||
|
"gate": "G14",
|
||||||
|
"task": "W21",
|
||||||
|
"lane": "e2e-nightly",
|
||||||
|
"suite": "e2e_test",
|
||||||
|
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash",
|
||||||
|
"oracle": "background-target-crash-ec8-4.json",
|
||||||
|
"evidence": "process-crash-restart",
|
||||||
|
"unclean_shutdown_marker": true,
|
||||||
|
"min_objects": 9,
|
||||||
|
"max_objects": 65,
|
||||||
|
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||||
|
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||||
|
"erasure_set_drive_count": 12,
|
||||||
|
"scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"release_pending": {
|
"release_lanes": {
|
||||||
"G01": "W02/W04 complete root and quota authority coverage",
|
"single-set-restart": {
|
||||||
"G02": "W03 bounded checkpoint progress and independent version inventory",
|
"status": "implemented",
|
||||||
"G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
|
"cases": ["background-target-restart", "background-target-crash"],
|
||||||
"G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
|
"covers": ["four-node one-drive topology", "unversioned objects", "target restart/crash"]
|
||||||
"G05": "W06/W07 per-object outcomes and bounded terminal retention",
|
},
|
||||||
"G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
|
"authority-coverage": {
|
||||||
"G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
|
"status": "pending",
|
||||||
"G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
|
"gates": ["G01", "G12"],
|
||||||
"G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
|
"requires": ["root authority coverage", "quota authority coverage"]
|
||||||
"G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
|
},
|
||||||
"G11": "W04/W19/W24 maintenance and complete producer coverage",
|
"checkpoint-and-crash": {
|
||||||
"G12": "W02/W15/W16 both quota paths during reset and settlement",
|
"status": "pending",
|
||||||
"G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
"gates": ["G02", "G04", "R-E"],
|
||||||
"G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
|
"requires": ["bounded checkpoint progress", "boundary crash matrix", "fixed-budget restart evidence"]
|
||||||
"P1": "W20 measured cold-walk share and foreground latency/throughput",
|
},
|
||||||
"P2": "W20/W24 measured post-stop convergence and cold segment reuse",
|
"status-and-outcome": {
|
||||||
"P3": "W20 measured two-hour pressure/heal capacity and recovery window",
|
"status": "pending",
|
||||||
"P4": "W20 measured MRF scale and replay cost with retained responsibility",
|
"gates": ["G05", "G06", "R-D"],
|
||||||
"R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
|
"requires": ["per-object outcomes", "legacy status clients", "manager/event/ledger disposition"]
|
||||||
"R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
|
},
|
||||||
"R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
|
"mrf-responsibility": {
|
||||||
|
"status": "pending",
|
||||||
|
"gates": ["G07", "G08", "P4"],
|
||||||
|
"requires": ["durable MRF responsibility", "disk-full and replica-loss matrix", "MRF replay cost"]
|
||||||
|
},
|
||||||
|
"mixed-version-rollback": {
|
||||||
|
"status": "pending",
|
||||||
|
"gates": ["G03", "G09", "R-L"],
|
||||||
|
"requires": ["mixed-version peers", "rollback payloads", "crash-safe source retirement"]
|
||||||
|
},
|
||||||
|
"scheduler-pressure": {
|
||||||
|
"status": "pending",
|
||||||
|
"gates": ["G10", "P1", "P2", "P3"],
|
||||||
|
"requires": ["bounded scheduling", "foreground latency and throughput", "two-hour pressure evidence"]
|
||||||
|
},
|
||||||
|
"maintenance-producers": {
|
||||||
|
"status": "pending",
|
||||||
|
"gates": ["G11", "G13"],
|
||||||
|
"requires": ["complete producer coverage", "quorum-minus-one and remount matrix"]
|
||||||
|
},
|
||||||
|
"ec8-4-multiset": {
|
||||||
|
"status": "pending",
|
||||||
|
"gates": ["G14"],
|
||||||
|
"requires": ["3x4 EC8+4 topology", "multi-set coverage", "multi-pool coverage"]
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"release_requirements": [
|
||||||
|
{
|
||||||
|
"gate": "G01",
|
||||||
|
"task": "W02/W04",
|
||||||
|
"lane": "authority-coverage",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Complete root and quota authority coverage",
|
||||||
|
"requires": ["root authority evidence", "quota authority evidence"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G02",
|
||||||
|
"task": "W03",
|
||||||
|
"lane": "checkpoint-and-crash",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Bounded checkpoint progress and independent version inventory",
|
||||||
|
"requires": ["bounded checkpoint oracle", "independent version inventory"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G03",
|
||||||
|
"task": "W17/W18",
|
||||||
|
"lane": "mixed-version-rollback",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Exact scoped ACK with durable publication and mixed peers",
|
||||||
|
"requires": ["durable scoped ACK publication", "mixed-peer evidence"],
|
||||||
|
"evidence_fields": [
|
||||||
|
"durable_root_publication_proof",
|
||||||
|
"scoped_ack_request_identity",
|
||||||
|
"participating_peer_capability_snapshot",
|
||||||
|
"mixed_peer_ack_fallback_oracle"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G04",
|
||||||
|
"task": "W03/W15/W16",
|
||||||
|
"lane": "checkpoint-and-crash",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Crash at every cache, root, floor and intent boundary",
|
||||||
|
"requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G05",
|
||||||
|
"task": "W06/W07",
|
||||||
|
"lane": "status-and-outcome",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Per-object outcomes and bounded terminal retention",
|
||||||
|
"requires": ["per-object outcome oracle", "terminal retention bounds"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G06",
|
||||||
|
"task": "W06/W08/W23",
|
||||||
|
"lane": "status-and-outcome",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Concurrent status, legacy clients and truncation",
|
||||||
|
"requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G07",
|
||||||
|
"task": "W12/W13/W14",
|
||||||
|
"lane": "mrf-responsibility",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Durable MRF responsibility at every commit boundary",
|
||||||
|
"requires": ["MRF responsibility oracle", "commit-boundary crash matrix"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G08",
|
||||||
|
"task": "W12/W13/W14",
|
||||||
|
"lane": "mrf-responsibility",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "MRF capacity, disk-full and replica-loss matrix",
|
||||||
|
"requires": ["MRF capacity evidence", "disk-full matrix", "replica-loss matrix"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G09",
|
||||||
|
"task": "W13/W18/W23",
|
||||||
|
"lane": "mixed-version-rollback",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Actual mixed-version reader/writer and rollback payloads",
|
||||||
|
"requires": ["mixed-version reader evidence", "mixed-version writer evidence", "rollback payload evidence"],
|
||||||
|
"evidence_fields": [
|
||||||
|
"mixed_version_reader_evidence",
|
||||||
|
"mixed_version_writer_evidence",
|
||||||
|
"rollback_payload_evidence"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G10",
|
||||||
|
"task": "W05/W09/W10/W11",
|
||||||
|
"lane": "scheduler-pressure",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Bounded scheduling and pressure recovery",
|
||||||
|
"requires": ["scheduler bound evidence", "pressure recovery evidence"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G11",
|
||||||
|
"task": "W04/W19/W24",
|
||||||
|
"lane": "maintenance-producers",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Maintenance and complete producer coverage",
|
||||||
|
"requires": ["maintenance producer matrix", "complete producer inventory"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G12",
|
||||||
|
"task": "W02/W15/W16",
|
||||||
|
"lane": "authority-coverage",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Both quota paths during reset and settlement",
|
||||||
|
"requires": ["reset quota-path evidence", "settlement quota-path evidence"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G13",
|
||||||
|
"task": "W07/W14",
|
||||||
|
"lane": "maintenance-producers",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||||
|
"requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "G14",
|
||||||
|
"task": "W20/W21",
|
||||||
|
"lane": "ec8-4-multiset",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Same-window field evidence with 3x4 EC8+4 and multi-set/pool coverage",
|
||||||
|
"requires": ["same-window field evidence", "3x4 EC8+4 evidence", "multi-set evidence", "multi-pool evidence"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "P1",
|
||||||
|
"task": "W20",
|
||||||
|
"lane": "scheduler-pressure",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Measured cold-walk share and foreground latency/throughput",
|
||||||
|
"requires": ["cold-walk share measurement", "foreground latency/throughput measurement"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "P2",
|
||||||
|
"task": "W20/W24",
|
||||||
|
"lane": "scheduler-pressure",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Measured post-stop convergence and cold segment reuse",
|
||||||
|
"requires": ["post-stop convergence measurement", "cold segment reuse measurement"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "P3",
|
||||||
|
"task": "W20",
|
||||||
|
"lane": "scheduler-pressure",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Measured two-hour pressure/heal capacity and recovery window",
|
||||||
|
"requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "P4",
|
||||||
|
"task": "W20",
|
||||||
|
"lane": "mrf-responsibility",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Measured MRF scale and replay cost with retained responsibility",
|
||||||
|
"requires": ["MRF scale measurement", "MRF replay-cost measurement", "retained responsibility evidence"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "R-E",
|
||||||
|
"task": "W03/W05",
|
||||||
|
"lane": "checkpoint-and-crash",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Fixed-budget real process restart through enumeration and classification",
|
||||||
|
"requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "R-D",
|
||||||
|
"task": "W07/W14",
|
||||||
|
"lane": "status-and-outcome",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Manager-to-event-to-ledger exact disposition, including grace",
|
||||||
|
"requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gate": "R-L",
|
||||||
|
"task": "W13/W14",
|
||||||
|
"lane": "mixed-version-rollback",
|
||||||
|
"status": "pending",
|
||||||
|
"description": "Legacy source conflicts, migration gaps and crash-safe source retirement",
|
||||||
|
"requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"]
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ name: Continuous Integration (docs only)
|
|||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
types: [ opened, synchronize, reopened ]
|
types: [ opened, synchronize, reopened ]
|
||||||
branches: [ main ]
|
branches: [ main, release ]
|
||||||
paths:
|
paths:
|
||||||
- "**.md"
|
- "**.md"
|
||||||
- "docs/**"
|
- "docs/**"
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ name: Continuous Integration
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [ main ]
|
branches: [ main, release ]
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- "**.md"
|
- "**.md"
|
||||||
- "docs/**"
|
- "docs/**"
|
||||||
@@ -36,7 +36,7 @@ on:
|
|||||||
- "flake.lock"
|
- "flake.lock"
|
||||||
pull_request:
|
pull_request:
|
||||||
types: [ opened, synchronize, reopened, closed ]
|
types: [ opened, synchronize, reopened, closed ]
|
||||||
branches: [ main ]
|
branches: [ main, release ]
|
||||||
# Keep this list in sync with the `paths` list in ci-docs-only.yml, which
|
# Keep this list in sync with the `paths` list in ci-docs-only.yml, which
|
||||||
# reports the required "Test and Lint" check for PRs skipped here.
|
# reports the required "Test and Lint" check for PRs skipped here.
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
@@ -872,13 +872,14 @@ jobs:
|
|||||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
# every PR, so it is gated to main/release pushes, the merge queue, and manual
|
||||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||||
if: >-
|
if: >-
|
||||||
github.event_name == 'workflow_dispatch' ||
|
github.event_name == 'workflow_dispatch' ||
|
||||||
github.event_name == 'merge_group' ||
|
github.event_name == 'merge_group' ||
|
||||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
(github.event_name == 'push' &&
|
||||||
|
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release'))
|
||||||
needs: [ build-rustfs-debug-binary ]
|
needs: [ build-rustfs-debug-binary ]
|
||||||
runs-on: sm-standard-2
|
runs-on: sm-standard-2
|
||||||
timeout-minutes: 55
|
timeout-minutes: 55
|
||||||
|
|||||||
@@ -145,6 +145,38 @@ pub fn consume_verified_mrf_repair_events(anchors: &mut Vec<MrfDurableRepairAnch
|
|||||||
before.saturating_sub(anchors.len())
|
before.saturating_sub(anchors.len())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Consume recorded verified repairs for one bucket without draining
|
||||||
|
/// unrelated or still-unmatched proofs. If the caller crashes before
|
||||||
|
/// persisting the retained anchor set, the proof may be replayed by repair
|
||||||
|
/// instead of silently deleting the old responsibility.
|
||||||
|
pub fn consume_recorded_verified_mrf_repair_events_for(bucket: &str, anchors: &mut Vec<MrfDurableRepairAnchor>) -> usize {
|
||||||
|
let Some(registry) = MRF_VERIFIED_REPAIR_EVENTS.get() else {
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
let Ok(mut events) = registry.lock() else {
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
let before = anchors.len();
|
||||||
|
let mut retained = std::collections::VecDeque::with_capacity(events.len());
|
||||||
|
while let Some(event) = events.pop_front() {
|
||||||
|
if event.bucket.as_ref() != bucket {
|
||||||
|
retained.push_back(event);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let mut matched = false;
|
||||||
|
anchors.retain(|anchor| {
|
||||||
|
let proven = anchor.is_proven_by(&event);
|
||||||
|
matched |= proven;
|
||||||
|
!proven
|
||||||
|
});
|
||||||
|
if !matched {
|
||||||
|
retained.push_back(event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*events = retained;
|
||||||
|
before.saturating_sub(anchors.len())
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||||
pub struct MrfScope {
|
pub struct MrfScope {
|
||||||
pub pool_index: u32,
|
pub pool_index: u32,
|
||||||
@@ -821,6 +853,71 @@ mod tests {
|
|||||||
assert!(retained.is_empty());
|
assert!(retained.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recorded_verified_repair_consumer_retains_unmatched_proofs() {
|
||||||
|
let bucket = Arc::<str>::from(format!("recorded-proof-{}", Uuid::new_v4()));
|
||||||
|
let other_bucket = Arc::<str>::from(format!("recorded-proof-other-{}", Uuid::new_v4()));
|
||||||
|
let incarnation = Uuid::new_v4();
|
||||||
|
let lease = MrfIngressLease::new(21);
|
||||||
|
let retained_anchor = MrfDurableRepairAnchor {
|
||||||
|
kind: MrfKind::PartialWrite,
|
||||||
|
bucket: bucket.clone(),
|
||||||
|
object: Arc::from("retained"),
|
||||||
|
version_id: Some([7; 16]),
|
||||||
|
scope: Some(MrfScope {
|
||||||
|
pool_index: 1,
|
||||||
|
set_index: 2,
|
||||||
|
}),
|
||||||
|
lease,
|
||||||
|
bucket_incarnation_id: incarnation,
|
||||||
|
};
|
||||||
|
let waiting_anchor = MrfDurableRepairAnchor {
|
||||||
|
object: Arc::from("waiting"),
|
||||||
|
lease: MrfIngressLease::new(22),
|
||||||
|
..retained_anchor.clone()
|
||||||
|
};
|
||||||
|
let matched_event = MrfVerifiedRepairEvent {
|
||||||
|
kind: retained_anchor.kind,
|
||||||
|
bucket: bucket.clone(),
|
||||||
|
object: retained_anchor.object.clone(),
|
||||||
|
version_id: retained_anchor.version_id,
|
||||||
|
scope: retained_anchor.scope,
|
||||||
|
lease: Some(retained_anchor.lease),
|
||||||
|
bucket_incarnation_id: retained_anchor.bucket_incarnation_id,
|
||||||
|
disposition: MrfVerifiedRepairDisposition::Repaired,
|
||||||
|
};
|
||||||
|
let same_bucket_unmatched = MrfVerifiedRepairEvent {
|
||||||
|
object: Arc::from("future"),
|
||||||
|
lease: Some(MrfIngressLease::new(23)),
|
||||||
|
..matched_event.clone()
|
||||||
|
};
|
||||||
|
let other_bucket_event = MrfVerifiedRepairEvent {
|
||||||
|
bucket: other_bucket.clone(),
|
||||||
|
..matched_event.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
note_mrf_verified_repair(matched_event);
|
||||||
|
note_mrf_verified_repair(same_bucket_unmatched.clone());
|
||||||
|
note_mrf_verified_repair(other_bucket_event.clone());
|
||||||
|
|
||||||
|
let mut anchors = vec![retained_anchor, waiting_anchor.clone()];
|
||||||
|
assert_eq!(consume_recorded_verified_mrf_repair_events_for(&bucket, &mut anchors), 1);
|
||||||
|
assert_eq!(anchors, vec![waiting_anchor]);
|
||||||
|
|
||||||
|
let remaining_bucket_events = take_mrf_verified_repair_events_for(&bucket);
|
||||||
|
assert_eq!(
|
||||||
|
remaining_bucket_events,
|
||||||
|
vec![same_bucket_unmatched],
|
||||||
|
"same-bucket proofs without a retained anchor must remain available"
|
||||||
|
);
|
||||||
|
let remaining_other_events = take_mrf_verified_repair_events_for(&other_bucket);
|
||||||
|
assert_eq!(
|
||||||
|
remaining_other_events,
|
||||||
|
vec![other_bucket_event],
|
||||||
|
"proofs for other buckets must not be drained by this consumer"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn try_send_delivers_and_respects_capacity() {
|
async fn try_send_delivers_and_respects_capacity() {
|
||||||
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
||||||
|
|||||||
@@ -59,6 +59,9 @@ const POOL_META_V3_ENV: [(&str, &str); 2] = [
|
|||||||
|
|
||||||
#[derive(Clone, Copy, Debug)]
|
#[derive(Clone, Copy, Debug)]
|
||||||
pub(crate) enum DistLayout {
|
pub(crate) enum DistLayout {
|
||||||
|
/// 3 nodes × 4 drives, one erasure pool. With `EC:4` this is the
|
||||||
|
/// release-evidence EC8+4 geometry.
|
||||||
|
ThreeByFourEc84,
|
||||||
/// 4 nodes × 4 drives, one erasure pool spanning every endpoint.
|
/// 4 nodes × 4 drives, one erasure pool spanning every endpoint.
|
||||||
FourByFour,
|
FourByFour,
|
||||||
/// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout).
|
/// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout).
|
||||||
@@ -94,6 +97,7 @@ impl DistCluster {
|
|||||||
|
|
||||||
pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
|
pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
|
||||||
let topology = match layout {
|
let topology = match layout {
|
||||||
|
DistLayout::ThreeByFourEc84 => ClusterTopology::single_pool_multidrive(3, DRIVES_PER_NODE),
|
||||||
DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE),
|
DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE),
|
||||||
DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT),
|
DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT),
|
||||||
DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]),
|
DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]),
|
||||||
@@ -101,7 +105,7 @@ impl DistCluster {
|
|||||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?;
|
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?;
|
||||||
let pool_storage_roots = match layout {
|
let pool_storage_roots = match layout {
|
||||||
DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?),
|
DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?),
|
||||||
DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
|
DistLayout::ThreeByFourEc84 | DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
|
||||||
};
|
};
|
||||||
let mut owned_pool_dirs = Vec::new();
|
let mut owned_pool_dirs = Vec::new();
|
||||||
if let Some(roots) = pool_storage_roots.as_deref() {
|
if let Some(roots) = pool_storage_roots.as_deref() {
|
||||||
|
|||||||
@@ -0,0 +1,367 @@
|
|||||||
|
// Copyright 2026 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use super::harness::{
|
||||||
|
DistCluster, DistLayout, TestResult, assert_inventory, get_object_bytes, payload_for, put_object, sha256_hex, unique_bucket,
|
||||||
|
wait_until,
|
||||||
|
};
|
||||||
|
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||||
|
use crate::common::init_logging;
|
||||||
|
use aws_sdk_s3::Client;
|
||||||
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
|
use serde_json::Value;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::collections::{BTreeMap, HashSet};
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
const EC84_NODE_COUNT: usize = 3;
|
||||||
|
const EC84_DRIVES_PER_NODE: usize = 4;
|
||||||
|
const EC84_DATA_BLOCKS: usize = 8;
|
||||||
|
const EC84_PARITY_BLOCKS: usize = 4;
|
||||||
|
const EC84_TARGET_DRIVE_RESTART_CASE: &str = "ec84-target-drive-restart";
|
||||||
|
const EC84_TARGET_DRIVE_RESTART_ORACLE: &str = "ec84-target-drive-restart.json";
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct ExpectedShard {
|
||||||
|
key: String,
|
||||||
|
body: Vec<u8>,
|
||||||
|
baseline: VersionShardCensus,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ScannerHealEvidenceContext {
|
||||||
|
directory: PathBuf,
|
||||||
|
run: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ScannerHealEvidencePayload<'a> {
|
||||||
|
dist: &'a DistCluster,
|
||||||
|
bucket: &'a str,
|
||||||
|
expected: &'a [ExpectedShard],
|
||||||
|
outage_key: &'a str,
|
||||||
|
outage_body: &'a [u8],
|
||||||
|
replaced_drive: &'a Path,
|
||||||
|
pid_before: u32,
|
||||||
|
pid_after: u32,
|
||||||
|
node_listings: Vec<Vec<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn file_sha256(path: &Path) -> TestResult<String> {
|
||||||
|
let mut file = std::fs::File::open(path)?;
|
||||||
|
let mut digest = Sha256::new();
|
||||||
|
let mut buffer = [0_u8; 64 * 1024];
|
||||||
|
loop {
|
||||||
|
let read = file.read(&mut buffer)?;
|
||||||
|
if read == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
digest.update(&buffer[..read]);
|
||||||
|
}
|
||||||
|
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn compiled_test_identity() -> Value {
|
||||||
|
serde_json::json!({
|
||||||
|
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||||
|
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||||
|
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||||
|
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||||
|
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||||
|
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||||
|
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn string_field<'a>(value: &'a Value, path: &str) -> TestResult<&'a str> {
|
||||||
|
let mut current = value;
|
||||||
|
for segment in path.split('.') {
|
||||||
|
current = current
|
||||||
|
.get(segment)
|
||||||
|
.ok_or_else(|| format!("scanner/heal run receipt missing {path}"))?;
|
||||||
|
}
|
||||||
|
current
|
||||||
|
.as_str()
|
||||||
|
.filter(|text| !text.is_empty())
|
||||||
|
.ok_or_else(|| format!("scanner/heal run receipt has invalid {path}").into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn scanner_heal_evidence_context() -> TestResult<Option<ScannerHealEvidenceContext>> {
|
||||||
|
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let directory = PathBuf::from(directory);
|
||||||
|
let receipt = directory.join("run.json");
|
||||||
|
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||||
|
return Err("oversized scanner/heal execution receipt".into());
|
||||||
|
}
|
||||||
|
let run: Value = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||||
|
let built = compiled_test_identity();
|
||||||
|
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||||
|
if built[key] != run["test_build"][key] {
|
||||||
|
return Err(format!("compiled test identity differs for {key}").into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let binary_path = PathBuf::from(string_field(&run, "binary.path")?);
|
||||||
|
if file_sha256(&binary_path)? != string_field(&run, "binary.sha256")? {
|
||||||
|
return Err("server binary must match the run receipt".into());
|
||||||
|
}
|
||||||
|
if file_sha256(&std::env::current_exe()?)? != string_field(&run, "test_binary.sha256")? {
|
||||||
|
return Err("test executable must match the run receipt".into());
|
||||||
|
}
|
||||||
|
if directory.join(EC84_TARGET_DRIVE_RESTART_ORACLE).exists() {
|
||||||
|
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||||
|
}
|
||||||
|
Ok(Some(ScannerHealEvidenceContext { directory, run }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_ec84_geometry(census: &VersionShardCensus, key: &str) -> TestResult {
|
||||||
|
if census.data_blocks != Some(EC84_DATA_BLOCKS) || census.parity_blocks != Some(EC84_PARITY_BLOCKS) {
|
||||||
|
return Err(format!("object {key} did not use EC8+4 geometry: {census:?}").into());
|
||||||
|
}
|
||||||
|
let erasure_index = census
|
||||||
|
.erasure_index
|
||||||
|
.ok_or_else(|| format!("object {key} did not record an erasure index: {census:?}"))?;
|
||||||
|
if !(1..=EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS).contains(&erasure_index) {
|
||||||
|
return Err(format!("object {key} has out-of-range erasure index {erasure_index}: {census:?}").into());
|
||||||
|
}
|
||||||
|
if !census.is_complete() || census.expected_part_numbers.is_empty() {
|
||||||
|
return Err(format!("object {key} does not have complete physical shard evidence: {census:?}").into());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write_scanner_heal_evidence(context: ScannerHealEvidenceContext, payload: ScannerHealEvidencePayload<'_>) -> TestResult {
|
||||||
|
let verifier = payload.dist.client(0)?;
|
||||||
|
let mut objects = Vec::new();
|
||||||
|
for item in payload.expected {
|
||||||
|
let actual = get_object_bytes(&verifier, payload.bucket, &item.key).await?;
|
||||||
|
let physical = census_object_version_on_disk(payload.replaced_drive, payload.bucket, &item.key, None)?;
|
||||||
|
objects.push(serde_json::json!({
|
||||||
|
"key": item.key,
|
||||||
|
"version_id": null,
|
||||||
|
"expected_bytes": item.body.len(),
|
||||||
|
"actual_bytes": actual.len(),
|
||||||
|
"expected_sha256": sha256_hex(&item.body),
|
||||||
|
"actual_sha256": sha256_hex(&actual),
|
||||||
|
"expected_physical": item.baseline,
|
||||||
|
"physical": physical,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
let actual = get_object_bytes(&verifier, payload.bucket, payload.outage_key).await?;
|
||||||
|
let physical = census_object_version_on_disk(payload.replaced_drive, payload.bucket, payload.outage_key, None)?;
|
||||||
|
objects.push(serde_json::json!({
|
||||||
|
"key": payload.outage_key,
|
||||||
|
"version_id": null,
|
||||||
|
"expected_bytes": payload.outage_body.len(),
|
||||||
|
"actual_bytes": actual.len(),
|
||||||
|
"expected_sha256": sha256_hex(payload.outage_body),
|
||||||
|
"actual_sha256": sha256_hex(&actual),
|
||||||
|
"expected_physical": null,
|
||||||
|
"physical": physical,
|
||||||
|
}));
|
||||||
|
|
||||||
|
let evidence = serde_json::json!({
|
||||||
|
"schema": 1,
|
||||||
|
"case": EC84_TARGET_DRIVE_RESTART_CASE,
|
||||||
|
"evidence": "process-restart",
|
||||||
|
"run_id": string_field(&context.run, "run_id")?,
|
||||||
|
"source_revision": string_field(&context.run, "source_revision")?,
|
||||||
|
"test_build": compiled_test_identity(),
|
||||||
|
"binary_sha256": string_field(&context.run, "binary.sha256")?,
|
||||||
|
"test_binary_sha256": string_field(&context.run, "test_binary.sha256")?,
|
||||||
|
"topology": {"nodes": EC84_NODE_COUNT, "drives_per_node": EC84_DRIVES_PER_NODE},
|
||||||
|
"pid_before": payload.pid_before,
|
||||||
|
"pid_after": payload.pid_after,
|
||||||
|
"unclean_shutdown_marker": false,
|
||||||
|
"objects": objects,
|
||||||
|
"node_listings": payload.node_listings,
|
||||||
|
});
|
||||||
|
let data = serde_json::to_vec(&evidence)?;
|
||||||
|
if data.len() > 1024 * 1024 {
|
||||||
|
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||||
|
}
|
||||||
|
let mut output = std::fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.open(context.directory.join(EC84_TARGET_DRIVE_RESTART_ORACLE))?;
|
||||||
|
output.write_all(&data)?;
|
||||||
|
output.sync_all()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> TestResult {
|
||||||
|
for key in keys {
|
||||||
|
let census = census_object_version_on_disk(drive, bucket, key, None)?;
|
||||||
|
if census.has_xl_meta {
|
||||||
|
return Err(format!("replacement drive unexpectedly retained {bucket}/{key}: {census:?}").into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
|
||||||
|
let mut expected = Vec::new();
|
||||||
|
for index in 0..4 {
|
||||||
|
let key = format!("ec84/prefix-{}/object-{index:04}.bin", index % 2);
|
||||||
|
let body = payload_for(&key, 10 * 1024 * 1024);
|
||||||
|
put_object(client, bucket, &key, body.clone()).await?;
|
||||||
|
expected.push(ExpectedShard {
|
||||||
|
key,
|
||||||
|
body,
|
||||||
|
baseline: VersionShardCensus {
|
||||||
|
version_id: None,
|
||||||
|
has_xl_meta: false,
|
||||||
|
data_dir: None,
|
||||||
|
erasure_index: None,
|
||||||
|
data_blocks: None,
|
||||||
|
parity_blocks: None,
|
||||||
|
expected_part_numbers: Default::default(),
|
||||||
|
present_part_fingerprints: Default::default(),
|
||||||
|
inline_data_fingerprint: None,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart() -> TestResult {
|
||||||
|
init_logging();
|
||||||
|
let evidence_context = scanner_heal_evidence_context()?;
|
||||||
|
let mut dist = DistCluster::start_with_env(
|
||||||
|
DistLayout::ThreeByFourEc84,
|
||||||
|
&[
|
||||||
|
("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
|
||||||
|
("RUSTFS_HEAL_ENABLED", "true"),
|
||||||
|
("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"),
|
||||||
|
("RUSTFS_HEAL_MRF_ENABLE", "false"),
|
||||||
|
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
assert_eq!(dist.cluster.nodes.len(), EC84_NODE_COUNT);
|
||||||
|
assert_eq!(dist.cluster.topology.drives_per_node, EC84_DRIVES_PER_NODE);
|
||||||
|
|
||||||
|
let bucket = unique_bucket("healec84");
|
||||||
|
dist.create_bucket(&bucket).await?;
|
||||||
|
let writer = dist.client(0)?;
|
||||||
|
let mut expected = put_large_inventory(&writer, &bucket).await?;
|
||||||
|
let replaced_node = 1;
|
||||||
|
let replaced_drive_index = 2;
|
||||||
|
let replaced_drive = PathBuf::from(&dist.cluster.nodes[replaced_node].data_dirs[replaced_drive_index]);
|
||||||
|
|
||||||
|
for item in &mut expected {
|
||||||
|
item.baseline = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
|
||||||
|
assert_ec84_geometry(&item.baseline, &item.key)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let format_path = replaced_drive.join(".rustfs.sys").join("format.json");
|
||||||
|
let format_json = std::fs::read(&format_path)?;
|
||||||
|
let target_pid_before = dist.cluster.nodes[replaced_node]
|
||||||
|
.process
|
||||||
|
.as_ref()
|
||||||
|
.ok_or("target process is absent before graceful restart")?
|
||||||
|
.id();
|
||||||
|
dist.cluster.stop_node_gracefully(replaced_node).await?;
|
||||||
|
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
|
||||||
|
std::fs::rename(&replaced_drive, &retired_drive)?;
|
||||||
|
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path has no parent")?)?;
|
||||||
|
std::fs::write(&format_path, format_json)?;
|
||||||
|
assert_replaced_drive_empty(
|
||||||
|
&replaced_drive,
|
||||||
|
&bucket,
|
||||||
|
&expected.iter().map(|item| item.key.clone()).collect::<Vec<_>>(),
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let outage_key = "ec84/written-while-node-restarting.bin";
|
||||||
|
let outage_body = payload_for(outage_key, 10 * 1024 * 1024);
|
||||||
|
writer
|
||||||
|
.put_object()
|
||||||
|
.bucket(&bucket)
|
||||||
|
.key(outage_key)
|
||||||
|
.body(ByteStream::from(outage_body.clone()))
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
dist.cluster.start_node(replaced_node).await?;
|
||||||
|
let target_pid_after = dist.cluster.nodes[replaced_node]
|
||||||
|
.process
|
||||||
|
.as_ref()
|
||||||
|
.ok_or("target process is absent after restart")?
|
||||||
|
.id();
|
||||||
|
let heal_body =
|
||||||
|
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||||
|
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
|
||||||
|
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||||
|
|
||||||
|
wait_until(
|
||||||
|
Duration::from_secs(120),
|
||||||
|
|| async {
|
||||||
|
for item in &expected {
|
||||||
|
let current = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
|
||||||
|
if !current.matches_manifest(&item.baseline) {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let outage = census_object_version_on_disk(&replaced_drive, &bucket, outage_key, None)?;
|
||||||
|
Ok(outage.is_complete()
|
||||||
|
&& outage.data_blocks == Some(EC84_DATA_BLOCKS)
|
||||||
|
&& outage.parity_blocks == Some(EC84_PARITY_BLOCKS))
|
||||||
|
},
|
||||||
|
"EC8+4 replacement drive rebuilt baseline and outage shards",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let inventory = expected
|
||||||
|
.iter()
|
||||||
|
.map(|item| (item.key.clone(), item.body.clone()))
|
||||||
|
.chain(std::iter::once((outage_key.to_string(), outage_body.clone())))
|
||||||
|
.collect::<BTreeMap<_, _>>();
|
||||||
|
let expected_keys = inventory.keys().cloned().collect::<HashSet<_>>();
|
||||||
|
let mut node_listings = Vec::new();
|
||||||
|
for node_index in 0..dist.cluster.nodes.len() {
|
||||||
|
let client = dist.client(node_index)?;
|
||||||
|
assert_inventory(&client, &bucket, &inventory).await?;
|
||||||
|
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||||
|
let observed = listing
|
||||||
|
.contents()
|
||||||
|
.iter()
|
||||||
|
.filter_map(|object| object.key().map(str::to_owned))
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
assert_eq!(observed, expected_keys, "node {node_index} listing diverged after EC8+4 heal");
|
||||||
|
let mut observed = observed.into_iter().collect::<Vec<_>>();
|
||||||
|
observed.sort();
|
||||||
|
node_listings.push(observed);
|
||||||
|
}
|
||||||
|
if let Some(context) = evidence_context {
|
||||||
|
write_scanner_heal_evidence(
|
||||||
|
context,
|
||||||
|
ScannerHealEvidencePayload {
|
||||||
|
dist: &dist,
|
||||||
|
bucket: &bucket,
|
||||||
|
expected: &expected,
|
||||||
|
outage_key,
|
||||||
|
outage_body: &outage_body,
|
||||||
|
replaced_drive: &replaced_drive,
|
||||||
|
pid_before: target_pid_before,
|
||||||
|
pid_after: target_pid_after,
|
||||||
|
node_listings,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -25,6 +25,7 @@ mod data_integrity_movement_test;
|
|||||||
mod expand_decommission_rebalance_test;
|
mod expand_decommission_rebalance_test;
|
||||||
mod extra_test;
|
mod extra_test;
|
||||||
mod harness;
|
mod harness;
|
||||||
|
mod heal_test;
|
||||||
mod object_lock_test;
|
mod object_lock_test;
|
||||||
mod observability_test;
|
mod observability_test;
|
||||||
mod replication_quota_test;
|
mod replication_quota_test;
|
||||||
|
|||||||
@@ -18,8 +18,8 @@
|
|||||||
mod tests {
|
mod tests {
|
||||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
|
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
|
||||||
use crate::common::{
|
use crate::common::{
|
||||||
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
|
ClusterTopology, FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request,
|
||||||
rustfs_binary_path,
|
init_logging, rustfs_binary_path,
|
||||||
};
|
};
|
||||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
@@ -60,6 +60,29 @@ mod tests {
|
|||||||
oracle: &'static str,
|
oracle: &'static str,
|
||||||
evidence: &'static str,
|
evidence: &'static str,
|
||||||
unclean_shutdown_marker: bool,
|
unclean_shutdown_marker: bool,
|
||||||
|
topology: EvidenceTopology,
|
||||||
|
storage_class_standard: Option<&'static str>,
|
||||||
|
erasure_set_drive_count: Option<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct EvidenceTopology {
|
||||||
|
nodes: usize,
|
||||||
|
drives_per_node: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EvidenceTopology {
|
||||||
|
const fn new(nodes: usize, drives_per_node: usize) -> Self {
|
||||||
|
Self { nodes, drives_per_node }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn total_drives(self) -> usize {
|
||||||
|
self.nodes * self.drives_per_node
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cluster_topology(self) -> ClusterTopology {
|
||||||
|
ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||||
@@ -67,6 +90,9 @@ mod tests {
|
|||||||
oracle: "background-target-restart.json",
|
oracle: "background-target-restart.json",
|
||||||
evidence: "process-restart",
|
evidence: "process-restart",
|
||||||
unclean_shutdown_marker: false,
|
unclean_shutdown_marker: false,
|
||||||
|
topology: EvidenceTopology::new(4, 1),
|
||||||
|
storage_class_standard: None,
|
||||||
|
erasure_set_drive_count: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||||
@@ -74,6 +100,29 @@ mod tests {
|
|||||||
oracle: "background-target-crash.json",
|
oracle: "background-target-crash.json",
|
||||||
evidence: "process-crash-restart",
|
evidence: "process-crash-restart",
|
||||||
unclean_shutdown_marker: true,
|
unclean_shutdown_marker: true,
|
||||||
|
topology: EvidenceTopology::new(4, 1),
|
||||||
|
storage_class_standard: None,
|
||||||
|
erasure_set_drive_count: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
const BACKGROUND_TARGET_RESTART_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||||
|
id: "background-target-restart-ec8-4",
|
||||||
|
oracle: "background-target-restart-ec8-4.json",
|
||||||
|
evidence: "process-restart",
|
||||||
|
unclean_shutdown_marker: false,
|
||||||
|
topology: EvidenceTopology::new(3, 4),
|
||||||
|
storage_class_standard: Some("EC:4"),
|
||||||
|
erasure_set_drive_count: Some("12"),
|
||||||
|
};
|
||||||
|
|
||||||
|
const BACKGROUND_TARGET_CRASH_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||||
|
id: "background-target-crash-ec8-4",
|
||||||
|
oracle: "background-target-crash-ec8-4.json",
|
||||||
|
evidence: "process-crash-restart",
|
||||||
|
unclean_shutdown_marker: true,
|
||||||
|
topology: EvidenceTopology::new(3, 4),
|
||||||
|
storage_class_standard: Some("EC:4"),
|
||||||
|
erasure_set_drive_count: Some("12"),
|
||||||
};
|
};
|
||||||
|
|
||||||
struct RestartEvidenceContext {
|
struct RestartEvidenceContext {
|
||||||
@@ -332,11 +381,10 @@ mod tests {
|
|||||||
|
|
||||||
// Healing may rewrite non-identity bookkeeping in xl.meta. The census
|
// Healing may rewrite non-identity bookkeeping in xl.meta. The census
|
||||||
// therefore compares the canonical selected metadata fields plus every
|
// therefore compares the canonical selected metadata fields plus every
|
||||||
// physical shard, while the payload seed makes object mix-ups observable.
|
// physical shard.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct PhysicalObjectManifest {
|
struct PhysicalObjectManifest {
|
||||||
key: String,
|
key: String,
|
||||||
payload_seed: u8,
|
|
||||||
shard_census: VersionShardCensus,
|
shard_census: VersionShardCensus,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -973,6 +1021,26 @@ mod tests {
|
|||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart()
|
||||||
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
|
timeout(
|
||||||
|
Duration::from_secs(420),
|
||||||
|
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84),
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash()
|
||||||
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
|
timeout(
|
||||||
|
Duration::from_secs(420),
|
||||||
|
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrashEc84),
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||||
{
|
{
|
||||||
@@ -1010,29 +1078,38 @@ mod tests {
|
|||||||
IsolatedTargetRestart,
|
IsolatedTargetRestart,
|
||||||
BackgroundTargetRestart,
|
BackgroundTargetRestart,
|
||||||
BackgroundTargetCrash,
|
BackgroundTargetCrash,
|
||||||
|
BackgroundTargetRestartEc84,
|
||||||
|
BackgroundTargetCrashEc84,
|
||||||
BackgroundCoordinatorRestart,
|
BackgroundCoordinatorRestart,
|
||||||
TargetEndpointBlackhole,
|
TargetEndpointBlackhole,
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
let server_binary = rustfs_binary_path();
|
let server_binary = rustfs_binary_path();
|
||||||
let evidence_run = match scenario {
|
let evidence_case = match scenario {
|
||||||
InterruptionScenario::BackgroundTargetRestart => {
|
InterruptionScenario::BackgroundTargetRestart => Some(BACKGROUND_TARGET_RESTART_EVIDENCE),
|
||||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)?
|
InterruptionScenario::BackgroundTargetCrash => Some(BACKGROUND_TARGET_CRASH_EVIDENCE),
|
||||||
}
|
InterruptionScenario::BackgroundTargetRestartEc84 => Some(BACKGROUND_TARGET_RESTART_EC84_EVIDENCE),
|
||||||
InterruptionScenario::BackgroundTargetCrash => {
|
InterruptionScenario::BackgroundTargetCrashEc84 => Some(BACKGROUND_TARGET_CRASH_EC84_EVIDENCE),
|
||||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_CRASH_EVIDENCE)?
|
|
||||||
}
|
|
||||||
_ => None,
|
_ => None,
|
||||||
};
|
};
|
||||||
|
let evidence_run = match evidence_case {
|
||||||
|
Some(case) => restart_evidence_run(&server_binary, case)?,
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
let mut evidence_objects = Vec::new();
|
let mut evidence_objects = Vec::new();
|
||||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||||
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
||||||
InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"),
|
InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"),
|
||||||
|
InterruptionScenario::BackgroundTargetRestartEc84 => (true, 1, "background_target_restart_ec8_4"),
|
||||||
|
InterruptionScenario::BackgroundTargetCrashEc84 => (true, 1, "background_target_crash_ec8_4"),
|
||||||
InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
|
InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
|
||||||
InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
|
InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
|
||||||
};
|
};
|
||||||
|
let topology = evidence_case
|
||||||
|
.map(|case| case.topology)
|
||||||
|
.unwrap_or_else(|| EvidenceTopology::new(4, 1));
|
||||||
init_logging();
|
init_logging();
|
||||||
info!(
|
info!(
|
||||||
event = "heal_interruption_started",
|
event = "heal_interruption_started",
|
||||||
@@ -1044,9 +1121,15 @@ mod tests {
|
|||||||
"Starting root-heal interruption test"
|
"Starting root-heal interruption test"
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology.cluster_topology()).await?;
|
||||||
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
||||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||||
|
if let Some(storage_class) = evidence_case.and_then(|case| case.storage_class_standard) {
|
||||||
|
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", storage_class);
|
||||||
|
}
|
||||||
|
if let Some(erasure_set_drive_count) = evidence_case.and_then(|case| case.erasure_set_drive_count) {
|
||||||
|
cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count);
|
||||||
|
}
|
||||||
// Heal control uses the first lexicographically sorted grid host.
|
// Heal control uses the first lexicographically sorted grid host.
|
||||||
// Keep that coordinator distinct from the remote target at index 1.
|
// Keep that coordinator distinct from the remote target at index 1.
|
||||||
cluster.nodes.sort_by(|left, right| left.url.cmp(&right.url));
|
cluster.nodes.sort_by(|left, right| left.url.cmp(&right.url));
|
||||||
@@ -1095,11 +1178,22 @@ mod tests {
|
|||||||
.and_then(|value| value.parse::<usize>().ok())
|
.and_then(|value| value.parse::<usize>().ok())
|
||||||
.unwrap_or(4 * 1024 * 1024)
|
.unwrap_or(4 * 1024 * 1024)
|
||||||
.clamp(1024 * 1024, 16 * 1024 * 1024);
|
.clamp(1024 * 1024, 16 * 1024 * 1024);
|
||||||
|
let mut created_online_objects = Vec::with_capacity(online_object_count);
|
||||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||||
let mut unclean_shutdown_marker_observed = None;
|
let mut unclean_shutdown_marker_observed = None;
|
||||||
for index in 0..online_object_count {
|
let mut attempt_count = 0usize;
|
||||||
let key = format!("cluster/online/object-{index:04}.bin");
|
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
|
||||||
let payload_seed = u8::try_from(index + 1).expect("clamped object count must fit in u8");
|
while expected_manifests.len() < online_object_count {
|
||||||
|
if attempt_count >= max_online_attempts {
|
||||||
|
return Err(format!(
|
||||||
|
"target replacement drive held only {}/{} baseline object shards after {attempt_count} writes",
|
||||||
|
expected_manifests.len(),
|
||||||
|
online_object_count
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
let key = format!("cluster/online/object-{attempt_count:04}.bin");
|
||||||
|
let payload_seed = ((attempt_count % 251) + 1) as u8;
|
||||||
timeout(
|
timeout(
|
||||||
Duration::from_secs(30),
|
Duration::from_secs(30),
|
||||||
clients[0]
|
clients[0]
|
||||||
@@ -1111,6 +1205,11 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.await??;
|
.await??;
|
||||||
let shard_census = census_object_version_on_disk(&replaced_disk, bucket, &key, None)?;
|
let shard_census = census_object_version_on_disk(&replaced_disk, bucket, &key, None)?;
|
||||||
|
if !shard_census.has_xl_meta {
|
||||||
|
timeout(Duration::from_secs(30), clients[0].delete_object().bucket(bucket).key(&key).send()).await??;
|
||||||
|
attempt_count += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
assert!(
|
assert!(
|
||||||
shard_census.is_complete(),
|
shard_census.is_complete(),
|
||||||
"node 1 should hold a complete baseline shard for {key}: {shard_census:?}"
|
"node 1 should hold a complete baseline shard for {key}: {shard_census:?}"
|
||||||
@@ -1119,11 +1218,9 @@ mod tests {
|
|||||||
!shard_census.expected_part_numbers.is_empty(),
|
!shard_census.expected_part_numbers.is_empty(),
|
||||||
"chaos objects must use physical part shards rather than inline data: {shard_census:?}"
|
"chaos objects must use physical part shards rather than inline data: {shard_census:?}"
|
||||||
);
|
);
|
||||||
expected_manifests.push(PhysicalObjectManifest {
|
created_online_objects.push((key.clone(), payload_seed));
|
||||||
key,
|
expected_manifests.push(PhysicalObjectManifest { key, shard_census });
|
||||||
payload_seed,
|
attempt_count += 1;
|
||||||
shard_census,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let expected_pool_metadata = if background_enabled {
|
let expected_pool_metadata = if background_enabled {
|
||||||
@@ -1169,31 +1266,38 @@ mod tests {
|
|||||||
if node_index == 1 {
|
if node_index == 1 {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let census = census_object_version_on_disk(Path::new(&node.data_dir), bucket, outage_key, None)?;
|
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||||
|
let census = census_object_version_on_disk(Path::new(drive), bucket, outage_key, None)?;
|
||||||
assert!(
|
assert!(
|
||||||
census.is_complete(),
|
census.is_complete(),
|
||||||
"online node {node_index} must hold a complete outage-object shard: {census:?}"
|
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||||
);
|
);
|
||||||
let erasure_index = census
|
let erasure_index = census.erasure_index.ok_or_else(|| {
|
||||||
.erasure_index
|
format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
|
||||||
.ok_or_else(|| format!("online node {node_index} outage-object shard has no erasure index: {census:?}"))?;
|
})?;
|
||||||
assert!(
|
assert!(
|
||||||
(1..=cluster.nodes.len()).contains(&erasure_index),
|
(1..=topology.total_drives()).contains(&erasure_index),
|
||||||
"online node {node_index} outage-object erasure index is out of range: {census:?}"
|
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
outage_peer_erasure_indices.insert(erasure_index),
|
outage_peer_erasure_indices.insert(erasure_index),
|
||||||
"outage-object erasure index {erasure_index} is duplicated across online nodes"
|
"outage-object erasure index {erasure_index} is duplicated across online drives"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
outage_peer_erasure_indices.len(),
|
outage_peer_erasure_indices.len(),
|
||||||
cluster.nodes.len().saturating_sub(1),
|
topology.total_drives().saturating_sub(cluster.nodes[1].data_dirs.len()),
|
||||||
"every online node must contribute one unique outage-object erasure index"
|
"every online drive must contribute one unique outage-object erasure index"
|
||||||
|
);
|
||||||
|
let missing_outage_erasure_indices = (1..=topology.total_drives())
|
||||||
|
.filter(|index| !outage_peer_erasure_indices.contains(index))
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
missing_outage_erasure_indices.len(),
|
||||||
|
cluster.nodes[1].data_dirs.len(),
|
||||||
|
"the stopped node must account for every missing outage-object erasure index"
|
||||||
);
|
);
|
||||||
let expected_outage_target_erasure_index = (1..=cluster.nodes.len())
|
|
||||||
.find(|index| !outage_peer_erasure_indices.contains(index))
|
|
||||||
.ok_or("online outage-object shards leave no erasure index for the replacement target")?;
|
|
||||||
|
|
||||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||||
if !background_enabled {
|
if !background_enabled {
|
||||||
@@ -1463,7 +1567,10 @@ mod tests {
|
|||||||
"Restored target endpoint forwarding"
|
"Restored target endpoint forwarding"
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
if scenario == InterruptionScenario::BackgroundTargetRestart {
|
if matches!(
|
||||||
|
scenario,
|
||||||
|
InterruptionScenario::BackgroundTargetRestart | InterruptionScenario::BackgroundTargetRestartEc84
|
||||||
|
) {
|
||||||
cluster.stop_node_gracefully(interruption_node).await?;
|
cluster.stop_node_gracefully(interruption_node).await?;
|
||||||
} else {
|
} else {
|
||||||
cluster.stop_node(interruption_node)?;
|
cluster.stop_node(interruption_node)?;
|
||||||
@@ -1485,7 +1592,10 @@ mod tests {
|
|||||||
if background_enabled {
|
if background_enabled {
|
||||||
let marker_exists = unclean_shutdown_marker.is_file();
|
let marker_exists = unclean_shutdown_marker.is_file();
|
||||||
unclean_shutdown_marker_observed = Some(marker_exists);
|
unclean_shutdown_marker_observed = Some(marker_exists);
|
||||||
let expected_marker = !matches!(scenario, InterruptionScenario::BackgroundTargetRestart);
|
let expected_marker = matches!(
|
||||||
|
scenario,
|
||||||
|
InterruptionScenario::BackgroundTargetCrash | InterruptionScenario::BackgroundTargetCrashEc84
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
marker_exists == expected_marker,
|
marker_exists == expected_marker,
|
||||||
"background restart/crash lane observed unexpected unclean-shutdown marker state"
|
"background restart/crash lane observed unexpected unclean-shutdown marker state"
|
||||||
@@ -1586,29 +1696,35 @@ mod tests {
|
|||||||
"outage object must have a complete target shard: {outage_census:?}"
|
"outage object must have a complete target shard: {outage_census:?}"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
outage_census
|
||||||
|
.erasure_index
|
||||||
|
.filter(|index| missing_outage_erasure_indices.contains(index)),
|
||||||
outage_census.erasure_index,
|
outage_census.erasure_index,
|
||||||
Some(expected_outage_target_erasure_index),
|
"the outage object must be rebuilt into one of the stopped node's missing erasure slots"
|
||||||
"the outage object must be rebuilt into its own missing erasure slot"
|
|
||||||
);
|
);
|
||||||
|
|
||||||
if let Some(cycle_end) = scanner_cycle_floor {
|
if let Some(cycle_end) = scanner_cycle_floor {
|
||||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut expected_keys = expected_manifests
|
let mut expected_keys = created_online_objects
|
||||||
.iter()
|
.iter()
|
||||||
.map(|manifest| manifest.key.clone())
|
.map(|(key, _)| key.clone())
|
||||||
.collect::<HashSet<_>>();
|
.collect::<HashSet<_>>();
|
||||||
assert!(expected_keys.insert(outage_key.to_string()));
|
assert!(expected_keys.insert(outage_key.to_string()));
|
||||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||||
|
|
||||||
let target_client = cluster.create_s3_client(1)?;
|
let target_client = cluster.create_s3_client(1)?;
|
||||||
for expected in &expected_manifests {
|
for (key, payload_seed) in &created_online_objects {
|
||||||
let response = target_client.get_object().bucket(bucket).key(&expected.key).send().await?;
|
let response = target_client.get_object().bucket(bucket).key(key).send().await?;
|
||||||
let actual = response.body.collect().await?.into_bytes();
|
let actual = response.body.collect().await?.into_bytes();
|
||||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
let expected_body = deterministic_object_body(object_size_bytes, *payload_seed);
|
||||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {key}");
|
||||||
if evidence_run.is_some() {
|
if evidence_run.is_some()
|
||||||
|
&& let Some(expected) = expected_manifests
|
||||||
|
.iter()
|
||||||
|
.find(|manifest| manifest.key.as_str() == key.as_str())
|
||||||
|
{
|
||||||
evidence_objects.push(serde_json::json!({
|
evidence_objects.push(serde_json::json!({
|
||||||
"key": expected.key, "version_id": expected.shard_census.version_id,
|
"key": expected.key, "version_id": expected.shard_census.version_id,
|
||||||
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
|
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
|
||||||
|
|||||||
@@ -563,7 +563,7 @@ pub mod store_list {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub mod storage {
|
pub mod storage {
|
||||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
|
||||||
pub use crate::store::HealWalkVersion;
|
pub use crate::store::HealWalkVersion;
|
||||||
pub use crate::store::{
|
pub use crate::store::{
|
||||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||||
|
|||||||
@@ -8220,7 +8220,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
mod multipart_transport_tests {
|
mod multipart_transport_tests {
|
||||||
use super::super::super::replication_filemeta_boundary::ObjectPartInfo;
|
use super::super::super::replication_filemeta_boundary::ObjectPartInfo;
|
||||||
use super::super::super::replication_storage_boundary::ObjectIO as _;
|
use super::super::super::replication_storage_boundary::{ObjectIO as _, ReadPlan};
|
||||||
use super::*;
|
use super::*;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use http_body_util::{BodyExt, Full};
|
use http_body_util::{BodyExt, Full};
|
||||||
@@ -8264,8 +8264,7 @@ mod tests {
|
|||||||
} else {
|
} else {
|
||||||
self.full_reads.fetch_add(1, Ordering::Relaxed);
|
self.full_reads.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
let plan =
|
let plan = ReadPlan::build_for_request(range, &self.info, opts, &HeaderMap::new(), None).await?;
|
||||||
crate::object_api::ReadPlan::build_for_request(range, &self.info, opts, &HeaderMap::new(), None).await?;
|
|
||||||
let start = plan.storage_offset();
|
let start = plan.storage_offset();
|
||||||
let end = start + usize::try_from(plan.storage_length()).expect("nonnegative storage length");
|
let end = start + usize::try_from(plan.storage_length()).expect("nonnegative storage length");
|
||||||
return plan.into_object_reader(Box::new(std::io::Cursor::new(stored.slice(start..end))), &self.info);
|
return plan.into_object_reader(Box::new(std::io::Cursor::new(stored.slice(start..end))), &self.info);
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ pub(crate) use crate::object_api::{
|
|||||||
GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
|
GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
|
||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence};
|
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence, ReadPlan};
|
||||||
pub(crate) use crate::storage_api_contracts::list::{
|
pub(crate) use crate::storage_api_contracts::list::{
|
||||||
ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
|
ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -285,6 +285,20 @@ fn peer_replay_state(audience: &str) -> PeerReplayState {
|
|||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn clear_peer_replay_state_for_addr(addr: &str) -> std::io::Result<()> {
|
||||||
|
let uri = addr
|
||||||
|
.parse::<Uri>()
|
||||||
|
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
|
||||||
|
let audience = uri
|
||||||
|
.authority()
|
||||||
|
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
|
||||||
|
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))??;
|
||||||
|
if let Ok(mut states) = PEER_REPLAY_STATES.lock() {
|
||||||
|
states.remove(&audience);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn apply_peer_replay_response(
|
fn apply_peer_replay_response(
|
||||||
audience: String,
|
audience: String,
|
||||||
sent_state: PeerReplayState,
|
sent_state: PeerReplayState,
|
||||||
@@ -619,6 +633,13 @@ mod tests {
|
|||||||
.remove(audience);
|
.remove(audience);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn set_peer_capability(audience: &str, state: PeerReplayState) {
|
||||||
|
PEER_REPLAY_STATES
|
||||||
|
.lock()
|
||||||
|
.expect("peer capability cache lock must not be poisoned")
|
||||||
|
.insert(audience.to_string(), state);
|
||||||
|
}
|
||||||
|
|
||||||
fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> {
|
fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> {
|
||||||
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest {
|
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest {
|
||||||
args: "canonical mutation request".to_string(),
|
args: "canonical mutation request".to_string(),
|
||||||
@@ -1090,6 +1111,23 @@ mod tests {
|
|||||||
clear_peer_capability(audience);
|
clear_peer_capability(audience);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn clear_peer_replay_state_for_addr_removes_normalized_audience() {
|
||||||
|
let audience = "clear-peer-replay-state-test:9000";
|
||||||
|
let boot_epoch = Uuid::new_v4();
|
||||||
|
set_peer_capability(
|
||||||
|
audience,
|
||||||
|
PeerReplayState {
|
||||||
|
boot_epoch: Some(boot_epoch),
|
||||||
|
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
clear_peer_replay_state_for_addr("http://clear-peer-replay-state-test:9000").expect("peer URI should clear replay state");
|
||||||
|
|
||||||
|
assert_eq!(peer_replay_state(audience), PeerReplayState::default());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() {
|
fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() {
|
||||||
ensure_test_rpc_secret();
|
ensure_test_rpc_secret();
|
||||||
|
|||||||
@@ -13,8 +13,9 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::cluster::rpc::client::{
|
use crate::cluster::rpc::client::{
|
||||||
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
AuthenticatedChannel, TonicInterceptor, clear_peer_replay_state_for_addr, embedded_tonic_status,
|
||||||
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
gen_tonic_signature_interceptor, heal_control_time_out_client, is_network_like_status, message_has_network_needle,
|
||||||
|
node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||||
};
|
};
|
||||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
@@ -544,6 +545,16 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
|
|||||||
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
|
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn heal_control_auth_may_need_replay_scope_refresh(err: &Error) -> bool {
|
||||||
|
matches!(
|
||||||
|
err,
|
||||||
|
Error::Io(io_err)
|
||||||
|
if embedded_tonic_status(io_err).is_some_and(|status| {
|
||||||
|
status.code() == tonic::Code::Unauthenticated && status.message() == "No valid auth token"
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
|
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
|
||||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
|
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
|
||||||
if topology_member != expected_member {
|
if topology_member != expected_member {
|
||||||
@@ -1720,19 +1731,54 @@ impl PeerRestClient {
|
|||||||
return Err(Error::other("heal control command exceeds size limit"));
|
return Err(Error::other("heal control command exceeds size limit"));
|
||||||
}
|
}
|
||||||
let capability_probe = rustfs_protos::is_heal_control_capability_probe(&command);
|
let capability_probe = rustfs_protos::is_heal_control_capability_probe(&command);
|
||||||
self.finalize_result(
|
let result = self
|
||||||
async {
|
.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
|
||||||
|
.await;
|
||||||
|
if result
|
||||||
|
.as_ref()
|
||||||
|
.err()
|
||||||
|
.is_some_and(heal_control_auth_may_need_replay_scope_refresh)
|
||||||
|
{
|
||||||
|
self.prepare_heal_control_auth_retry().await;
|
||||||
|
return self
|
||||||
|
.finalize_result(
|
||||||
|
self.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
|
||||||
|
.await,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
self.finalize_result(result).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn prepare_heal_control_auth_retry(&self) {
|
||||||
|
if let Err(err) = clear_peer_replay_state_for_addr(&self.grid_host) {
|
||||||
|
debug!(
|
||||||
|
peer = %self.grid_host,
|
||||||
|
error = %err,
|
||||||
|
"could not clear heal control replay state before retry"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
self.evict_connection().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn heal_control_once(
|
||||||
|
&self,
|
||||||
|
version: u32,
|
||||||
|
topology_fingerprint: &str,
|
||||||
|
command: &[u8],
|
||||||
|
capability_probe: bool,
|
||||||
|
) -> Result<Vec<u8>> {
|
||||||
let mut client = self
|
let mut client = self
|
||||||
.get_heal_control_client()
|
.get_heal_control_client()
|
||||||
.await?
|
.await?
|
||||||
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
|
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
|
||||||
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
|
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
|
||||||
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, &topology_fingerprint, &command)
|
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, topology_fingerprint, command)
|
||||||
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
|
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
|
||||||
let mut request = Request::new(HealControlRequest {
|
let mut request = Request::new(HealControlRequest {
|
||||||
version,
|
version,
|
||||||
topology_fingerprint: topology_fingerprint.clone(),
|
topology_fingerprint: topology_fingerprint.to_string(),
|
||||||
command: command.clone().into(),
|
command: command.to_vec().into(),
|
||||||
});
|
});
|
||||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||||
@@ -1745,21 +1791,13 @@ impl PeerRestClient {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
if !capability_probe {
|
if !capability_probe {
|
||||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
let canonical_response =
|
||||||
version,
|
rustfs_protos::canonical_heal_control_response_body(version, topology_fingerprint, command, &response.result)
|
||||||
&topology_fingerprint,
|
|
||||||
&command,
|
|
||||||
&response.result,
|
|
||||||
)
|
|
||||||
.map_err(|_| Error::other("heal control response length cannot be represented"))?;
|
.map_err(|_| Error::other("heal control response length cannot be represented"))?;
|
||||||
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
|
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
|
||||||
}
|
}
|
||||||
Ok(response.result.to_vec())
|
Ok(response.result.to_vec())
|
||||||
}
|
}
|
||||||
.await,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Confirms that a peer supports the current heal-control coordination
|
/// Confirms that a peer supports the current heal-control coordination
|
||||||
/// contract and has the same storage
|
/// contract and has the same storage
|
||||||
@@ -3728,6 +3766,22 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn heal_control_auth_retry_is_limited_to_transport_auth_rejection() {
|
||||||
|
assert!(heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||||
|
tonic::Status::unauthenticated("No valid auth token")
|
||||||
|
)));
|
||||||
|
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||||
|
tonic::Status::permission_denied("bad signature")
|
||||||
|
)));
|
||||||
|
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||||
|
tonic::Status::unauthenticated("application rejected heal control")
|
||||||
|
)));
|
||||||
|
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::other(
|
||||||
|
"Io error: code: 'Unauthenticated', message: \"No valid auth token\""
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn peer_rest_client_network_classifier_keeps_slow_peers_online() {
|
fn peer_rest_client_network_classifier_keeps_slow_peers_online() {
|
||||||
// The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s)
|
// The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s)
|
||||||
|
|||||||
@@ -22463,6 +22463,86 @@ mod test {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn conditional_mrf_manifest_dir_fsync_failure_keeps_recovery_anchors() {
|
||||||
|
use tempfile::tempdir;
|
||||||
|
|
||||||
|
const MRF_COMMIT_MANIFEST_SLOT_0: &str = ".heal-mrf-commit.0.bin";
|
||||||
|
const MRF_COMMIT_MANIFEST_SLOT_1: &str = ".heal-mrf-commit.1.bin";
|
||||||
|
const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin";
|
||||||
|
|
||||||
|
let _mode = durability_mode_override::set(DurabilityMode::Relaxed);
|
||||||
|
let dir = tempdir().expect("temp dir should be created");
|
||||||
|
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||||
|
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||||
|
let previous_manifest = Bytes::from_static(b"mrf-committed-manifest-v1");
|
||||||
|
let successor_manifest = Bytes::from_static(b"mrf-committed-manifest-v2");
|
||||||
|
let legacy_journal = Bytes::from_static(b"legacy-mrf-journal-records");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
disk.compare_and_update_file(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0, None, Some(previous_manifest.clone()),)
|
||||||
|
.await
|
||||||
|
.expect("previous MRF manifest should commit"),
|
||||||
|
ConditionalFileUpdate::Updated
|
||||||
|
);
|
||||||
|
disk.write_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, legacy_journal.clone())
|
||||||
|
.await
|
||||||
|
.expect("legacy MRF journal should be retained");
|
||||||
|
|
||||||
|
let manifest_path = disk
|
||||||
|
.get_object_path(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0)
|
||||||
|
.expect("MRF manifest path should resolve");
|
||||||
|
let parent = manifest_path.parent().expect("MRF manifest path should have a parent");
|
||||||
|
assert!(
|
||||||
|
os::fsync_dir_recorder::was_fsynced(parent),
|
||||||
|
"system metadata MRF manifest publication must fsync the metadata directory even under relaxed durability"
|
||||||
|
);
|
||||||
|
os::fsync_dir_recorder::set_failure(parent, ErrorKind::Other);
|
||||||
|
|
||||||
|
let err = disk
|
||||||
|
.compare_and_update_file(
|
||||||
|
RUSTFS_META_BUCKET,
|
||||||
|
MRF_COMMIT_MANIFEST_SLOT_0,
|
||||||
|
Some(previous_manifest.clone()),
|
||||||
|
Some(successor_manifest),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("directory fsync failure must fail the MRF manifest successor commit");
|
||||||
|
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::Other));
|
||||||
|
assert_eq!(
|
||||||
|
disk.read_all(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0)
|
||||||
|
.await
|
||||||
|
.expect("previous committed MRF manifest should remain readable after rollback"),
|
||||||
|
previous_manifest
|
||||||
|
);
|
||||||
|
|
||||||
|
os::fsync_dir_recorder::set_failure(parent, ErrorKind::Other);
|
||||||
|
let err = disk
|
||||||
|
.compare_and_update_file(
|
||||||
|
RUSTFS_META_BUCKET,
|
||||||
|
MRF_COMMIT_MANIFEST_SLOT_1,
|
||||||
|
None,
|
||||||
|
Some(Bytes::from_static(b"first-successor-manifest")),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("directory fsync failure must fail first MRF manifest commit");
|
||||||
|
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::Other));
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
disk.read_all(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_1).await,
|
||||||
|
Err(DiskError::FileNotFound)
|
||||||
|
),
|
||||||
|
"uncommitted first MRF manifest must be removed when no committed anchor exists"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
disk.read_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
|
||||||
|
.await
|
||||||
|
.expect("legacy MRF journal should remain readable after failed manifest publication"),
|
||||||
|
legacy_journal
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn conditional_file_update_dir_fsync_failure_removes_new_file_without_anchor() {
|
async fn conditional_file_update_dir_fsync_failure_removes_new_file_without_anchor() {
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ use std::time::{Duration, UNIX_EPOCH};
|
|||||||
use tokio::sync::{RwLock, Semaphore};
|
use tokio::sync::{RwLock, Semaphore};
|
||||||
use tracing::{debug, error, warn};
|
use tracing::{debug, error, warn};
|
||||||
|
|
||||||
use super::{DiskStore, EcstoreError};
|
use super::{DiskStore, EcstoreError, POOL_META_NAME, RUSTFS_META_BUCKET};
|
||||||
|
|
||||||
/// Outcome of classifying an error returned by [`HealStorageAPI::heal_object`].
|
/// Outcome of classifying an error returned by [`HealStorageAPI::heal_object`].
|
||||||
enum HealObjectOutcome {
|
enum HealObjectOutcome {
|
||||||
@@ -68,6 +68,13 @@ struct PageConcurrencyGuard {
|
|||||||
set_label: String,
|
set_label: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct ErasureSetPassCounters<'a> {
|
||||||
|
processed_objects: &'a mut u64,
|
||||||
|
successful_objects: &'a mut u64,
|
||||||
|
failed_objects: &'a mut u64,
|
||||||
|
skipped_objects: &'a mut u64,
|
||||||
|
}
|
||||||
|
|
||||||
impl PageConcurrencyGuard {
|
impl PageConcurrencyGuard {
|
||||||
fn new(in_flight: Arc<AtomicUsize>, set_label: String) -> Self {
|
fn new(in_flight: Arc<AtomicUsize>, set_label: String) -> Self {
|
||||||
let current = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
|
let current = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
|
||||||
@@ -106,6 +113,7 @@ pub struct ErasureSetHealer {
|
|||||||
heal_opts: HealOpts,
|
heal_opts: HealOpts,
|
||||||
source: HealRequestSource,
|
source: HealRequestSource,
|
||||||
target_endpoints: Arc<[String]>,
|
target_endpoints: Arc<[String]>,
|
||||||
|
pool_metadata_target_endpoints: Arc<[String]>,
|
||||||
replacement_task_id: Option<String>,
|
replacement_task_id: Option<String>,
|
||||||
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
|
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
|
||||||
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
||||||
@@ -355,6 +363,7 @@ impl ErasureSetHealer {
|
|||||||
heal_opts,
|
heal_opts,
|
||||||
source,
|
source,
|
||||||
target_endpoints: Vec::new().into(),
|
target_endpoints: Vec::new().into(),
|
||||||
|
pool_metadata_target_endpoints: Vec::new().into(),
|
||||||
replacement_task_id: None,
|
replacement_task_id: None,
|
||||||
replacement_target_identities: None,
|
replacement_target_identities: None,
|
||||||
mainline_pacer: None,
|
mainline_pacer: None,
|
||||||
@@ -378,6 +387,13 @@ impl ErasureSetHealer {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_pool_metadata_targets(mut self, mut target_endpoints: Vec<String>) -> Self {
|
||||||
|
target_endpoints.sort_unstable();
|
||||||
|
target_endpoints.dedup();
|
||||||
|
self.pool_metadata_target_endpoints = target_endpoints.into();
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn with_replacement_identity_fence(
|
pub(crate) fn with_replacement_identity_fence(
|
||||||
mut self,
|
mut self,
|
||||||
replacement_target_identities: Option<Vec<ReplacementTargetIdentity>>,
|
replacement_target_identities: Option<Vec<ReplacementTargetIdentity>>,
|
||||||
@@ -834,6 +850,21 @@ impl ErasureSetHealer {
|
|||||||
current_object_index = 0;
|
current_object_index = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if failed_objects == 0 && skipped_objects == 0 && failed_buckets == 0 {
|
||||||
|
self.heal_replacement_pool_metadata(
|
||||||
|
set_disk_id,
|
||||||
|
&mut ErasureSetPassCounters {
|
||||||
|
processed_objects: &mut processed_objects,
|
||||||
|
successful_objects: &mut successful_objects,
|
||||||
|
failed_objects: &mut failed_objects,
|
||||||
|
skipped_objects: &mut skipped_objects,
|
||||||
|
},
|
||||||
|
resume_manager,
|
||||||
|
checkpoint_manager,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
// 5. finalize. Only declare the set healed when nothing failed AND
|
// 5. finalize. Only declare the set healed when nothing failed AND
|
||||||
// nothing was transiently skipped — otherwise the resume/checkpoint
|
// nothing was transiently skipped — otherwise the resume/checkpoint
|
||||||
// state must survive so the failed/skipped versions are retried instead
|
// state must survive so the failed/skipped versions are retried instead
|
||||||
@@ -919,6 +950,207 @@ impl ErasureSetHealer {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn heal_replacement_pool_metadata(
|
||||||
|
&self,
|
||||||
|
set_disk_id: &str,
|
||||||
|
counters: &mut ErasureSetPassCounters<'_>,
|
||||||
|
resume_manager: &ResumeManager,
|
||||||
|
checkpoint_manager: &CheckpointManager,
|
||||||
|
) -> Result<()> {
|
||||||
|
let target_endpoints = if self.pool_metadata_target_endpoints.is_empty() {
|
||||||
|
self.target_endpoints.as_ref()
|
||||||
|
} else {
|
||||||
|
self.pool_metadata_target_endpoints.as_ref()
|
||||||
|
};
|
||||||
|
let target_scoped_recreate = !self.heal_opts.dry_run && self.heal_opts.recreate && !target_endpoints.is_empty();
|
||||||
|
if self.replacement_task_id.is_none() && !target_scoped_recreate {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
if target_endpoints.is_empty() {
|
||||||
|
return Err(Error::TaskExecutionFailed {
|
||||||
|
message: "Replacement pool metadata heal requires target endpoints".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
|
||||||
|
let checkpoint_key = compose_key(&object_key, None);
|
||||||
|
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||||
|
if checkpoint.processed_objects.contains(&checkpoint_key)
|
||||||
|
|| checkpoint.failed_objects.contains(&checkpoint_key)
|
||||||
|
|| checkpoint.skipped_objects.contains(&checkpoint_key)
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
drop(checkpoint);
|
||||||
|
|
||||||
|
self.verify_replacement_identity_fence("pool metadata").await?;
|
||||||
|
resume_manager
|
||||||
|
.set_current_item(Some(RUSTFS_META_BUCKET.to_string()), Some(POOL_META_NAME.to_string()))
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let result = match self
|
||||||
|
.storage
|
||||||
|
.heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, None, &self.heal_opts)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok((result, None)) if target_outcomes_complete(&result, target_endpoints) => {
|
||||||
|
let object_size = result_object_size_u64(&result);
|
||||||
|
match self
|
||||||
|
.storage
|
||||||
|
.replacement_targets_have_version(RUSTFS_META_BUCKET, POOL_META_NAME, None, &self.heal_opts, target_endpoints)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(true) => (object_size, Ok(())),
|
||||||
|
Ok(false) => (
|
||||||
|
object_size,
|
||||||
|
Err(Error::transient_skip(
|
||||||
|
"Skipped replacement pool metadata heal because target readback did not confirm the committed version",
|
||||||
|
)),
|
||||||
|
),
|
||||||
|
Err(err) => (
|
||||||
|
object_size,
|
||||||
|
Err(Error::transient_skip(format!(
|
||||||
|
"Skipped replacement pool metadata heal because target readback failed: {err}"
|
||||||
|
))),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok((result, None)) => (
|
||||||
|
result_object_size_u64(&result),
|
||||||
|
Err(Error::transient_skip(
|
||||||
|
"Skipped replacement pool metadata heal because a replacement target was not committed",
|
||||||
|
)),
|
||||||
|
),
|
||||||
|
Ok((result, Some(err))) => {
|
||||||
|
let object_size = result_object_size_u64(&result);
|
||||||
|
match Self::classify_heal_object_error(&err) {
|
||||||
|
HealObjectOutcome::Absent | HealObjectOutcome::Transient => (
|
||||||
|
object_size,
|
||||||
|
Err(Error::transient_skip(format!(
|
||||||
|
"Skipped replacement pool metadata heal due to transient error: {err}"
|
||||||
|
))),
|
||||||
|
),
|
||||||
|
HealObjectOutcome::Failed => (object_size, Err(err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||||
|
Err(err) => match Self::classify_heal_object_error(&err) {
|
||||||
|
HealObjectOutcome::Absent | HealObjectOutcome::Transient => (
|
||||||
|
0,
|
||||||
|
Err(Error::transient_skip(format!(
|
||||||
|
"Skipped replacement pool metadata heal due to transient error: {err}"
|
||||||
|
))),
|
||||||
|
),
|
||||||
|
HealObjectOutcome::Failed => (0, Err(err)),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let (object_size, result) = result;
|
||||||
|
let mut bytes_processed = self.progress.read().await.bytes_processed;
|
||||||
|
let mut telemetry_unknown = false;
|
||||||
|
let checkpoint_outcome = match result {
|
||||||
|
Ok(()) => {
|
||||||
|
telemetry_unknown |= !increment_counter(counters.successful_objects);
|
||||||
|
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||||
|
debug!(
|
||||||
|
target: "rustfs::heal::erasure_healer",
|
||||||
|
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||||
|
component = LOG_COMPONENT_HEAL,
|
||||||
|
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||||
|
set_disk_id,
|
||||||
|
bucket = RUSTFS_META_BUCKET,
|
||||||
|
object = POOL_META_NAME,
|
||||||
|
state = "healed",
|
||||||
|
"Replacement pool metadata healed"
|
||||||
|
);
|
||||||
|
CheckpointObjectOutcome::Processed
|
||||||
|
}
|
||||||
|
Err(Error::TransientSkip { message }) => {
|
||||||
|
telemetry_unknown |= !increment_counter(counters.skipped_objects);
|
||||||
|
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||||
|
warn!(
|
||||||
|
target: "rustfs::heal::erasure_healer",
|
||||||
|
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||||
|
component = LOG_COMPONENT_HEAL,
|
||||||
|
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||||
|
set_disk_id,
|
||||||
|
bucket = RUSTFS_META_BUCKET,
|
||||||
|
object = POOL_META_NAME,
|
||||||
|
state = "transient_skip",
|
||||||
|
error = %message,
|
||||||
|
"Replacement pool metadata heal skipped due to transient error"
|
||||||
|
);
|
||||||
|
CheckpointObjectOutcome::Skipped
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
telemetry_unknown |= !increment_counter(counters.failed_objects);
|
||||||
|
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||||
|
warn!(
|
||||||
|
target: "rustfs::heal::erasure_healer",
|
||||||
|
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||||
|
component = LOG_COMPONENT_HEAL,
|
||||||
|
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||||
|
set_disk_id,
|
||||||
|
bucket = RUSTFS_META_BUCKET,
|
||||||
|
object = POOL_META_NAME,
|
||||||
|
state = "failed",
|
||||||
|
error = %err,
|
||||||
|
"Replacement pool metadata heal failed"
|
||||||
|
);
|
||||||
|
CheckpointObjectOutcome::Failed
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
telemetry_unknown |= !increment_counter(counters.processed_objects);
|
||||||
|
let (outcome_record, counter_unknown, skipped_new_versions, skipped_ilm_expired) = {
|
||||||
|
let mut progress = self.progress.write().await;
|
||||||
|
progress.set_current_object(Some(object_key.clone()));
|
||||||
|
progress.update_object_progress(
|
||||||
|
*counters.processed_objects,
|
||||||
|
*counters.successful_objects,
|
||||||
|
*counters.failed_objects,
|
||||||
|
*counters.skipped_objects,
|
||||||
|
bytes_processed,
|
||||||
|
);
|
||||||
|
if telemetry_unknown {
|
||||||
|
progress.mark_unknown();
|
||||||
|
}
|
||||||
|
(
|
||||||
|
CheckpointObjectOutcomeRecord {
|
||||||
|
object: checkpoint_key,
|
||||||
|
outcome: checkpoint_outcome,
|
||||||
|
successful: progress.objects_healed,
|
||||||
|
failed: progress.objects_failed,
|
||||||
|
skipped: progress.skipped_objects,
|
||||||
|
bytes: progress.bytes_processed,
|
||||||
|
skipped_new_versions: progress.skipped_new_versions,
|
||||||
|
skipped_ilm_expired: progress.skipped_ilm_expired,
|
||||||
|
counter_unknown: progress.counter_unknown,
|
||||||
|
},
|
||||||
|
progress.counter_unknown,
|
||||||
|
progress.skipped_new_versions,
|
||||||
|
progress.skipped_ilm_expired,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
checkpoint_manager.record_object_outcome(outcome_record).await?;
|
||||||
|
resume_manager
|
||||||
|
.update_progress_with_bytes(
|
||||||
|
*counters.processed_objects,
|
||||||
|
*counters.successful_objects,
|
||||||
|
*counters.failed_objects,
|
||||||
|
*counters.skipped_objects,
|
||||||
|
bytes_processed,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
resume_manager
|
||||||
|
.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired)
|
||||||
|
.await?;
|
||||||
|
if counter_unknown {
|
||||||
|
resume_manager.mark_counter_unknown().await?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// heal single bucket with resume
|
/// heal single bucket with resume
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
|
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
|
||||||
@@ -1679,7 +1911,8 @@ mod resume_loop_tests {
|
|||||||
use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||||
use crate::heal::storage_api::status::BucketInfo;
|
use crate::heal::storage_api::status::BucketInfo;
|
||||||
use crate::heal::{
|
use crate::heal::{
|
||||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk,
|
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, POOL_META_NAME, RUSTFS_META_BUCKET,
|
||||||
|
new_disk,
|
||||||
};
|
};
|
||||||
use crate::{Error, Result};
|
use crate::{Error, Result};
|
||||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource};
|
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource};
|
||||||
@@ -1773,6 +2006,28 @@ mod resume_loop_tests {
|
|||||||
assert!(!target_outcomes_complete(&duplicate, &["replacement-a".to_string()]));
|
assert!(!target_outcomes_complete(&duplicate, &["replacement-a".to_string()]));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn replacement_target_ok_result(endpoint: &str, object: &str) -> HealResultItem {
|
||||||
|
HealResultItem {
|
||||||
|
object: object.to_string(),
|
||||||
|
object_size: 1024,
|
||||||
|
before: Infos {
|
||||||
|
drives: vec![HealDriveInfo {
|
||||||
|
endpoint: endpoint.to_string(),
|
||||||
|
state: "missing".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
after: Infos {
|
||||||
|
drives: vec![HealDriveInfo {
|
||||||
|
endpoint: endpoint.to_string(),
|
||||||
|
state: "ok".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct Page {
|
struct Page {
|
||||||
items: Vec<HealListItem>,
|
items: Vec<HealListItem>,
|
||||||
@@ -2444,6 +2699,8 @@ mod resume_loop_tests {
|
|||||||
HealRequestSource::AutoHeal,
|
HealRequestSource::AutoHeal,
|
||||||
)
|
)
|
||||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||||
|
env.storage
|
||||||
|
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||||
|
|
||||||
healer
|
healer
|
||||||
.heal_erasure_set(&["b".to_string()], "pool_0_set_0")
|
.heal_erasure_set(&["b".to_string()], "pool_0_set_0")
|
||||||
@@ -2461,9 +2718,121 @@ mod resume_loop_tests {
|
|||||||
CheckpointManager::has_checkpoint(&env.healer.disk, &replacement_task_id).await,
|
CheckpointManager::has_checkpoint(&env.healer.disk, &replacement_task_id).await,
|
||||||
"the checkpoint must survive until the caller clears the healing marker"
|
"the checkpoint must survive until the caller clears the healing marker"
|
||||||
);
|
);
|
||||||
|
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||||
drop(checkpoint);
|
drop(checkpoint);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn replacement_pool_metadata_readback_failure_schedules_retry() {
|
||||||
|
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||||
|
let replacement_task_id = ResumeUtils::generate_task_id();
|
||||||
|
ResumeManager::new_replacement_intent(
|
||||||
|
env.healer.disk.clone(),
|
||||||
|
replacement_task_id.clone(),
|
||||||
|
"pool_0_set_0".to_string(),
|
||||||
|
vec!["b".to_string()],
|
||||||
|
vec!["replacement-a".to_string()],
|
||||||
|
vec![crate::heal::resume::ReplacementTargetIdentity {
|
||||||
|
endpoint: "replacement-a".to_string(),
|
||||||
|
canonical_path: "/mnt/replacement-a".to_string(),
|
||||||
|
physical_device_ids: vec!["device-a".to_string()],
|
||||||
|
filesystem_identity: "1:2:3".to_string(),
|
||||||
|
}],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("replacement intent should persist");
|
||||||
|
env.storage
|
||||||
|
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||||
|
env.storage.set_replacement_commit_evidence(POOL_META_NAME, None, false);
|
||||||
|
let healer = ErasureSetHealer::new(
|
||||||
|
env.storage.clone(),
|
||||||
|
Arc::new(RwLock::new(HealProgress::new())),
|
||||||
|
CancellationToken::new(),
|
||||||
|
env.healer.disk.clone(),
|
||||||
|
HealOpts::default(),
|
||||||
|
HealRequestSource::AutoHeal,
|
||||||
|
)
|
||||||
|
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||||
|
|
||||||
|
let error = healer
|
||||||
|
.heal_erasure_set(&["b".to_string()], "pool_0_set_0")
|
||||||
|
.await
|
||||||
|
.expect_err("unconfirmed pool metadata readback must keep the replacement incomplete");
|
||||||
|
|
||||||
|
assert!(error.to_string().contains("Replacement erasure set heal incomplete"));
|
||||||
|
let state = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id)
|
||||||
|
.await
|
||||||
|
.expect("replacement retry state must remain")
|
||||||
|
.get_state()
|
||||||
|
.await;
|
||||||
|
assert!(!state.completed);
|
||||||
|
assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent);
|
||||||
|
assert_eq!(state.retry_count, 1);
|
||||||
|
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn admin_recreate_target_heals_pool_metadata_before_completion() {
|
||||||
|
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||||
|
let healer = ErasureSetHealer::new(
|
||||||
|
env.storage.clone(),
|
||||||
|
Arc::new(RwLock::new(HealProgress::new())),
|
||||||
|
CancellationToken::new(),
|
||||||
|
env.healer.disk.clone(),
|
||||||
|
HealOpts {
|
||||||
|
recreate: true,
|
||||||
|
pool: Some(0),
|
||||||
|
set: Some(0),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
HealRequestSource::Admin,
|
||||||
|
)
|
||||||
|
.with_pool_metadata_targets(vec!["replacement-a".to_string()]);
|
||||||
|
env.storage
|
||||||
|
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||||
|
|
||||||
|
healer
|
||||||
|
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||||
|
.await
|
||||||
|
.expect("admin recreate should heal and verify pool metadata");
|
||||||
|
|
||||||
|
assert!(env.resume.get_state().await.completed);
|
||||||
|
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn admin_recreate_pool_metadata_readback_failure_keeps_resume_state() {
|
||||||
|
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||||
|
let healer = ErasureSetHealer::new(
|
||||||
|
env.storage.clone(),
|
||||||
|
Arc::new(RwLock::new(HealProgress::new())),
|
||||||
|
CancellationToken::new(),
|
||||||
|
env.healer.disk.clone(),
|
||||||
|
HealOpts {
|
||||||
|
recreate: true,
|
||||||
|
pool: Some(0),
|
||||||
|
set: Some(0),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
HealRequestSource::Admin,
|
||||||
|
)
|
||||||
|
.with_pool_metadata_targets(vec!["replacement-a".to_string()]);
|
||||||
|
env.storage
|
||||||
|
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||||
|
env.storage.set_replacement_commit_evidence(POOL_META_NAME, None, false);
|
||||||
|
|
||||||
|
let error = healer
|
||||||
|
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||||
|
.await
|
||||||
|
.expect_err("unconfirmed admin recreate pool metadata must keep the set incomplete");
|
||||||
|
|
||||||
|
assert!(error.to_string().contains("Erasure set heal incomplete"));
|
||||||
|
let state = env.resume.get_state().await;
|
||||||
|
assert!(!state.completed);
|
||||||
|
assert_eq!(state.retry_count, 1);
|
||||||
|
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn retry_exhaustion_keeps_resume_artifacts_for_recovery() {
|
async fn retry_exhaustion_keeps_resume_artifacts_for_recovery() {
|
||||||
let env = make_env().await;
|
let env = make_env().await;
|
||||||
|
|||||||
@@ -506,6 +506,25 @@ fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key:
|
|||||||
.map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
|
.map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn request_matches_task(request: &HealRequest, task: &HealTask) -> bool {
|
||||||
|
request.heal_type == task.heal_type
|
||||||
|
&& request.options == task.options
|
||||||
|
&& request.priority == task.priority
|
||||||
|
&& request.source == task.source
|
||||||
|
&& request.retry_attempts == task.retry_attempts
|
||||||
|
&& request.heal_endpoints == task.heal_endpoints
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request_matches_request(request: &HealRequest, existing: &HealRequest) -> bool {
|
||||||
|
request.heal_type == existing.heal_type
|
||||||
|
&& request.options == existing.options
|
||||||
|
&& request.priority == existing.priority
|
||||||
|
&& request.source == existing.source
|
||||||
|
&& request.force_start == existing.force_start
|
||||||
|
&& request.retry_attempts == existing.retry_attempts
|
||||||
|
&& request.heal_endpoints == existing.heal_endpoints
|
||||||
|
}
|
||||||
|
|
||||||
fn retrying_heal_for_dedup_key(retrying_heals: &HashMap<String, RetryingHeal>, key: &str) -> Option<(String, HealType)> {
|
fn retrying_heal_for_dedup_key(retrying_heals: &HashMap<String, RetryingHeal>, key: &str) -> Option<(String, HealType)> {
|
||||||
retrying_heals
|
retrying_heals
|
||||||
.iter()
|
.iter()
|
||||||
@@ -1508,7 +1527,7 @@ impl HealManager {
|
|||||||
request: HealRequest,
|
request: HealRequest,
|
||||||
preserve_alias: bool,
|
preserve_alias: bool,
|
||||||
) -> Result<HealAdmissionReceipt> {
|
) -> Result<HealAdmissionReceipt> {
|
||||||
self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, preserve_alias, None)
|
self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, preserve_alias, true, None)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1544,14 +1563,27 @@ impl HealManager {
|
|||||||
request: HealRequest,
|
request: HealRequest,
|
||||||
mrf_notice_target: MrfRepairNoticeTarget,
|
mrf_notice_target: MrfRepairNoticeTarget,
|
||||||
) -> Result<HealAdmissionReceipt> {
|
) -> Result<HealAdmissionReceipt> {
|
||||||
self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, Some(mrf_notice_target))
|
self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, true, Some(mrf_notice_target))
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn durable_mrf_repair_anchor(
|
||||||
|
&self,
|
||||||
|
intent: &rustfs_common::mrf_channel::MrfIntent,
|
||||||
|
) -> Option<rustfs_common::mrf_channel::MrfDurableRepairAnchor> {
|
||||||
|
match self.storage.mrf_bucket_incarnation_id(intent.bucket.as_ref()).await {
|
||||||
|
Ok(Some(bucket_incarnation_id)) => {
|
||||||
|
rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(intent, bucket_incarnation_id)
|
||||||
|
}
|
||||||
|
Ok(None) | Err(_) => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn submit_heal_request_with_receipt_alias_and_mrf_notice(
|
async fn submit_heal_request_with_receipt_alias_and_mrf_notice(
|
||||||
&self,
|
&self,
|
||||||
request: HealRequest,
|
request: HealRequest,
|
||||||
preserve_alias: bool,
|
preserve_alias: bool,
|
||||||
|
accept_same_request_id_replay: bool,
|
||||||
mrf_notice_target: Option<MrfRepairNoticeTarget>,
|
mrf_notice_target: Option<MrfRepairNoticeTarget>,
|
||||||
) -> Result<HealAdmissionReceipt> {
|
) -> Result<HealAdmissionReceipt> {
|
||||||
let admission_start = Instant::now();
|
let admission_start = Instant::now();
|
||||||
@@ -1613,6 +1645,68 @@ impl HealManager {
|
|||||||
pause_duplicate_admission_after_active_lock(&request.id).await;
|
pause_duplicate_admission_after_active_lock(&request.id).await;
|
||||||
let mut queue = self.heal_queue.lock().await;
|
let mut queue = self.heal_queue.lock().await;
|
||||||
let retrying_heals = self.retrying_heals.lock().await;
|
let retrying_heals = self.retrying_heals.lock().await;
|
||||||
|
|
||||||
|
let request_id_admission = active_heals
|
||||||
|
.get(&request.id)
|
||||||
|
.map(|task| (request_matches_task(&request, task), "active"))
|
||||||
|
.or_else(|| {
|
||||||
|
queue
|
||||||
|
.requests()
|
||||||
|
.find(|queued| queued.id == request.id)
|
||||||
|
.map(|queued| (request_matches_request(&request, queued), "queued"))
|
||||||
|
})
|
||||||
|
.or_else(|| {
|
||||||
|
retrying_heals
|
||||||
|
.get(&request.id)
|
||||||
|
.map(|retrying| (request_matches_request(&request, &retrying.request), "retrying"))
|
||||||
|
});
|
||||||
|
if let Some((matches_existing, duplicate_state)) = request_id_admission {
|
||||||
|
let admission = if matches_existing {
|
||||||
|
if accept_same_request_id_replay {
|
||||||
|
HealAdmissionResult::Accepted
|
||||||
|
} else {
|
||||||
|
Self::duplicate_admission_for_request(&request, &config)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||||
|
};
|
||||||
|
if matches!(admission, HealAdmissionResult::Accepted | HealAdmissionResult::Merged)
|
||||||
|
&& let Some(target) = mrf_notice_target
|
||||||
|
{
|
||||||
|
let mut targets = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets);
|
||||||
|
Self::insert_mrf_repair_notice_target(&mut targets, &request.id, target);
|
||||||
|
}
|
||||||
|
drop(retrying_heals);
|
||||||
|
drop(queue);
|
||||||
|
drop(active_heals);
|
||||||
|
let lock_phase = lock_phase_start.elapsed();
|
||||||
|
Self::record_admission_metric(request.source, admission, "duplicate");
|
||||||
|
self.record_admission_observation(HealAdmissionObservation {
|
||||||
|
source,
|
||||||
|
result: admission,
|
||||||
|
context: "duplicate",
|
||||||
|
force_start,
|
||||||
|
displaced: false,
|
||||||
|
start_duration: admission_start.elapsed(),
|
||||||
|
lock_phase,
|
||||||
|
});
|
||||||
|
debug!(
|
||||||
|
target: "rustfs::heal::manager",
|
||||||
|
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||||
|
component = LOG_COMPONENT_HEAL,
|
||||||
|
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||||
|
request_id = %request.id,
|
||||||
|
duplicate_state,
|
||||||
|
result = admission.result_label(),
|
||||||
|
reason = admission.reason_label(),
|
||||||
|
"Heal queue admission reused an existing request id"
|
||||||
|
);
|
||||||
|
return Ok(HealAdmissionReceipt {
|
||||||
|
result: admission,
|
||||||
|
task_id: request.id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
let duplicate = (!request.force_start).then(|| {
|
let duplicate = (!request.force_start).then(|| {
|
||||||
active_heal_for_dedup_key(&active_heals, &dedup_key)
|
active_heal_for_dedup_key(&active_heals, &dedup_key)
|
||||||
.map(|(task_id, _)| (task_id, "active"))
|
.map(|(task_id, _)| (task_id, "active"))
|
||||||
@@ -1819,7 +1913,10 @@ impl HealManager {
|
|||||||
|
|
||||||
/// Submit heal request.
|
/// Submit heal request.
|
||||||
pub async fn submit_heal_request(&self, request: HealRequest) -> Result<HealAdmissionResult> {
|
pub async fn submit_heal_request(&self, request: HealRequest) -> Result<HealAdmissionResult> {
|
||||||
Ok(self.submit_heal_request_with_receipt_and_alias(request, true).await?.result)
|
Ok(self
|
||||||
|
.submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, false, None)
|
||||||
|
.await?
|
||||||
|
.result)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get task status
|
/// Get task status
|
||||||
|
|||||||
@@ -733,23 +733,30 @@ pub(super) fn mrf_verified_repair_event_for_target(
|
|||||||
HealObjectDisposition::AuthoritativelyAbsent => MrfVerifiedRepairDisposition::AuthoritativelyAbsent,
|
HealObjectDisposition::AuthoritativelyAbsent => MrfVerifiedRepairDisposition::AuthoritativelyAbsent,
|
||||||
_ => return None,
|
_ => return None,
|
||||||
};
|
};
|
||||||
if target.kind != MrfKind::PartialWrite {
|
let expected_kind = match target.kind {
|
||||||
return None;
|
MrfKind::DecodeFailure => HealObjectKind::Decode,
|
||||||
}
|
MrfKind::MetadataCorruption => HealObjectKind::Metadata,
|
||||||
let expected_kind = HealObjectKind::Object;
|
MrfKind::PartialWrite => HealObjectKind::Object,
|
||||||
|
};
|
||||||
if outcome.identity.kind != expected_kind
|
if outcome.identity.kind != expected_kind
|
||||||
|| outcome.identity.bucket.as_str() != target.bucket.as_ref()
|
|| outcome.identity.bucket.as_str() != target.bucket.as_ref()
|
||||||
|| outcome.identity.object.as_str() != target.object.as_ref()
|
|| outcome.identity.object.as_str() != target.object.as_ref()
|
||||||
{
|
{
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let version_id = target.version_id.filter(|bytes| *bytes != [0; 16]);
|
let version_id = (!matches!(target.kind, MrfKind::MetadataCorruption))
|
||||||
|
.then_some(target.version_id)
|
||||||
|
.flatten()
|
||||||
|
.filter(|bytes| *bytes != [0; 16]);
|
||||||
let expected_version = version_id.map(|bytes| uuid::Uuid::from_bytes(bytes).to_string());
|
let expected_version = version_id.map(|bytes| uuid::Uuid::from_bytes(bytes).to_string());
|
||||||
if outcome.identity.version_id != expected_version {
|
if outcome.identity.version_id != expected_version {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let expected_pool = target.scope.and_then(|scope| usize::try_from(scope.pool_index).ok());
|
let scope = (!matches!(target.kind, MrfKind::MetadataCorruption))
|
||||||
let expected_set = target.scope.and_then(|scope| usize::try_from(scope.set_index).ok());
|
.then_some(target.scope)
|
||||||
|
.flatten();
|
||||||
|
let expected_pool = scope.and_then(|scope| usize::try_from(scope.pool_index).ok());
|
||||||
|
let expected_set = scope.and_then(|scope| usize::try_from(scope.set_index).ok());
|
||||||
if outcome.identity.pool_index != expected_pool || outcome.identity.set_index != expected_set {
|
if outcome.identity.pool_index != expected_pool || outcome.identity.set_index != expected_set {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -759,7 +766,7 @@ pub(super) fn mrf_verified_repair_event_for_target(
|
|||||||
bucket: target.bucket.clone(),
|
bucket: target.bucket.clone(),
|
||||||
object: target.object.clone(),
|
object: target.object.clone(),
|
||||||
version_id,
|
version_id,
|
||||||
scope: target.scope,
|
scope,
|
||||||
lease: target.lease,
|
lease: target.lease,
|
||||||
bucket_incarnation_id,
|
bucket_incarnation_id,
|
||||||
disposition,
|
disposition,
|
||||||
|
|||||||
@@ -1142,18 +1142,48 @@ fn mrf_verified_repair_event_requires_positive_exact_identity() {
|
|||||||
assert_eq!(event.bucket_incarnation_id, incarnation);
|
assert_eq!(event.bucket_incarnation_id, incarnation);
|
||||||
assert_eq!(event.disposition, MrfVerifiedRepairDisposition::Repaired);
|
assert_eq!(event.disposition, MrfVerifiedRepairDisposition::Repaired);
|
||||||
|
|
||||||
assert!(
|
let decode_target = MrfRepairNoticeTarget {
|
||||||
mrf_verified_repair_event_for_target(
|
|
||||||
&MrfRepairNoticeTarget {
|
|
||||||
kind: MrfKind::DecodeFailure,
|
kind: MrfKind::DecodeFailure,
|
||||||
..target.clone()
|
..target.clone()
|
||||||
|
};
|
||||||
|
let decode_outcome = HealObjectOutcome {
|
||||||
|
identity: HealObjectIdentity {
|
||||||
|
kind: HealObjectKind::Decode,
|
||||||
|
..matching.identity.clone()
|
||||||
},
|
},
|
||||||
&matching
|
..matching.clone()
|
||||||
)
|
};
|
||||||
.is_none(),
|
let decode_event =
|
||||||
"only receipt-producing partial-write object heals can publish verified events today"
|
mrf_verified_repair_event_for_target(&decode_target, &decode_outcome).expect("decode repairs publish exact proofs");
|
||||||
|
assert_eq!(decode_event.kind, MrfKind::DecodeFailure);
|
||||||
|
assert_eq!(
|
||||||
|
decode_event.scope,
|
||||||
|
Some(MrfScope {
|
||||||
|
pool_index: 1,
|
||||||
|
set_index: 2
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let metadata_target = MrfRepairNoticeTarget {
|
||||||
|
kind: MrfKind::MetadataCorruption,
|
||||||
|
..target.clone()
|
||||||
|
};
|
||||||
|
let metadata_outcome = HealObjectOutcome {
|
||||||
|
identity: HealObjectIdentity {
|
||||||
|
kind: HealObjectKind::Metadata,
|
||||||
|
version_id: None,
|
||||||
|
pool_index: None,
|
||||||
|
set_index: None,
|
||||||
|
..matching.identity.clone()
|
||||||
|
},
|
||||||
|
..matching.clone()
|
||||||
|
};
|
||||||
|
let metadata_event =
|
||||||
|
mrf_verified_repair_event_for_target(&metadata_target, &metadata_outcome).expect("metadata repairs publish exact proofs");
|
||||||
|
assert_eq!(metadata_event.kind, MrfKind::MetadataCorruption);
|
||||||
|
assert_eq!(metadata_event.version_id, None);
|
||||||
|
assert_eq!(metadata_event.scope, None);
|
||||||
|
|
||||||
for rejected in [
|
for rejected in [
|
||||||
HealObjectOutcome {
|
HealObjectOutcome {
|
||||||
disposition: HealObjectDisposition::Unknown,
|
disposition: HealObjectDisposition::Unknown,
|
||||||
@@ -4537,6 +4567,48 @@ async fn test_force_start_marks_dedup_key_for_future_duplicates() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn same_request_id_replay_reuses_existing_task_without_force_start_duplication() {
|
||||||
|
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||||
|
let manager = HealManager::new(storage, None);
|
||||||
|
|
||||||
|
let mut original = admin_prefix_request("bucket", "logs/");
|
||||||
|
original.force_start = true;
|
||||||
|
let original_id = original.id.clone();
|
||||||
|
let accepted = manager
|
||||||
|
.submit_heal_request_with_receipt(original.clone())
|
||||||
|
.await
|
||||||
|
.expect("original forceStart request should queue");
|
||||||
|
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
|
||||||
|
assert_eq!(accepted.task_id, original_id);
|
||||||
|
|
||||||
|
let replayed = manager
|
||||||
|
.submit_heal_request_with_receipt(original.clone())
|
||||||
|
.await
|
||||||
|
.expect("same request id and payload should reuse the existing task");
|
||||||
|
assert_eq!(replayed.result, HealAdmissionResult::Accepted);
|
||||||
|
assert_eq!(replayed.task_id, original_id);
|
||||||
|
assert_eq!(
|
||||||
|
manager.get_queue_length().await,
|
||||||
|
1,
|
||||||
|
"exact forceStart replay must not create a second queued task"
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut changed = original;
|
||||||
|
changed.options.remove_corrupted = true;
|
||||||
|
let changed = manager
|
||||||
|
.submit_heal_request_with_receipt(changed)
|
||||||
|
.await
|
||||||
|
.expect("same request id with a changed payload should fail closed");
|
||||||
|
assert_eq!(changed.result, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning));
|
||||||
|
assert_eq!(changed.task_id, original_id);
|
||||||
|
assert_eq!(
|
||||||
|
manager.get_queue_length().await,
|
||||||
|
1,
|
||||||
|
"same-id conflict must not displace or duplicate the original task"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_running_heal_set_counts_groups_set_scoped_tasks() {
|
fn test_running_heal_set_counts_groups_set_scoped_tasks() {
|
||||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||||
|
|||||||
@@ -27,10 +27,10 @@ pub mod task;
|
|||||||
pub mod utils;
|
pub mod utils;
|
||||||
|
|
||||||
use storage_api::owner::{
|
use storage_api::owner::{
|
||||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME,
|
||||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskOption,
|
ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes,
|
||||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO,
|
EcstoreDiskError, EcstoreDiskOption, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
|
||||||
ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk,
|
EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub use erasure_healer::ErasureSetHealer;
|
pub use erasure_healer::ErasureSetHealer;
|
||||||
@@ -41,6 +41,7 @@ pub use task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
|
|||||||
pub(crate) const DATA_USAGE_CACHE_NAME: &str = ECSTORE_DATA_USAGE_CACHE_NAME;
|
pub(crate) const DATA_USAGE_CACHE_NAME: &str = ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||||
pub(crate) const BUCKET_META_PREFIX: &str = ECSTORE_BUCKET_META_PREFIX;
|
pub(crate) const BUCKET_META_PREFIX: &str = ECSTORE_BUCKET_META_PREFIX;
|
||||||
pub(crate) const RUSTFS_META_BUCKET: &str = ECSTORE_RUSTFS_META_BUCKET;
|
pub(crate) const RUSTFS_META_BUCKET: &str = ECSTORE_RUSTFS_META_BUCKET;
|
||||||
|
pub(crate) const POOL_META_NAME: &str = ECSTORE_POOL_META_NAME;
|
||||||
|
|
||||||
/// Marker written to every local disk while the process runs; removed by
|
/// Marker written to every local disk while the process runs; removed by
|
||||||
/// [`clear_unclean_shutdown_markers`] on graceful shutdown. Finding it at
|
/// [`clear_unclean_shutdown_markers`] on graceful shutdown. Finding it at
|
||||||
|
|||||||
@@ -36,7 +36,7 @@
|
|||||||
use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
|
use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
|
||||||
use crate::heal::manager::{HealManager, MrfRepairNoticeTarget};
|
use crate::heal::manager::{HealManager, MrfRepairNoticeTarget};
|
||||||
use metrics::{counter, gauge};
|
use metrics::{counter, gauge};
|
||||||
use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIngressResult, MrfIntent};
|
use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfDurableRepairAnchor, MrfIngressResult, MrfIntent};
|
||||||
use rustfs_heal_contracts::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
|
use rustfs_heal_contracts::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
|
||||||
use std::collections::{HashSet, VecDeque};
|
use std::collections::{HashSet, VecDeque};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -294,7 +294,11 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
|
|||||||
};
|
};
|
||||||
let attempts = data[3];
|
let attempts = data[3];
|
||||||
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().ok()?);
|
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().ok()?);
|
||||||
let has_version = data[12] != 0;
|
let has_version = match data[12] {
|
||||||
|
0 => false,
|
||||||
|
1 => true,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
let mut cursor = MRF_RECORD_FIXED_HEAD;
|
let mut cursor = MRF_RECORD_FIXED_HEAD;
|
||||||
let version_id = if has_version {
|
let version_id = if has_version {
|
||||||
if data.len() < cursor + 16 {
|
if data.len() < cursor + 16 {
|
||||||
@@ -517,6 +521,8 @@ async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> c
|
|||||||
struct MrfRuntime {
|
struct MrfRuntime {
|
||||||
queue: MrfQueue,
|
queue: MrfQueue,
|
||||||
config: MrfConsumerConfig,
|
config: MrfConsumerConfig,
|
||||||
|
checkpoint_owner: Uuid,
|
||||||
|
next_checkpoint_sequence: u64,
|
||||||
new_since_flush: usize,
|
new_since_flush: usize,
|
||||||
/// True while the in-memory pending set has changed since the last
|
/// True while the in-memory pending set has changed since the last
|
||||||
/// journal flush (push, pop, or an attempts bump that alters the encoded
|
/// journal flush (push, pop, or an attempts bump that alters the encoded
|
||||||
@@ -527,6 +533,18 @@ struct MrfRuntime {
|
|||||||
/// True while a journal snapshot exists on disk that may still be needed
|
/// True while a journal snapshot exists on disk that may still be needed
|
||||||
/// for replay or cleanup.
|
/// for replay or cleanup.
|
||||||
journal_on_disk: bool,
|
journal_on_disk: bool,
|
||||||
|
/// True when replay observed a responsibility that cannot be discharged by
|
||||||
|
/// a complete verified repair proof in this process.
|
||||||
|
retain_replay_journal: bool,
|
||||||
|
/// Partial-write responsibilities accepted from replay and waiting for an
|
||||||
|
/// exact storage-owned proof before the startup journal can be deleted.
|
||||||
|
durable_replay_anchors: Vec<MrfDurableRepairAnchor>,
|
||||||
|
/// Startup replay source to remove after the retained replay
|
||||||
|
/// responsibilities are discharged. `None` means the runtime only needs
|
||||||
|
/// the legacy journal cleanup path for snapshots it wrote itself.
|
||||||
|
replay_cleanup: Option<ReplayCleanup>,
|
||||||
|
/// Last committed checkpoint published by this runtime flush path.
|
||||||
|
runtime_checkpoint: Option<(Uuid, u64)>,
|
||||||
/// Earliest instant a full-admission retry may proceed.
|
/// Earliest instant a full-admission retry may proceed.
|
||||||
backoff_until: Option<tokio::time::Instant>,
|
backoff_until: Option<tokio::time::Instant>,
|
||||||
}
|
}
|
||||||
@@ -550,6 +568,34 @@ impl MrfRuntime {
|
|||||||
|
|
||||||
async fn flush(&mut self) {
|
async fn flush(&mut self) {
|
||||||
let (authoritative, legacy) = self.snapshot();
|
let (authoritative, legacy) = self.snapshot();
|
||||||
|
let (committed_persisted, committed_on_disk) = if authoritative.is_empty() {
|
||||||
|
(true, false)
|
||||||
|
} else {
|
||||||
|
match snapshot::publish_committed_snapshot(
|
||||||
|
&journal_disks().await,
|
||||||
|
self.checkpoint_owner,
|
||||||
|
self.next_checkpoint_sequence,
|
||||||
|
&authoritative,
|
||||||
|
self.config.journal_max_bytes,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(publication) => {
|
||||||
|
self.runtime_checkpoint = Some((publication.owner, publication.sequence));
|
||||||
|
self.next_checkpoint_sequence = publication.sequence.saturating_add(1);
|
||||||
|
(true, true)
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "rustfs::heal::mrf",
|
||||||
|
error = %err,
|
||||||
|
sequence = self.next_checkpoint_sequence,
|
||||||
|
"MRF committed checkpoint publish failed; retaining previous replay anchor"
|
||||||
|
);
|
||||||
|
(false, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
let authoritative_persisted = write_journal(MRF_SCOPED_JOURNAL_PATH, &authoritative).await;
|
let authoritative_persisted = write_journal(MRF_SCOPED_JOURNAL_PATH, &authoritative).await;
|
||||||
if !authoritative.is_empty() {
|
if !authoritative.is_empty() {
|
||||||
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
|
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
|
||||||
@@ -560,10 +606,10 @@ impl MrfRuntime {
|
|||||||
// old reader from observing a newer epoch that a new reader cannot
|
// old reader from observing a newer epoch that a new reader cannot
|
||||||
// see when the canonical write is unavailable.
|
// see when the canonical write is unavailable.
|
||||||
let legacy_persisted = authoritative_persisted && write_journal(MRF_JOURNAL_PATH, &legacy).await;
|
let legacy_persisted = authoritative_persisted && write_journal(MRF_JOURNAL_PATH, &legacy).await;
|
||||||
// Keep dirty until both the authoritative snapshot and its
|
// Keep dirty until the committed checkpoint, authoritative snapshot,
|
||||||
// compatibility mirror have been accepted; otherwise a one-sided
|
// and compatibility mirror have all been accepted; otherwise a
|
||||||
// failure would never retry the missing file.
|
// one-sided failure would never retry the missing recovery anchor.
|
||||||
let persisted = authoritative_persisted && legacy_persisted;
|
let persisted = committed_persisted && authoritative_persisted && legacy_persisted;
|
||||||
self.new_since_flush = 0;
|
self.new_since_flush = 0;
|
||||||
// Keep the dirty flag when every disk write failed: a clean backlog
|
// Keep the dirty flag when every disk write failed: a clean backlog
|
||||||
// would otherwise never rewrite, losing the periodic persist retry a
|
// would otherwise never rewrite, losing the periodic persist retry a
|
||||||
@@ -571,7 +617,7 @@ impl MrfRuntime {
|
|||||||
if persisted {
|
if persisted {
|
||||||
self.dirty = false;
|
self.dirty = false;
|
||||||
}
|
}
|
||||||
self.journal_on_disk |= authoritative_persisted || legacy_persisted;
|
self.journal_on_disk |= committed_on_disk || authoritative_persisted || legacy_persisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drain pending intents into the heal manager until it is full, the
|
/// Drain pending intents into the heal manager until it is full, the
|
||||||
@@ -624,6 +670,68 @@ impl MrfRuntime {
|
|||||||
gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(self.queue.depth()));
|
gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(self.queue.depth()));
|
||||||
gauge!("rustfs_heal_mrf_queue_bytes").set(metric_f64(self.queue.bytes()));
|
gauge!("rustfs_heal_mrf_queue_bytes").set(metric_f64(self.queue.bytes()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn retained_replay_journal(&self) -> bool {
|
||||||
|
self.retain_replay_journal || !self.durable_replay_anchors.is_empty()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn replay_cleanup_to_delete(&self) -> Option<ReplayCleanup> {
|
||||||
|
if self.journal_on_disk && !self.retained_replay_journal() {
|
||||||
|
Some(self.replay_cleanup.unwrap_or(ReplayCleanup::Legacy))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_idle_recovery_anchors(&mut self) -> bool {
|
||||||
|
let runtime_deleted = match self.runtime_checkpoint {
|
||||||
|
Some((owner, sequence)) => {
|
||||||
|
match snapshot::delete_committed_snapshots_through(owner, sequence, self.config.journal_max_bytes).await {
|
||||||
|
Ok(deleted) => deleted,
|
||||||
|
Err(err) => {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "rustfs::heal::mrf",
|
||||||
|
error = %err,
|
||||||
|
sequence,
|
||||||
|
"MRF runtime checkpoint cleanup failed"
|
||||||
|
);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => true,
|
||||||
|
};
|
||||||
|
let replay_deleted = match self.replay_cleanup_to_delete() {
|
||||||
|
Some(cleanup) => delete_replay_source(cleanup, self.config.journal_max_bytes).await,
|
||||||
|
None => true,
|
||||||
|
};
|
||||||
|
if runtime_deleted && replay_deleted {
|
||||||
|
self.runtime_checkpoint = None;
|
||||||
|
self.replay_cleanup = None;
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn discharge_durable_replay_anchors(&mut self) {
|
||||||
|
if self.durable_replay_anchors.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut buckets: Vec<Arc<str>> = self
|
||||||
|
.durable_replay_anchors
|
||||||
|
.iter()
|
||||||
|
.map(|anchor| anchor.bucket.clone())
|
||||||
|
.collect();
|
||||||
|
buckets.sort_unstable();
|
||||||
|
buckets.dedup();
|
||||||
|
for bucket in buckets {
|
||||||
|
rustfs_common::mrf_channel::consume_recorded_verified_mrf_repair_events_for(
|
||||||
|
bucket.as_ref(),
|
||||||
|
&mut self.durable_replay_anchors,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Initialize the global MRF channel (honoring `RUSTFS_HEAL_MRF_ENABLE`) and
|
/// Initialize the global MRF channel (honoring `RUSTFS_HEAL_MRF_ENABLE`) and
|
||||||
@@ -673,10 +781,77 @@ pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
|
|||||||
struct ReplayOutcome {
|
struct ReplayOutcome {
|
||||||
replayed: usize,
|
replayed: usize,
|
||||||
journal_on_disk: bool,
|
journal_on_disk: bool,
|
||||||
|
retain_journal_for_replay: bool,
|
||||||
|
durable_replay_anchors: Vec<MrfDurableRepairAnchor>,
|
||||||
|
cleanup: Option<ReplayCleanup>,
|
||||||
|
next_checkpoint_sequence: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn replay_must_retain_journal(rearm_incomplete: bool, pending_depth: usize) -> bool {
|
fn replay_must_retain_journal(
|
||||||
rearm_incomplete || pending_depth > 0
|
rearm_incomplete: bool,
|
||||||
|
pending_depth: usize,
|
||||||
|
accepted_without_durable_anchor: bool,
|
||||||
|
durable_replay_anchors: usize,
|
||||||
|
) -> bool {
|
||||||
|
rearm_incomplete || pending_depth > 0 || accepted_without_durable_anchor || durable_replay_anchors > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
enum ReplayCleanup {
|
||||||
|
Legacy,
|
||||||
|
Committed { owner: Uuid, sequence: u64 },
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ReplaySource {
|
||||||
|
data: Vec<u8>,
|
||||||
|
cleanup: ReplayCleanup,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read_replay_source(max_bytes: usize) -> Result<Option<ReplaySource>, snapshot::SnapshotError> {
|
||||||
|
if let Some(committed) = snapshot::inspect_local_committed_snapshot(max_bytes).await? {
|
||||||
|
return Ok(Some(ReplaySource {
|
||||||
|
data: committed.payload().to_vec(),
|
||||||
|
cleanup: ReplayCleanup::Committed {
|
||||||
|
owner: committed.owner(),
|
||||||
|
sequence: committed.sequence(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
// The scoped file is a complete authoritative legacy snapshot. Fall back
|
||||||
|
// to the v1 mirror only when the authoritative path is unavailable;
|
||||||
|
// merging both files could combine records from different flush epochs.
|
||||||
|
let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await {
|
||||||
|
Some(data) => data,
|
||||||
|
None => match read_journal(MRF_JOURNAL_PATH).await {
|
||||||
|
Some(data) => data,
|
||||||
|
None => return Ok(None),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
Ok(Some(ReplaySource {
|
||||||
|
data,
|
||||||
|
cleanup: ReplayCleanup::Legacy,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_replay_source(cleanup: ReplayCleanup, max_bytes: usize) -> bool {
|
||||||
|
let committed_deleted = match cleanup {
|
||||||
|
ReplayCleanup::Legacy => true,
|
||||||
|
ReplayCleanup::Committed { owner, sequence } => {
|
||||||
|
match snapshot::delete_committed_snapshots_through(owner, sequence, max_bytes).await {
|
||||||
|
Ok(deleted) => deleted,
|
||||||
|
Err(err) => {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "rustfs::heal::mrf",
|
||||||
|
error = %err,
|
||||||
|
sequence,
|
||||||
|
"MRF committed replay checkpoint cleanup failed"
|
||||||
|
);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
committed_deleted && delete_journals().await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Shared replay core: read + decode + re-arm, then drain what fits. The
|
/// Shared replay core: read + decode + re-arm, then drain what fits. The
|
||||||
@@ -687,21 +862,40 @@ async fn replay_into(
|
|||||||
queue: &mut MrfQueue,
|
queue: &mut MrfQueue,
|
||||||
backoff_until: &mut Option<tokio::time::Instant>,
|
backoff_until: &mut Option<tokio::time::Instant>,
|
||||||
) -> ReplayOutcome {
|
) -> ReplayOutcome {
|
||||||
// The scoped file is a complete authoritative snapshot. Fall back to the
|
let source = match read_replay_source(queue.byte_budget).await {
|
||||||
// legacy mirror only when the authoritative path is unavailable; merging
|
Ok(Some(source)) => source,
|
||||||
// both files could combine records from different flush epochs.
|
Ok(None) => {
|
||||||
let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await {
|
|
||||||
Some(data) => data,
|
|
||||||
None => match read_journal(MRF_JOURNAL_PATH).await {
|
|
||||||
Some(data) => data,
|
|
||||||
None => {
|
|
||||||
return ReplayOutcome {
|
return ReplayOutcome {
|
||||||
replayed: 0,
|
replayed: 0,
|
||||||
journal_on_disk: false,
|
journal_on_disk: false,
|
||||||
|
retain_journal_for_replay: false,
|
||||||
|
durable_replay_anchors: Vec::new(),
|
||||||
|
cleanup: None,
|
||||||
|
next_checkpoint_sequence: 1,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
},
|
Err(err) => {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "rustfs::heal::mrf",
|
||||||
|
error = %err,
|
||||||
|
"MRF committed replay checkpoint could not be inspected"
|
||||||
|
);
|
||||||
|
return ReplayOutcome {
|
||||||
|
replayed: 0,
|
||||||
|
journal_on_disk: true,
|
||||||
|
retain_journal_for_replay: true,
|
||||||
|
durable_replay_anchors: Vec::new(),
|
||||||
|
cleanup: None,
|
||||||
|
next_checkpoint_sequence: 1,
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let cleanup = source.cleanup;
|
||||||
|
let next_checkpoint_sequence = match cleanup {
|
||||||
|
ReplayCleanup::Legacy => 1,
|
||||||
|
ReplayCleanup::Committed { sequence, .. } => sequence.saturating_add(1),
|
||||||
|
};
|
||||||
|
let data = source.data;
|
||||||
let (decoded, truncated) = decode_journal(&data);
|
let (decoded, truncated) = decode_journal(&data);
|
||||||
let replayed = decoded.len();
|
let replayed = decoded.len();
|
||||||
let intents = decoded;
|
let intents = decoded;
|
||||||
@@ -722,6 +916,8 @@ async fn replay_into(
|
|||||||
// prefix.
|
// prefix.
|
||||||
queue.raise_limits_for_replay(intents.len(), replay_bytes);
|
queue.raise_limits_for_replay(intents.len(), replay_bytes);
|
||||||
let mut rearm_incomplete = false;
|
let mut rearm_incomplete = false;
|
||||||
|
let mut accepted_without_durable_anchor = false;
|
||||||
|
let mut durable_replay_anchors = Vec::new();
|
||||||
for intent in intents {
|
for intent in intents {
|
||||||
let result = queue.try_push_typed(intent.clone());
|
let result = queue.try_push_typed(intent.clone());
|
||||||
match result {
|
match result {
|
||||||
@@ -748,7 +944,13 @@ async fn replay_into(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
match submit_mrf_heal_request(manager, &intent).await {
|
match submit_mrf_heal_request(manager, &intent).await {
|
||||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {
|
||||||
|
if let Some(anchor) = manager.durable_mrf_repair_anchor(&intent).await {
|
||||||
|
durable_replay_anchors.push(anchor);
|
||||||
|
} else {
|
||||||
|
accepted_without_durable_anchor = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||||
intent.attempts = intent.attempts.saturating_add(1);
|
intent.attempts = intent.attempts.saturating_add(1);
|
||||||
if intent.attempts < MRF_MAX_ATTEMPTS {
|
if intent.attempts < MRF_MAX_ATTEMPTS {
|
||||||
@@ -779,14 +981,25 @@ async fn replay_into(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let journal_on_disk = if replay_must_retain_journal(rearm_incomplete, queue.depth()) {
|
let must_retain_journal = replay_must_retain_journal(
|
||||||
|
rearm_incomplete,
|
||||||
|
queue.depth(),
|
||||||
|
accepted_without_durable_anchor,
|
||||||
|
durable_replay_anchors.len(),
|
||||||
|
);
|
||||||
|
let retain_journal_for_replay = rearm_incomplete || accepted_without_durable_anchor;
|
||||||
|
let journal_on_disk = if must_retain_journal {
|
||||||
true
|
true
|
||||||
} else {
|
} else {
|
||||||
!delete_journals().await
|
!delete_replay_source(cleanup, queue.byte_budget).await
|
||||||
};
|
};
|
||||||
ReplayOutcome {
|
ReplayOutcome {
|
||||||
replayed,
|
replayed,
|
||||||
journal_on_disk,
|
journal_on_disk,
|
||||||
|
retain_journal_for_replay,
|
||||||
|
durable_replay_anchors,
|
||||||
|
cleanup: journal_on_disk.then_some(cleanup),
|
||||||
|
next_checkpoint_sequence,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -797,9 +1010,15 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
|||||||
let mut runtime = MrfRuntime {
|
let mut runtime = MrfRuntime {
|
||||||
queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes),
|
queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes),
|
||||||
config: config.clone(),
|
config: config.clone(),
|
||||||
|
checkpoint_owner: Uuid::new_v4(),
|
||||||
|
next_checkpoint_sequence: 1,
|
||||||
new_since_flush: 0,
|
new_since_flush: 0,
|
||||||
dirty: false,
|
dirty: false,
|
||||||
journal_on_disk: false,
|
journal_on_disk: false,
|
||||||
|
retain_replay_journal: false,
|
||||||
|
durable_replay_anchors: Vec::new(),
|
||||||
|
replay_cleanup: None,
|
||||||
|
runtime_checkpoint: None,
|
||||||
backoff_until: None,
|
backoff_until: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -807,6 +1026,10 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
|||||||
// on disk whenever any replayed intent still needs a successor snapshot.
|
// on disk whenever any replayed intent still needs a successor snapshot.
|
||||||
let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
||||||
runtime.journal_on_disk = replay.journal_on_disk;
|
runtime.journal_on_disk = replay.journal_on_disk;
|
||||||
|
runtime.retain_replay_journal = replay.retain_journal_for_replay;
|
||||||
|
runtime.durable_replay_anchors = replay.durable_replay_anchors;
|
||||||
|
runtime.replay_cleanup = replay.cleanup;
|
||||||
|
runtime.next_checkpoint_sequence = replay.next_checkpoint_sequence;
|
||||||
// Anything still pending (e.g. the manager was full and backoff armed)
|
// Anything still pending (e.g. the manager was full and backoff armed)
|
||||||
// must be re-persisted by the next flush before replay can delete the
|
// must be re-persisted by the next flush before replay can delete the
|
||||||
// startup anchor.
|
// startup anchor.
|
||||||
@@ -850,10 +1073,12 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = flush_tick.tick() => {
|
_ = flush_tick.tick() => {
|
||||||
|
runtime.discharge_durable_replay_anchors();
|
||||||
match tick_action(
|
match tick_action(
|
||||||
runtime.dirty,
|
runtime.dirty,
|
||||||
runtime.queue.depth(),
|
runtime.queue.depth(),
|
||||||
runtime.journal_on_disk,
|
runtime.journal_on_disk,
|
||||||
|
runtime.retained_replay_journal(),
|
||||||
) {
|
) {
|
||||||
TickAction::Flush => {
|
TickAction::Flush => {
|
||||||
runtime.flush().await;
|
runtime.flush().await;
|
||||||
@@ -869,7 +1094,7 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
|||||||
TickAction::DeleteJournal => {
|
TickAction::DeleteJournal => {
|
||||||
// All replayed intents have either been accepted,
|
// All replayed intents have either been accepted,
|
||||||
// merged, or replaced by a pending successor snapshot.
|
// merged, or replaced by a pending successor snapshot.
|
||||||
if delete_journals().await {
|
if runtime.delete_idle_recovery_anchors().await {
|
||||||
runtime.journal_on_disk = false;
|
runtime.journal_on_disk = false;
|
||||||
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
||||||
}
|
}
|
||||||
@@ -897,12 +1122,12 @@ enum TickAction {
|
|||||||
Idle,
|
Idle,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction {
|
fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool, retain_replay_journal: bool) -> TickAction {
|
||||||
if dirty {
|
if dirty {
|
||||||
TickAction::Flush
|
TickAction::Flush
|
||||||
} else if depth > 0 {
|
} else if depth > 0 {
|
||||||
TickAction::Retry
|
TickAction::Retry
|
||||||
} else if journal_on_disk {
|
} else if journal_on_disk && !retain_replay_journal {
|
||||||
TickAction::DeleteJournal
|
TickAction::DeleteJournal
|
||||||
} else {
|
} else {
|
||||||
TickAction::Idle
|
TickAction::Idle
|
||||||
@@ -912,7 +1137,8 @@ fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use rustfs_common::mrf_channel::{MrfIntent, MrfKind};
|
use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfVerifiedRepairDisposition, MrfVerifiedRepairEvent};
|
||||||
|
use serial_test::serial;
|
||||||
use std::sync::Arc as StdArc;
|
use std::sync::Arc as StdArc;
|
||||||
|
|
||||||
fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent {
|
fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent {
|
||||||
@@ -928,41 +1154,203 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn encoded_payload(intent: &MrfIntent) -> Vec<u8> {
|
||||||
|
let mut payload = Vec::new();
|
||||||
|
assert!(encode_intent(intent, &mut payload), "fixture intent must encode");
|
||||||
|
payload
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn tick_action_table() {
|
fn tick_action_table() {
|
||||||
use TickAction::*;
|
use TickAction::*;
|
||||||
|
|
||||||
// Dirty dominates: a changed pending set flushes even when idle
|
// Dirty dominates: a changed pending set flushes even when idle
|
||||||
// otherwise.
|
// otherwise.
|
||||||
assert!(matches!(tick_action(true, 0, false), Flush));
|
assert!(matches!(tick_action(true, 0, false, false), Flush));
|
||||||
assert!(matches!(tick_action(true, 3, true), Flush));
|
assert!(matches!(tick_action(true, 3, true, false), Flush));
|
||||||
|
|
||||||
// Clean backlog: no rewrite, but keep draining so an expired
|
// Clean backlog: no rewrite, but keep draining so an expired
|
||||||
// admission backoff retries on time.
|
// admission backoff retries on time.
|
||||||
assert!(matches!(tick_action(false, 1, false), Retry));
|
assert!(matches!(tick_action(false, 1, false, false), Retry));
|
||||||
assert!(matches!(tick_action(false, 2, true), Retry));
|
assert!(matches!(tick_action(false, 2, true, false), Retry));
|
||||||
|
|
||||||
// Quiescent with a stale journal file on disk: remove it.
|
// Quiescent with a stale journal file on disk: remove it.
|
||||||
assert!(matches!(tick_action(false, 0, true), DeleteJournal));
|
assert!(matches!(tick_action(false, 0, true, false), DeleteJournal));
|
||||||
|
assert!(matches!(tick_action(false, 0, true, true), Idle));
|
||||||
|
|
||||||
// Fully quiescent: nothing to do.
|
// Fully quiescent: nothing to do.
|
||||||
assert!(matches!(tick_action(false, 0, false), Idle));
|
assert!(matches!(tick_action(false, 0, false, false), Idle));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replay_cleanup_retains_journal_for_unarmed_or_refused_records() {
|
fn replay_cleanup_retains_journal_for_unarmed_or_refused_records() {
|
||||||
assert!(
|
assert!(
|
||||||
replay_must_retain_journal(true, 0),
|
replay_must_retain_journal(true, 0, false, 0),
|
||||||
"a rejected replay record still needs its disk anchor"
|
"a rejected replay record still needs its disk anchor"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
replay_must_retain_journal(false, 1),
|
replay_must_retain_journal(false, 1, false, 0),
|
||||||
"a Full admission retry must keep the startup journal until the next snapshot"
|
"a Full admission retry must keep the startup journal until the next snapshot"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
!replay_must_retain_journal(false, 0),
|
!replay_must_retain_journal(false, 0, false, 0),
|
||||||
"only a fully consumed replay snapshot may be deleted"
|
"a fully consumed replay snapshot without accepts may be deleted"
|
||||||
);
|
);
|
||||||
|
assert!(
|
||||||
|
replay_must_retain_journal(false, 0, true, 0),
|
||||||
|
"accepted or merged replay records without a proof identity still need a durable successor"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
replay_must_retain_journal(false, 0, false, 1),
|
||||||
|
"accepted replay records with a durable proof anchor must retain the journal until proof arrives"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_releases_retained_replay_journal_after_verified_repair_proof() {
|
||||||
|
let mut intent = intent("proof-bucket", "proof-object", 0);
|
||||||
|
intent.kind = MrfKind::PartialWrite;
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut intent),
|
||||||
|
MrfIngressResult::Enqueued
|
||||||
|
);
|
||||||
|
let bucket_incarnation_id = uuid::Uuid::new_v4();
|
||||||
|
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, bucket_incarnation_id)
|
||||||
|
.expect("fresh replay lease and bucket incarnation build a durable anchor");
|
||||||
|
let cleanup_owner = uuid::Uuid::new_v4();
|
||||||
|
let cleanup = ReplayCleanup::Committed {
|
||||||
|
owner: cleanup_owner,
|
||||||
|
sequence: 17,
|
||||||
|
};
|
||||||
|
let mut runtime = MrfRuntime {
|
||||||
|
queue: MrfQueue::new(2, usize::MAX),
|
||||||
|
config: MrfConsumerConfig::default(),
|
||||||
|
checkpoint_owner: Uuid::new_v4(),
|
||||||
|
next_checkpoint_sequence: 1,
|
||||||
|
new_since_flush: 0,
|
||||||
|
dirty: false,
|
||||||
|
journal_on_disk: true,
|
||||||
|
retain_replay_journal: false,
|
||||||
|
durable_replay_anchors: vec![anchor],
|
||||||
|
replay_cleanup: Some(cleanup),
|
||||||
|
runtime_checkpoint: None,
|
||||||
|
backoff_until: None,
|
||||||
|
};
|
||||||
|
rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent {
|
||||||
|
kind: intent.kind,
|
||||||
|
bucket: intent.bucket.clone(),
|
||||||
|
object: intent.object.clone(),
|
||||||
|
version_id: intent.version_id,
|
||||||
|
scope: intent.scope,
|
||||||
|
lease: intent.lease,
|
||||||
|
bucket_incarnation_id,
|
||||||
|
disposition: MrfVerifiedRepairDisposition::Repaired,
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
runtime.retained_replay_journal(),
|
||||||
|
"anchor must retain the startup journal before proof is consumed"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime.replay_cleanup_to_delete(),
|
||||||
|
None,
|
||||||
|
"the committed replay source must not be reclaimed before the exact proof"
|
||||||
|
);
|
||||||
|
runtime.discharge_durable_replay_anchors();
|
||||||
|
assert!(
|
||||||
|
!runtime.retained_replay_journal(),
|
||||||
|
"matching verified proof discharges the durable replay anchor"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime.replay_cleanup_to_delete(),
|
||||||
|
Some(cleanup),
|
||||||
|
"proof discharge must preserve the committed owner/sequence cleanup target"
|
||||||
|
);
|
||||||
|
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_cleanup_defaults_to_legacy_for_runtime_written_journals() {
|
||||||
|
let runtime = MrfRuntime {
|
||||||
|
queue: MrfQueue::new(2, usize::MAX),
|
||||||
|
config: MrfConsumerConfig::default(),
|
||||||
|
checkpoint_owner: Uuid::new_v4(),
|
||||||
|
next_checkpoint_sequence: 1,
|
||||||
|
new_since_flush: 0,
|
||||||
|
dirty: false,
|
||||||
|
journal_on_disk: true,
|
||||||
|
retain_replay_journal: false,
|
||||||
|
durable_replay_anchors: Vec::new(),
|
||||||
|
replay_cleanup: None,
|
||||||
|
runtime_checkpoint: None,
|
||||||
|
backoff_until: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
runtime.replay_cleanup_to_delete(),
|
||||||
|
Some(ReplayCleanup::Legacy),
|
||||||
|
"journals written by the runtime still use the legacy cleanup path"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn runtime_idle_cleanup_deletes_runtime_and_replay_recovery_anchors() {
|
||||||
|
let _env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||||
|
.prefix("rustfs_mrf_runtime_idle_cleanup")
|
||||||
|
.build()
|
||||||
|
.await;
|
||||||
|
let disks = journal_disks().await;
|
||||||
|
assert!(!disks.is_empty(), "test environment must register local disks");
|
||||||
|
|
||||||
|
let replay_owner = Uuid::new_v4();
|
||||||
|
let runtime_owner = Uuid::new_v4();
|
||||||
|
let config = MrfConsumerConfig::default();
|
||||||
|
let journal_max_bytes = config.journal_max_bytes;
|
||||||
|
let replay_payload = encoded_payload(&intent("cleanup-bucket", "replay-object", 0));
|
||||||
|
let runtime_payload = encoded_payload(&intent("cleanup-bucket", "runtime-object", 0));
|
||||||
|
snapshot::publish_committed_snapshot(&disks, replay_owner, 7, &replay_payload, journal_max_bytes)
|
||||||
|
.await
|
||||||
|
.expect("publish retained replay checkpoint");
|
||||||
|
snapshot::publish_committed_snapshot(&disks, runtime_owner, 8, &runtime_payload, journal_max_bytes)
|
||||||
|
.await
|
||||||
|
.expect("publish runtime checkpoint");
|
||||||
|
assert!(write_journal(MRF_SCOPED_JOURNAL_PATH, &runtime_payload).await);
|
||||||
|
assert!(write_journal(MRF_JOURNAL_PATH, &runtime_payload).await);
|
||||||
|
|
||||||
|
let mut runtime = MrfRuntime {
|
||||||
|
queue: MrfQueue::new(2, usize::MAX),
|
||||||
|
config,
|
||||||
|
checkpoint_owner: Uuid::new_v4(),
|
||||||
|
next_checkpoint_sequence: 9,
|
||||||
|
new_since_flush: 0,
|
||||||
|
dirty: false,
|
||||||
|
journal_on_disk: true,
|
||||||
|
retain_replay_journal: false,
|
||||||
|
durable_replay_anchors: Vec::new(),
|
||||||
|
replay_cleanup: Some(ReplayCleanup::Committed {
|
||||||
|
owner: replay_owner,
|
||||||
|
sequence: 7,
|
||||||
|
}),
|
||||||
|
runtime_checkpoint: Some((runtime_owner, 8)),
|
||||||
|
backoff_until: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
runtime.delete_idle_recovery_anchors().await,
|
||||||
|
"idle cleanup should remove both runtime and replay recovery anchors"
|
||||||
|
);
|
||||||
|
assert_eq!(runtime.replay_cleanup, None);
|
||||||
|
assert_eq!(runtime.runtime_checkpoint, None);
|
||||||
|
assert!(
|
||||||
|
snapshot::inspect_local_committed_snapshot(journal_max_bytes)
|
||||||
|
.await
|
||||||
|
.expect("inspect committed checkpoints after cleanup")
|
||||||
|
.is_none(),
|
||||||
|
"both committed checkpoint generations must be gone after idle cleanup"
|
||||||
|
);
|
||||||
|
assert_eq!(read_journal(MRF_SCOPED_JOURNAL_PATH).await, None);
|
||||||
|
assert_eq!(read_journal(MRF_JOURNAL_PATH).await, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1166,6 +1554,25 @@ mod tests {
|
|||||||
assert_eq!(truncated, corrupt.len());
|
assert_eq!(truncated, corrupt.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn journal_rejects_unknown_version_presence_flag_even_with_valid_crc() {
|
||||||
|
let mut versioned = intent("rollback-bucket", "object", 0);
|
||||||
|
versioned.version_id = Some([9; 16]);
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
assert!(encode_intent(&versioned, &mut buf));
|
||||||
|
|
||||||
|
buf[12] = 2;
|
||||||
|
let crc_offset = buf.len() - 4;
|
||||||
|
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||||
|
hasher.update(&buf[..crc_offset]);
|
||||||
|
let checksum = u32::try_from(hasher.finalize()).expect("CRC32 fits");
|
||||||
|
buf[crc_offset..].copy_from_slice(&checksum.to_le_bytes());
|
||||||
|
|
||||||
|
let (decoded, truncated) = decode_journal(&buf);
|
||||||
|
assert!(decoded.is_empty(), "unknown boolean encodings are not rollback-compatible payloads");
|
||||||
|
assert_eq!(truncated, buf.len());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn heal_request_mapping_follows_priority_matrix() {
|
fn heal_request_mapping_follows_priority_matrix() {
|
||||||
let decode = build_heal_request(&intent("b", "o", 0));
|
let decode = build_heal_request(&intent("b", "o", 0));
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -110,7 +110,8 @@ impl HealObjectReceipt {
|
|||||||
&& self.identity.version_id == expected.version_id
|
&& self.identity.version_id == expected.version_id
|
||||||
&& self.identity.pool_index == expected.pool_index
|
&& self.identity.pool_index == expected.pool_index
|
||||||
&& self.identity.set_index == expected.set_index
|
&& self.identity.set_index == expected.set_index
|
||||||
&& self.identity.bucket_incarnation_id.is_some()
|
&& self.identity.bucket_incarnation_id == expected.bucket_incarnation_id
|
||||||
|
&& expected.bucket_incarnation_id.is_some()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -496,21 +497,31 @@ mod canonical_outcome_tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn positive_receipt_requires_exact_identity_and_bucket_incarnation() {
|
fn positive_receipt_requires_exact_identity_and_bucket_incarnation() {
|
||||||
let expected = item(HealObjectDisposition::Unknown).identity;
|
let incarnation = Uuid::new_v4();
|
||||||
|
let expected = HealObjectIdentity {
|
||||||
|
bucket_incarnation_id: Some(incarnation),
|
||||||
|
..item(HealObjectDisposition::Unknown).identity
|
||||||
|
};
|
||||||
let mut receipt = HealObjectReceipt {
|
let mut receipt = HealObjectReceipt {
|
||||||
identity: expected.clone(),
|
identity: expected.clone(),
|
||||||
disposition: HealObjectDisposition::Repaired,
|
disposition: HealObjectDisposition::Repaired,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
receipt.identity.bucket_incarnation_id = None;
|
||||||
assert!(
|
assert!(
|
||||||
!receipt.verified_for(&expected),
|
!receipt.verified_for(&expected),
|
||||||
"a positive storage receipt without bucket incarnation must remain untrusted"
|
"a positive storage receipt without bucket incarnation must remain untrusted"
|
||||||
);
|
);
|
||||||
|
|
||||||
let incarnation = Uuid::new_v4();
|
|
||||||
receipt.identity.bucket_incarnation_id = Some(incarnation);
|
receipt.identity.bucket_incarnation_id = Some(incarnation);
|
||||||
assert!(receipt.verified_for(&expected));
|
assert!(receipt.verified_for(&expected));
|
||||||
|
|
||||||
|
receipt.identity.bucket_incarnation_id = Some(Uuid::new_v4());
|
||||||
|
assert!(
|
||||||
|
!receipt.verified_for(&expected),
|
||||||
|
"a storage receipt for a different bucket incarnation must not clear the requested responsibility"
|
||||||
|
);
|
||||||
|
|
||||||
receipt.identity.version_id = Some("older-version".to_string());
|
receipt.identity.version_id = Some("older-version".to_string());
|
||||||
assert!(
|
assert!(
|
||||||
!receipt.verified_for(&expected),
|
!receipt.verified_for(&expected),
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tracing::{debug, error, warn};
|
use tracing::{debug, error, warn};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::outcome::{HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectReceipt};
|
use super::outcome::{HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectReceipt};
|
||||||
use super::progress::stable_generation;
|
use super::progress::stable_generation;
|
||||||
@@ -355,6 +356,14 @@ pub trait HealStorageAPI: Send + Sync {
|
|||||||
/// Get bucket info
|
/// Get bucket info
|
||||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
|
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
|
||||||
|
|
||||||
|
/// Return the current bucket incarnation for exact MRF durable proof
|
||||||
|
/// matching. Alternate backends that cannot expose this must return
|
||||||
|
/// `None`, leaving replay anchors retained instead of acknowledged with an
|
||||||
|
/// incomplete identity.
|
||||||
|
async fn mrf_bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
|
||||||
/// Aggregate usage-cache baselines for the requested buckets.
|
/// Aggregate usage-cache baselines for the requested buckets.
|
||||||
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||||
Ok(None)
|
Ok(None)
|
||||||
@@ -383,6 +392,11 @@ pub trait HealStorageAPI: Send + Sync {
|
|||||||
/// Check object exists
|
/// Check object exists
|
||||||
async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool>;
|
async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool>;
|
||||||
|
|
||||||
|
/// Stable bucket incarnation observed before an object heal starts.
|
||||||
|
async fn bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
|
||||||
/// Heal object using ecstore
|
/// Heal object using ecstore
|
||||||
async fn heal_object(
|
async fn heal_object(
|
||||||
&self,
|
&self,
|
||||||
@@ -809,6 +823,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn mrf_bucket_incarnation_id(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||||
|
self.ecstore
|
||||||
|
.bucket_incarnation_id(bucket)
|
||||||
|
.await
|
||||||
|
.map(Some)
|
||||||
|
.map_err(Error::Storage)
|
||||||
|
}
|
||||||
|
|
||||||
async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||||
if buckets.is_empty() {
|
if buckets.is_empty() {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -1028,6 +1050,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||||
|
self.ecstore
|
||||||
|
.bucket_incarnation_id(bucket)
|
||||||
|
.await
|
||||||
|
.map(Some)
|
||||||
|
.map_err(Error::Storage)
|
||||||
|
}
|
||||||
|
|
||||||
async fn heal_object(
|
async fn heal_object(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageEr
|
|||||||
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
||||||
pub(crate) use rustfs_ecstore::api::storage::{
|
pub(crate) use rustfs_ecstore::api::storage::{
|
||||||
ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext,
|
ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext,
|
||||||
|
POOL_META_NAME as ECSTORE_POOL_META_NAME,
|
||||||
};
|
};
|
||||||
use rustfs_storage_api as storage_contracts;
|
use rustfs_storage_api as storage_contracts;
|
||||||
|
|
||||||
@@ -36,10 +37,11 @@ pub(crate) mod owner {
|
|||||||
pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
|
pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
|
||||||
|
|
||||||
pub(crate) use super::{
|
pub(crate) use super::{
|
||||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME,
|
||||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError,
|
ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes,
|
||||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext,
|
EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
|
||||||
EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
|
EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached,
|
||||||
|
ecstore_local_disk_map_read,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};
|
pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ use tokio::sync::RwLock;
|
|||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, RUSTFS_META_BUCKET};
|
use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, POOL_META_NAME, RUSTFS_META_BUCKET};
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) struct OutcomeFinishTestHook {
|
pub(crate) struct OutcomeFinishTestHook {
|
||||||
@@ -106,7 +106,7 @@ const EVENT_HEAL_ERASURE_SET_STAGE: &str = "heal_erasure_set_stage";
|
|||||||
const EVENT_HEAL_ERASURE_SET_RESULT: &str = "heal_erasure_set_result";
|
const EVENT_HEAL_ERASURE_SET_RESULT: &str = "heal_erasure_set_result";
|
||||||
|
|
||||||
/// Heal type
|
/// Heal type
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub enum HealType {
|
pub enum HealType {
|
||||||
/// Cluster heal
|
/// Cluster heal
|
||||||
Cluster,
|
Cluster,
|
||||||
@@ -209,7 +209,7 @@ impl HealPriority {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Heal options
|
/// Heal options
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct HealOptions {
|
pub struct HealOptions {
|
||||||
/// Scan mode
|
/// Scan mode
|
||||||
pub scan_mode: HealScanMode,
|
pub scan_mode: HealScanMode,
|
||||||
@@ -574,6 +574,18 @@ impl HealTask {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) async fn outcome_bucket_incarnation_id(&self, bucket: &str, dry_run: bool) -> Result<Option<Uuid>> {
|
||||||
|
if dry_run {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
match self.await_with_control(self.storage.bucket_incarnation_id(bucket)).await {
|
||||||
|
Ok(incarnation_id) => Ok(incarnation_id),
|
||||||
|
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||||
|
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||||
|
Err(_) => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn single_object_identity(&self) -> Option<HealObjectIdentity> {
|
fn single_object_identity(&self) -> Option<HealObjectIdentity> {
|
||||||
let (bucket, object, version) = match &self.heal_type {
|
let (bucket, object, version) = match &self.heal_type {
|
||||||
HealType::Object {
|
HealType::Object {
|
||||||
@@ -597,6 +609,9 @@ impl HealTask {
|
|||||||
expected: HealObjectIdentity,
|
expected: HealObjectIdentity,
|
||||||
receipt: Option<HealObjectReceipt>,
|
receipt: Option<HealObjectReceipt>,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
|
if self.options.dry_run || self.cancel_token.is_cancelled() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
let Some(receipt) = receipt else {
|
let Some(receipt) = receipt else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -340,9 +340,88 @@ impl HealTask {
|
|||||||
return Err(self.record_batch_failure(failure).await);
|
return Err(self.record_batch_failure(failure).await);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if self.options.recreate_missing && !self.options.dry_run {
|
||||||
|
self.heal_cluster_pool_metadata().await?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn heal_cluster_pool_metadata(&self) -> Result<()> {
|
||||||
|
let heal_opts = HealOpts {
|
||||||
|
recursive: false,
|
||||||
|
dry_run: self.options.dry_run,
|
||||||
|
remove: false,
|
||||||
|
recreate: self.options.recreate_missing,
|
||||||
|
scan_mode: self.options.scan_mode,
|
||||||
|
update_parity: self.options.update_parity,
|
||||||
|
no_lock: self.options.no_lock,
|
||||||
|
read_repair: false,
|
||||||
|
pool: self.options.pool_index,
|
||||||
|
set: self.options.set_index,
|
||||||
|
};
|
||||||
|
|
||||||
|
let heal_result = self
|
||||||
|
.await_with_control(self.storage.heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, None, &heal_opts))
|
||||||
|
.await;
|
||||||
|
match heal_result {
|
||||||
|
Ok((result, None)) => {
|
||||||
|
debug!(
|
||||||
|
target: "rustfs::heal::task",
|
||||||
|
event = EVENT_HEAL_BUCKET_RESULT,
|
||||||
|
component = LOG_COMPONENT_HEAL,
|
||||||
|
subsystem = LOG_SUBSYSTEM_TASK,
|
||||||
|
task_id = %self.id,
|
||||||
|
bucket = RUSTFS_META_BUCKET,
|
||||||
|
object = POOL_META_NAME,
|
||||||
|
drives_healed = result.drives_healed(),
|
||||||
|
drives_total = result.drives_reported(),
|
||||||
|
result = "pool_metadata_ok",
|
||||||
|
"Heal cluster pool metadata repaired"
|
||||||
|
);
|
||||||
|
self.record_result_item(result).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Ok((result, Some(err))) => {
|
||||||
|
self.record_result_item(result).await;
|
||||||
|
warn!(
|
||||||
|
target: "rustfs::heal::task",
|
||||||
|
event = EVENT_HEAL_BUCKET_RESULT,
|
||||||
|
component = LOG_COMPONENT_HEAL,
|
||||||
|
subsystem = LOG_SUBSYSTEM_TASK,
|
||||||
|
task_id = %self.id,
|
||||||
|
bucket = RUSTFS_META_BUCKET,
|
||||||
|
object = POOL_META_NAME,
|
||||||
|
result = "pool_metadata_failed",
|
||||||
|
error = %err,
|
||||||
|
"Heal cluster pool metadata failed"
|
||||||
|
);
|
||||||
|
Err(Error::TaskExecutionFailed {
|
||||||
|
message: format!("Failed to heal cluster pool metadata: {err}"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||||
|
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||||
|
Err(err) => {
|
||||||
|
warn!(
|
||||||
|
target: "rustfs::heal::task",
|
||||||
|
event = EVENT_HEAL_BUCKET_RESULT,
|
||||||
|
component = LOG_COMPONENT_HEAL,
|
||||||
|
subsystem = LOG_SUBSYSTEM_TASK,
|
||||||
|
task_id = %self.id,
|
||||||
|
bucket = RUSTFS_META_BUCKET,
|
||||||
|
object = POOL_META_NAME,
|
||||||
|
result = "pool_metadata_failed",
|
||||||
|
error = %err,
|
||||||
|
"Heal cluster pool metadata failed"
|
||||||
|
);
|
||||||
|
Err(Error::TaskExecutionFailed {
|
||||||
|
message: format!("Failed to heal cluster pool metadata: {err}"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) async fn heal_prefix(&self, bucket: &str, prefix: &str) -> Result<()> {
|
pub(super) async fn heal_prefix(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||||
debug!(
|
debug!(
|
||||||
target: "rustfs::heal::task",
|
target: "rustfs::heal::task",
|
||||||
@@ -403,6 +482,7 @@ impl HealTask {
|
|||||||
};
|
};
|
||||||
|
|
||||||
for (set_disk_id, heal_opts) in listing_scopes {
|
for (set_disk_id, heal_opts) in listing_scopes {
|
||||||
|
let bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, heal_opts.dry_run).await?;
|
||||||
let mut continuation_token: Option<String> = None;
|
let mut continuation_token: Option<String> = None;
|
||||||
let mut deferred = DeferredWindow::default();
|
let mut deferred = DeferredWindow::default();
|
||||||
let mut inline_retry: Option<DeferredObject> = None;
|
let mut inline_retry: Option<DeferredObject> = None;
|
||||||
@@ -501,13 +581,15 @@ impl HealTask {
|
|||||||
let retry_attempt = item.attempt;
|
let retry_attempt = item.attempt;
|
||||||
let mut telemetry_unknown = false;
|
let mut telemetry_unknown = false;
|
||||||
let object = item.name.as_str();
|
let object = item.name.as_str();
|
||||||
let identity =
|
let mut identity =
|
||||||
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||||
|
identity.bucket_incarnation_id = bucket_incarnation_id;
|
||||||
let mut disposition = if heal_opts.dry_run {
|
let mut disposition = if heal_opts.dry_run {
|
||||||
HealObjectDisposition::DryRunObserved
|
HealObjectDisposition::DryRunObserved
|
||||||
} else {
|
} else {
|
||||||
HealObjectDisposition::Unknown
|
HealObjectDisposition::Unknown
|
||||||
};
|
};
|
||||||
|
let mut recorded_authoritative_outcome = false;
|
||||||
let mut detail = None;
|
let mut detail = None;
|
||||||
{
|
{
|
||||||
let mut progress = self.progress.write().await;
|
let mut progress = self.progress.write().await;
|
||||||
@@ -520,23 +602,37 @@ impl HealTask {
|
|||||||
Some(Error::other("heal object retry age exhausted"))
|
Some(Error::other("heal object retry age exhausted"))
|
||||||
} else {
|
} else {
|
||||||
match self
|
match self
|
||||||
.await_with_control(
|
.await_with_control(self.storage.heal_object_with_receipt(
|
||||||
self.storage
|
bucket,
|
||||||
.heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
|
object,
|
||||||
)
|
item.version_id.as_deref(),
|
||||||
|
&heal_opts,
|
||||||
|
))
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok((result, None)) => match unavailable_recreate_error(&result, &heal_opts) {
|
Ok(storage_result) if storage_result.error.is_none() => {
|
||||||
|
match unavailable_recreate_error(&storage_result.item, &heal_opts) {
|
||||||
Some(error) => Some(error),
|
Some(error) => Some(error),
|
||||||
None => {
|
None => {
|
||||||
telemetry_unknown |= !increment_counter(&mut healed);
|
telemetry_unknown |= !increment_counter(&mut healed);
|
||||||
telemetry_unknown |=
|
telemetry_unknown |= !add_bytes(
|
||||||
!add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
|
&mut bytes,
|
||||||
self.record_result_item(result).await;
|
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
|
||||||
|
);
|
||||||
|
recorded_authoritative_outcome = self
|
||||||
|
.record_verified_storage_receipt(identity.clone(), storage_result.receipt)
|
||||||
|
.await;
|
||||||
|
self.record_result_item(storage_result.item).await;
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
|
}
|
||||||
|
Ok(storage_result)
|
||||||
|
if storage_result
|
||||||
|
.error
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|err| is_missing_object_dir_heal_result(object, err)) =>
|
||||||
|
{
|
||||||
telemetry_unknown |= !increment_counter(&mut healed);
|
telemetry_unknown |= !increment_counter(&mut healed);
|
||||||
debug!(
|
debug!(
|
||||||
target: "rustfs::heal::task",
|
target: "rustfs::heal::task",
|
||||||
@@ -551,7 +647,8 @@ impl HealTask {
|
|||||||
);
|
);
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
Ok((_, Some(err))) | Err(err) => Some(err),
|
Ok(storage_result) => storage_result.error,
|
||||||
|
Err(err) => Some(err),
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -674,11 +771,13 @@ impl HealTask {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !recorded_authoritative_outcome {
|
||||||
self.outcome.write().await.record(HealObjectOutcome {
|
self.outcome.write().await.record(HealObjectOutcome {
|
||||||
identity,
|
identity,
|
||||||
disposition,
|
disposition,
|
||||||
detail,
|
detail,
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
let mut progress = self.progress.write().await;
|
let mut progress = self.progress.write().await;
|
||||||
progress.update_object_progress(
|
progress.update_object_progress(
|
||||||
|
|||||||
@@ -451,6 +451,11 @@ impl HealTask {
|
|||||||
self.source,
|
self.source,
|
||||||
)
|
)
|
||||||
.with_replacement_targets(replacement_targets, is_auto_replacement.then(|| self.id.clone()))
|
.with_replacement_targets(replacement_targets, is_auto_replacement.then(|| self.id.clone()))
|
||||||
|
.with_pool_metadata_targets(if self.options.recreate_missing && !self.options.dry_run {
|
||||||
|
self.heal_endpoints.clone()
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
})
|
||||||
.with_replacement_identity_fence(replacement_target_identities.clone())
|
.with_replacement_identity_fence(replacement_target_identities.clone())
|
||||||
.with_mainline_pacer(self.mainline_pacer.clone());
|
.with_mainline_pacer(self.mainline_pacer.clone());
|
||||||
|
|
||||||
|
|||||||
@@ -162,6 +162,9 @@ impl HealTask {
|
|||||||
pool: self.options.pool_index,
|
pool: self.options.pool_index,
|
||||||
set: self.options.set_index,
|
set: self.options.set_index,
|
||||||
};
|
};
|
||||||
|
let mut expected_identity =
|
||||||
|
self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
||||||
|
expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?;
|
||||||
|
|
||||||
let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
|
let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
|
||||||
let heal_result = if self.source == HealRequestSource::ReadRepair {
|
let heal_result = if self.source == HealRequestSource::ReadRepair {
|
||||||
@@ -266,8 +269,6 @@ impl HealTask {
|
|||||||
let mut progress = self.progress.write().await;
|
let mut progress = self.progress.write().await;
|
||||||
progress.update_object_progress(1, 1, 0, 0, object_size);
|
progress.update_object_progress(1, 1, 0, 0, object_size);
|
||||||
}
|
}
|
||||||
let expected_identity =
|
|
||||||
self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
|
||||||
self.record_verified_storage_receipt(expected_identity, storage_result.receipt)
|
self.record_verified_storage_receipt(expected_identity, storage_result.receipt)
|
||||||
.await;
|
.await;
|
||||||
self.record_result_item(result).await;
|
self.record_result_item(result).await;
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ mod canonical_outcome {
|
|||||||
assert_eq!(task.get_progress().await.objects_scanned, 2);
|
assert_eq!(task.get_progress().await.objects_scanned, 2);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||||
["object-a", "object-b"]
|
["object-a", "object-b", POOL_META_NAME]
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
storage.listing_tokens.lock().expect("listing tokens").as_slice(),
|
storage.listing_tokens.lock().expect("listing tokens").as_slice(),
|
||||||
@@ -154,6 +154,126 @@ mod canonical_outcome {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bucket_heal_records_matching_positive_storage_receipts() {
|
||||||
|
let incarnation = Uuid::new_v4();
|
||||||
|
let storage = Arc::new(MockStorage {
|
||||||
|
heal_object_receipts: Mutex::new(HashMap::from([
|
||||||
|
(
|
||||||
|
"object-a".to_string(),
|
||||||
|
VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"object-b".to_string(),
|
||||||
|
VecDeque::from([object_receipt("object-b", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||||
|
),
|
||||||
|
])),
|
||||||
|
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let task = bucket_task(storage);
|
||||||
|
|
||||||
|
task.execute()
|
||||||
|
.await
|
||||||
|
.expect("bucket heal should record verified object receipts");
|
||||||
|
|
||||||
|
let outcome = task.get_outcome().await;
|
||||||
|
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||||
|
assert_eq!(outcome.counters.healed, 2);
|
||||||
|
assert_eq!(outcome.counters.unknown, 0);
|
||||||
|
assert_eq!(outcome.objects.len(), 2);
|
||||||
|
assert!(outcome.objects.iter().all(|item| {
|
||||||
|
item.identity.bucket_incarnation_id == Some(incarnation) && item.disposition == HealObjectDisposition::Repaired
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bucket_heal_keeps_repairing_when_bucket_incarnation_is_unavailable() {
|
||||||
|
let storage = Arc::new(MockStorage {
|
||||||
|
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||||
|
"object-a".to_string(),
|
||||||
|
VecDeque::from([object_receipt(
|
||||||
|
"object-a",
|
||||||
|
None,
|
||||||
|
HealObjectDisposition::Repaired,
|
||||||
|
Uuid::new_v4(),
|
||||||
|
)]),
|
||||||
|
)])),
|
||||||
|
bucket_incarnation_unavailable: Mutex::new(true),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let task = bucket_task(storage.clone());
|
||||||
|
|
||||||
|
task.execute()
|
||||||
|
.await
|
||||||
|
.expect("bucket heal should continue when only proof ownership is unavailable");
|
||||||
|
|
||||||
|
let outcome = task.get_outcome().await;
|
||||||
|
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||||
|
assert_eq!(outcome.counters.healed, 0);
|
||||||
|
assert_eq!(outcome.counters.unknown, 2);
|
||||||
|
assert!(
|
||||||
|
outcome
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.all(|item| item.disposition == HealObjectDisposition::Unknown)
|
||||||
|
);
|
||||||
|
assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bucket_heal_rejects_stale_receipts_without_double_recording() {
|
||||||
|
let expected_incarnation = Uuid::new_v4();
|
||||||
|
let storage = Arc::new(MockStorage {
|
||||||
|
heal_object_receipts: Mutex::new(HashMap::from([
|
||||||
|
(
|
||||||
|
"object-a".to_string(),
|
||||||
|
VecDeque::from([object_receipt(
|
||||||
|
"object-a",
|
||||||
|
None,
|
||||||
|
HealObjectDisposition::Repaired,
|
||||||
|
Uuid::new_v4(),
|
||||||
|
)]),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"object-b".to_string(),
|
||||||
|
VecDeque::from([object_receipt(
|
||||||
|
"object-b",
|
||||||
|
None,
|
||||||
|
HealObjectDisposition::Repaired,
|
||||||
|
expected_incarnation,
|
||||||
|
)]),
|
||||||
|
),
|
||||||
|
])),
|
||||||
|
bucket_incarnation_id: Mutex::new(Some(expected_incarnation)),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let task = bucket_task(storage);
|
||||||
|
|
||||||
|
task.execute()
|
||||||
|
.await
|
||||||
|
.expect("stale bucket receipt should not fail the legacy heal");
|
||||||
|
|
||||||
|
let outcome = task.get_outcome().await;
|
||||||
|
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||||
|
assert_eq!(outcome.counters.healed, 1);
|
||||||
|
assert_eq!(outcome.counters.unknown, 1);
|
||||||
|
assert_eq!(outcome.objects.len(), 2);
|
||||||
|
let object_a = outcome
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.find(|item| item.identity.object == "object-a")
|
||||||
|
.expect("stale receipt object outcome");
|
||||||
|
assert_eq!(object_a.disposition, HealObjectDisposition::Unknown);
|
||||||
|
let object_b = outcome
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.find(|item| item.identity.object == "object-b")
|
||||||
|
.expect("matching receipt object outcome");
|
||||||
|
assert_eq!(object_b.disposition, HealObjectDisposition::Repaired);
|
||||||
|
assert_eq!(object_b.identity.bucket_incarnation_id, Some(expected_incarnation));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() {
|
async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() {
|
||||||
let storage = Arc::new(MockStorage::default());
|
let storage = Arc::new(MockStorage::default());
|
||||||
@@ -313,6 +433,96 @@ mod canonical_outcome {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn mixed_grace_and_repaired_receipt_transfer_only_repaired_responsibility() {
|
||||||
|
let incarnation = Uuid::new_v4();
|
||||||
|
let storage = Arc::new(MockStorage {
|
||||||
|
heal_object_outcomes: Mutex::new(HashMap::from([(
|
||||||
|
"object-a".to_string(),
|
||||||
|
VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred]),
|
||||||
|
)])),
|
||||||
|
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||||
|
"object-b".to_string(),
|
||||||
|
VecDeque::from([object_receipt("object-b", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||||
|
)])),
|
||||||
|
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let task = bucket_task(storage);
|
||||||
|
|
||||||
|
task.execute()
|
||||||
|
.await
|
||||||
|
.expect("mixed grace and repaired receipt should complete");
|
||||||
|
|
||||||
|
let outcome = task.get_outcome().await;
|
||||||
|
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||||
|
assert_eq!(outcome.counters.processed, 2);
|
||||||
|
assert_eq!(outcome.counters.healed, 1);
|
||||||
|
assert_eq!(outcome.counters.skipped, 1);
|
||||||
|
let deferred = outcome
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.find(|item| item.identity.object == "object-a")
|
||||||
|
.expect("grace object should remain recorded");
|
||||||
|
assert!(matches!(
|
||||||
|
deferred.disposition,
|
||||||
|
HealObjectDisposition::Deferred {
|
||||||
|
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
));
|
||||||
|
assert_ne!(deferred.disposition, HealObjectDisposition::Repaired);
|
||||||
|
let repaired = outcome
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.find(|item| item.identity.object == "object-b")
|
||||||
|
.expect("receipt-backed object should be recorded");
|
||||||
|
assert_eq!(repaired.identity.bucket_incarnation_id, Some(incarnation));
|
||||||
|
assert_eq!(repaired.disposition, HealObjectDisposition::Repaired);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bucket_heal_records_matching_positive_storage_receipt() {
|
||||||
|
let incarnation = Uuid::new_v4();
|
||||||
|
let storage = Arc::new(MockStorage {
|
||||||
|
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||||
|
"object-a".to_string(),
|
||||||
|
VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||||
|
)])),
|
||||||
|
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let task = bucket_task(storage);
|
||||||
|
|
||||||
|
task.execute().await.expect("bucket heal should complete");
|
||||||
|
|
||||||
|
let outcome = task.get_outcome().await;
|
||||||
|
assert_eq!(outcome.counters.processed, 2);
|
||||||
|
assert_eq!(outcome.counters.healed, 1);
|
||||||
|
assert_eq!(outcome.counters.unknown, 1);
|
||||||
|
assert_eq!(
|
||||||
|
outcome
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.filter(|item| item.identity.object == "object-a")
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
let repaired = outcome
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.find(|item| item.identity.object == "object-a")
|
||||||
|
.expect("receipt-backed bucket object should be recorded");
|
||||||
|
assert_eq!(repaired.identity.bucket_incarnation_id, Some(incarnation));
|
||||||
|
assert_eq!(repaired.disposition, HealObjectDisposition::Repaired);
|
||||||
|
let legacy = outcome
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.find(|item| item.identity.object == "object-b")
|
||||||
|
.expect("legacy bucket object should still be recorded");
|
||||||
|
assert_eq!(legacy.disposition, HealObjectDisposition::Unknown);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn grace_single_object_is_completed_but_deferred() {
|
async fn grace_single_object_is_completed_but_deferred() {
|
||||||
let storage = Arc::new(MockStorage {
|
let storage = Arc::new(MockStorage {
|
||||||
@@ -344,6 +554,18 @@ mod canonical_outcome {
|
|||||||
.lock()
|
.lock()
|
||||||
.expect("existence fixture")
|
.expect("existence fixture")
|
||||||
.insert("object".to_string(), MockObjectExists::TransientSkip("retry later"));
|
.insert("object".to_string(), MockObjectExists::TransientSkip("retry later"));
|
||||||
|
} else {
|
||||||
|
let incarnation = Uuid::new_v4();
|
||||||
|
*storage.bucket_incarnation_id.lock().expect("bucket incarnation") = Some(incarnation);
|
||||||
|
storage.heal_object_receipts.lock().expect("receipt fixture").insert(
|
||||||
|
"object".to_string(),
|
||||||
|
VecDeque::from([object_receipt(
|
||||||
|
"object",
|
||||||
|
None,
|
||||||
|
HealObjectDisposition::VerifiedHealthy,
|
||||||
|
incarnation,
|
||||||
|
)]),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
|
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
|
||||||
request.options.dry_run = !transient;
|
request.options.dry_run = !transient;
|
||||||
@@ -1116,6 +1338,9 @@ struct MockStorage {
|
|||||||
heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
|
heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
|
||||||
heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
|
heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
|
||||||
heal_object_receipts: Mutex<HashMap<String, VecDeque<HealObjectReceipt>>>,
|
heal_object_receipts: Mutex<HashMap<String, VecDeque<HealObjectReceipt>>>,
|
||||||
|
bucket_incarnation_id: Mutex<Option<Uuid>>,
|
||||||
|
bucket_incarnation_after_object_heal: Mutex<Option<Uuid>>,
|
||||||
|
bucket_incarnation_unavailable: Mutex<bool>,
|
||||||
format_no_heal_required: Mutex<bool>,
|
format_no_heal_required: Mutex<bool>,
|
||||||
format_error: Mutex<Option<Error>>,
|
format_error: Mutex<Option<Error>>,
|
||||||
global_format_calls: Mutex<u32>,
|
global_format_calls: Mutex<u32>,
|
||||||
@@ -1219,14 +1444,19 @@ async fn execute_emits_heal_trace_task_state() {
|
|||||||
assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed"));
|
assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed"));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn object_receipt(object: &str, version_id: Option<&str>, disposition: HealObjectDisposition) -> HealObjectReceipt {
|
fn object_receipt(
|
||||||
|
object: &str,
|
||||||
|
version_id: Option<&str>,
|
||||||
|
disposition: HealObjectDisposition,
|
||||||
|
bucket_incarnation_id: Uuid,
|
||||||
|
) -> HealObjectReceipt {
|
||||||
HealObjectReceipt {
|
HealObjectReceipt {
|
||||||
identity: HealObjectIdentity {
|
identity: HealObjectIdentity {
|
||||||
kind: HealObjectKind::Object,
|
kind: HealObjectKind::Object,
|
||||||
bucket: "bucket-a".to_string(),
|
bucket: "bucket-a".to_string(),
|
||||||
object: object.to_string(),
|
object: object.to_string(),
|
||||||
version_id: version_id.map(ToOwned::to_owned),
|
version_id: version_id.map(ToOwned::to_owned),
|
||||||
bucket_incarnation_id: Some(Uuid::new_v4()),
|
bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||||
pool_index: None,
|
pool_index: None,
|
||||||
set_index: None,
|
set_index: None,
|
||||||
},
|
},
|
||||||
@@ -1236,11 +1466,18 @@ fn object_receipt(object: &str, version_id: Option<&str>, disposition: HealObjec
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn object_heal_records_matching_positive_storage_receipt() {
|
async fn object_heal_records_matching_positive_storage_receipt() {
|
||||||
|
let incarnation = Uuid::new_v4();
|
||||||
let storage = Arc::new(MockStorage {
|
let storage = Arc::new(MockStorage {
|
||||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||||
"object-a".to_string(),
|
"object-a".to_string(),
|
||||||
VecDeque::from([object_receipt("object-a", Some("version-a"), HealObjectDisposition::Repaired)]),
|
VecDeque::from([object_receipt(
|
||||||
|
"object-a",
|
||||||
|
Some("version-a"),
|
||||||
|
HealObjectDisposition::Repaired,
|
||||||
|
incarnation,
|
||||||
|
)]),
|
||||||
)])),
|
)])),
|
||||||
|
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
});
|
||||||
let task = HealTask::from_request(
|
let task = HealTask::from_request(
|
||||||
@@ -1261,16 +1498,165 @@ async fn object_heal_records_matching_positive_storage_receipt() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn object_heal_rejects_mismatched_or_legacy_storage_receipts() {
|
async fn cancelled_object_heal_rejects_matching_positive_storage_receipt() {
|
||||||
|
let incarnation = Uuid::new_v4();
|
||||||
|
let storage = Arc::new(MockStorage::default());
|
||||||
|
let task = HealTask::from_request(
|
||||||
|
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||||
|
storage,
|
||||||
|
);
|
||||||
|
task.cancel().await.expect("task cancellation should succeed");
|
||||||
|
|
||||||
|
let expected = HealObjectIdentity {
|
||||||
|
kind: HealObjectKind::Object,
|
||||||
|
bucket: "bucket-a".to_string(),
|
||||||
|
object: "object-a".to_string(),
|
||||||
|
version_id: Some("version-a".to_string()),
|
||||||
|
bucket_incarnation_id: Some(incarnation),
|
||||||
|
pool_index: None,
|
||||||
|
set_index: None,
|
||||||
|
};
|
||||||
|
let accepted = task
|
||||||
|
.record_verified_storage_receipt(
|
||||||
|
expected,
|
||||||
|
Some(object_receipt(
|
||||||
|
"object-a",
|
||||||
|
Some("version-a"),
|
||||||
|
HealObjectDisposition::Repaired,
|
||||||
|
incarnation,
|
||||||
|
)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let outcome = task.get_outcome().await;
|
||||||
|
assert!(!accepted);
|
||||||
|
assert_eq!(outcome.counters.healed, 0);
|
||||||
|
assert!(outcome.objects.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_object_heal_rejects_matching_positive_storage_receipt() {
|
||||||
|
let incarnation = Uuid::new_v4();
|
||||||
|
let storage = Arc::new(MockStorage {
|
||||||
|
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithOtherError("commit failed"))),
|
||||||
|
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||||
|
"object-a".to_string(),
|
||||||
|
VecDeque::from([object_receipt(
|
||||||
|
"object-a",
|
||||||
|
Some("version-a"),
|
||||||
|
HealObjectDisposition::Repaired,
|
||||||
|
incarnation,
|
||||||
|
)]),
|
||||||
|
)])),
|
||||||
|
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let task = HealTask::from_request(
|
||||||
|
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||||
|
storage,
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = task.execute().await;
|
||||||
|
|
||||||
|
let outcome = task.get_outcome().await;
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(outcome.counters.healed, 0);
|
||||||
|
assert_eq!(outcome.counters.unchanged, 0);
|
||||||
|
assert!(outcome.objects.iter().all(|object| {
|
||||||
|
!matches!(
|
||||||
|
object.disposition,
|
||||||
|
HealObjectDisposition::Repaired
|
||||||
|
| HealObjectDisposition::VerifiedHealthy
|
||||||
|
| HealObjectDisposition::AuthoritativelyAbsent
|
||||||
|
)
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn transient_quorum_object_heal_rejects_matching_positive_storage_receipt() {
|
||||||
|
let incarnation = Uuid::new_v4();
|
||||||
|
let storage = Arc::new(MockStorage {
|
||||||
|
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithReadQuorum)),
|
||||||
|
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||||
|
"object-a".to_string(),
|
||||||
|
VecDeque::from([object_receipt(
|
||||||
|
"object-a",
|
||||||
|
Some("version-a"),
|
||||||
|
HealObjectDisposition::Repaired,
|
||||||
|
incarnation,
|
||||||
|
)]),
|
||||||
|
)])),
|
||||||
|
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let task = HealTask::from_request(
|
||||||
|
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||||
|
storage,
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = task.execute().await;
|
||||||
|
|
||||||
|
let outcome = task.get_outcome().await;
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(outcome.counters.healed, 0);
|
||||||
|
assert_eq!(outcome.counters.unchanged, 0);
|
||||||
|
assert!(outcome.objects.iter().all(|object| {
|
||||||
|
!matches!(
|
||||||
|
object.disposition,
|
||||||
|
HealObjectDisposition::Repaired
|
||||||
|
| HealObjectDisposition::VerifiedHealthy
|
||||||
|
| HealObjectDisposition::AuthoritativelyAbsent
|
||||||
|
)
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn object_heal_latches_expected_incarnation_before_repair() {
|
||||||
|
let original_incarnation = Uuid::new_v4();
|
||||||
|
let successor_incarnation = Uuid::new_v4();
|
||||||
let storage = Arc::new(MockStorage {
|
let storage = Arc::new(MockStorage {
|
||||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||||
"object-a".to_string(),
|
"object-a".to_string(),
|
||||||
VecDeque::from([object_receipt(
|
VecDeque::from([object_receipt(
|
||||||
"object-a",
|
"object-a",
|
||||||
Some("old-version"),
|
Some("version-a"),
|
||||||
HealObjectDisposition::Repaired,
|
HealObjectDisposition::VerifiedHealthy,
|
||||||
|
original_incarnation,
|
||||||
)]),
|
)]),
|
||||||
)])),
|
)])),
|
||||||
|
bucket_incarnation_id: Mutex::new(Some(original_incarnation)),
|
||||||
|
bucket_incarnation_after_object_heal: Mutex::new(Some(successor_incarnation)),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let task = HealTask::from_request(
|
||||||
|
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||||
|
storage,
|
||||||
|
);
|
||||||
|
|
||||||
|
task.execute().await.expect("mock object heal should complete");
|
||||||
|
|
||||||
|
let outcome = task.get_outcome().await;
|
||||||
|
assert_eq!(outcome.counters.unchanged, 1);
|
||||||
|
assert_eq!(outcome.counters.unknown, 0);
|
||||||
|
let object = outcome.objects.front().expect("latched receipt should be recorded");
|
||||||
|
assert_eq!(object.identity.bucket_incarnation_id, Some(original_incarnation));
|
||||||
|
assert_eq!(object.disposition, HealObjectDisposition::VerifiedHealthy);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn object_heal_rejects_mismatched_or_legacy_storage_receipts() {
|
||||||
|
let expected_incarnation = Uuid::new_v4();
|
||||||
|
let storage = Arc::new(MockStorage {
|
||||||
|
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||||
|
"object-a".to_string(),
|
||||||
|
VecDeque::from([object_receipt(
|
||||||
|
"object-a",
|
||||||
|
Some("version-a"),
|
||||||
|
HealObjectDisposition::Repaired,
|
||||||
|
Uuid::new_v4(),
|
||||||
|
)]),
|
||||||
|
)])),
|
||||||
|
bucket_incarnation_id: Mutex::new(Some(expected_incarnation)),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
});
|
||||||
let task = HealTask::from_request(
|
let task = HealTask::from_request(
|
||||||
@@ -1361,6 +1747,7 @@ enum MockHealObjectOutcome {
|
|||||||
RetryableLock,
|
RetryableLock,
|
||||||
RetryableLockTimeout,
|
RetryableLockTimeout,
|
||||||
OkWithOtherError(&'static str),
|
OkWithOtherError(&'static str),
|
||||||
|
OkWithReadQuorum,
|
||||||
ErrOther(&'static str),
|
ErrOther(&'static str),
|
||||||
DanglingGraceDeferred,
|
DanglingGraceDeferred,
|
||||||
UnavailableDrive(DriveState),
|
UnavailableDrive(DriveState),
|
||||||
@@ -1456,6 +1843,13 @@ impl HealStorageAPI for MockStorage {
|
|||||||
Ok(self.object_exists.lock().unwrap().unwrap_or(true))
|
Ok(self.object_exists.lock().unwrap().unwrap_or(true))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||||
|
if *self.bucket_incarnation_unavailable.lock().unwrap() {
|
||||||
|
return Err(Error::Other("bucket incarnation unavailable".to_string()));
|
||||||
|
}
|
||||||
|
Ok(*self.bucket_incarnation_id.lock().unwrap())
|
||||||
|
}
|
||||||
|
|
||||||
async fn heal_object(
|
async fn heal_object(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -1513,6 +1907,13 @@ impl HealStorageAPI for MockStorage {
|
|||||||
MockHealObjectOutcome::RetryableSlowDown => {
|
MockHealObjectOutcome::RetryableSlowDown => {
|
||||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||||
}
|
}
|
||||||
|
MockHealObjectOutcome::OkWithReadQuorum => Ok((
|
||||||
|
HealResultItem::default(),
|
||||||
|
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||||
|
bucket.to_string(),
|
||||||
|
object.to_string(),
|
||||||
|
))),
|
||||||
|
)),
|
||||||
MockHealObjectOutcome::PermanentOther(message) => Err(Error::other(message)),
|
MockHealObjectOutcome::PermanentOther(message) => Err(Error::other(message)),
|
||||||
MockHealObjectOutcome::OkWithOtherError(message) => Ok((HealResultItem::default(), Some(Error::other(message)))),
|
MockHealObjectOutcome::OkWithOtherError(message) => Ok((HealResultItem::default(), Some(Error::other(message)))),
|
||||||
MockHealObjectOutcome::ErrOther(message) => Err(Error::other(message)),
|
MockHealObjectOutcome::ErrOther(message) => Err(Error::other(message)),
|
||||||
@@ -1552,6 +1953,13 @@ impl HealStorageAPI for MockStorage {
|
|||||||
MockHealObjectOutcome::RetryableSlowDown => {
|
MockHealObjectOutcome::RetryableSlowDown => {
|
||||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||||
}
|
}
|
||||||
|
MockHealObjectOutcome::OkWithReadQuorum => Ok((
|
||||||
|
HealResultItem::default(),
|
||||||
|
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||||
|
bucket.to_string(),
|
||||||
|
object.to_string(),
|
||||||
|
))),
|
||||||
|
)),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if bucket == RUSTFS_META_BUCKET && object == format!("{BUCKET_META_PREFIX}/{DATA_USAGE_CACHE_NAME}") {
|
if bucket == RUSTFS_META_BUCKET && object == format!("{BUCKET_META_PREFIX}/{DATA_USAGE_CACHE_NAME}") {
|
||||||
@@ -1566,6 +1974,9 @@ impl HealStorageAPI for MockStorage {
|
|||||||
return Ok((HealResultItem::default(), Some(Error::Disk(DiskError::FileNotFound))));
|
return Ok((HealResultItem::default(), Some(Error::Disk(DiskError::FileNotFound))));
|
||||||
}
|
}
|
||||||
self.healed_objects.lock().unwrap().push(object.to_string());
|
self.healed_objects.lock().unwrap().push(object.to_string());
|
||||||
|
if let Some(bucket_incarnation_id) = self.bucket_incarnation_after_object_heal.lock().unwrap().take() {
|
||||||
|
*self.bucket_incarnation_id.lock().unwrap() = Some(bucket_incarnation_id);
|
||||||
|
}
|
||||||
Ok((
|
Ok((
|
||||||
HealResultItem {
|
HealResultItem {
|
||||||
object_size: 1,
|
object_size: 1,
|
||||||
@@ -2171,11 +2582,95 @@ async fn test_cluster_heal_visits_bucket_objects() {
|
|||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
storage.healed_objects.lock().unwrap().as_slice(),
|
storage.healed_objects.lock().unwrap().as_slice(),
|
||||||
["object-a".to_string(), "object-b".to_string()]
|
["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()]
|
||||||
);
|
);
|
||||||
assert!(matches!(task.get_status().await, HealTaskStatus::Completed));
|
assert!(matches!(task.get_status().await, HealTaskStatus::Completed));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cluster_recreate_heals_pool_metadata_after_user_buckets() {
|
||||||
|
let storage = Arc::new(MockStorage::default());
|
||||||
|
let request = HealRequest::new(
|
||||||
|
HealType::Cluster,
|
||||||
|
HealOptions {
|
||||||
|
recursive: true,
|
||||||
|
recreate_missing: true,
|
||||||
|
timeout: None,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
HealPriority::Normal,
|
||||||
|
);
|
||||||
|
let task = HealTask::from_request(request, storage.clone());
|
||||||
|
|
||||||
|
task.execute()
|
||||||
|
.await
|
||||||
|
.expect("cluster recreate heal should include pool metadata");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||||
|
["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()]
|
||||||
|
);
|
||||||
|
let opts = storage.object_heal_opts.lock().expect("object opts");
|
||||||
|
assert!(opts.last().expect("pool metadata opts").recreate);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cluster_recreate_fails_when_pool_metadata_heal_fails() {
|
||||||
|
let storage = Arc::new(MockStorage::default());
|
||||||
|
storage.heal_object_outcomes.lock().expect("object outcomes").insert(
|
||||||
|
POOL_META_NAME.to_string(),
|
||||||
|
VecDeque::from([MockHealObjectOutcome::ErrOther("pool metadata missing")]),
|
||||||
|
);
|
||||||
|
let request = HealRequest::new(
|
||||||
|
HealType::Cluster,
|
||||||
|
HealOptions {
|
||||||
|
recursive: true,
|
||||||
|
recreate_missing: true,
|
||||||
|
timeout: None,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
HealPriority::Normal,
|
||||||
|
);
|
||||||
|
let task = HealTask::from_request(request, storage.clone());
|
||||||
|
|
||||||
|
let err = task
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.expect_err("cluster recreate heal must not hide pool metadata failure");
|
||||||
|
|
||||||
|
assert!(matches!(err, Error::TaskExecutionFailed { .. }));
|
||||||
|
assert_eq!(
|
||||||
|
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||||
|
["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cluster_dry_run_does_not_heal_pool_metadata() {
|
||||||
|
let storage = Arc::new(MockStorage::default());
|
||||||
|
let request = HealRequest::new(
|
||||||
|
HealType::Cluster,
|
||||||
|
HealOptions {
|
||||||
|
recursive: true,
|
||||||
|
dry_run: true,
|
||||||
|
recreate_missing: true,
|
||||||
|
timeout: None,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
HealPriority::Normal,
|
||||||
|
);
|
||||||
|
let task = HealTask::from_request(request, storage.clone());
|
||||||
|
|
||||||
|
task.execute()
|
||||||
|
.await
|
||||||
|
.expect("dry-run cluster heal should preserve existing coverage");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||||
|
["object-a".to_string(), "object-b".to_string()]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn object_heal_skips_dangling_delete_grace_without_failing_task() {
|
async fn object_heal_skips_dangling_delete_grace_without_failing_task() {
|
||||||
let storage = Arc::new(MockStorage {
|
let storage = Arc::new(MockStorage {
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ use rustfs_heal::heal::{
|
|||||||
storage::{ECStoreHealStorage, HealStorageAPI},
|
storage::{ECStoreHealStorage, HealStorageAPI},
|
||||||
};
|
};
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
use std::{
|
use std::{
|
||||||
fs::{File, OpenOptions},
|
fs::{File, OpenOptions},
|
||||||
@@ -48,6 +49,12 @@ use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, Pool
|
|||||||
const META_BUCKET: &str = ".rustfs.sys";
|
const META_BUCKET: &str = ".rustfs.sys";
|
||||||
const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
|
const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
|
||||||
const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin";
|
const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin";
|
||||||
|
const COMMITTED_PAYLOAD_REL: &str = ".heal-mrf-snapshot.0.bin";
|
||||||
|
const COMMITTED_MANIFEST_REL: &str = ".heal-mrf-commit.0.bin";
|
||||||
|
const COMMITTED_PAYLOAD_RELS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
|
||||||
|
const COMMITTED_MANIFEST_RELS: [&str; 2] = [".heal-mrf-commit.0.bin", ".heal-mrf-commit.1.bin"];
|
||||||
|
const COMMITTED_MAGIC: &[u8; 8] = b"RFMRFC01";
|
||||||
|
const COMMITTED_MANIFEST_LEN: usize = 8 + 1 + 16 + 8 + 8 + 32 + 32;
|
||||||
|
|
||||||
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
|
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
|
||||||
heal_env_at(None).await
|
heal_env_at(None).await
|
||||||
@@ -195,6 +202,29 @@ fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
|
|||||||
write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
|
write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn committed_manifest(owner: uuid::Uuid, sequence: u64, payload: &[u8]) -> Vec<u8> {
|
||||||
|
let mut manifest = Vec::with_capacity(COMMITTED_MANIFEST_LEN);
|
||||||
|
manifest.extend_from_slice(COMMITTED_MAGIC);
|
||||||
|
manifest.push(1);
|
||||||
|
manifest.extend_from_slice(owner.as_bytes());
|
||||||
|
manifest.extend_from_slice(&sequence.to_le_bytes());
|
||||||
|
manifest.extend_from_slice(
|
||||||
|
&u64::try_from(payload.len())
|
||||||
|
.expect("fixture payload length fits")
|
||||||
|
.to_le_bytes(),
|
||||||
|
);
|
||||||
|
manifest.extend_from_slice(&Sha256::digest(payload));
|
||||||
|
manifest.extend_from_slice(&Sha256::digest(&manifest));
|
||||||
|
assert_eq!(manifest.len(), COMMITTED_MANIFEST_LEN, "committed fixture manifest length");
|
||||||
|
manifest
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_committed_snapshot_to_disks(disk_paths: &[std::path::PathBuf], sequence: u64, payload: &[u8]) {
|
||||||
|
let manifest = committed_manifest(uuid::Uuid::new_v4(), sequence, payload);
|
||||||
|
write_journal_path_to_disks(disk_paths, COMMITTED_PAYLOAD_REL, payload);
|
||||||
|
write_journal_path_to_disks(disk_paths, COMMITTED_MANIFEST_REL, &manifest);
|
||||||
|
}
|
||||||
|
|
||||||
fn journal_exists_on_all_disks(disk_paths: &[std::path::PathBuf], relative_path: &str) -> bool {
|
fn journal_exists_on_all_disks(disk_paths: &[std::path::PathBuf], relative_path: &str) -> bool {
|
||||||
disk_paths
|
disk_paths
|
||||||
.iter()
|
.iter()
|
||||||
@@ -207,6 +237,44 @@ fn journal_matches_on_all_disks(disk_paths: &[PathBuf], relative_path: &str, exp
|
|||||||
.all(|path| std::fs::read(path.join(META_BUCKET).join(relative_path)).is_ok_and(|actual| actual == expected))
|
.all(|path| std::fs::read(path.join(META_BUCKET).join(relative_path)).is_ok_and(|actual| actual == expected))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn committed_checkpoint_matches_on_all_disks(disk_paths: &[PathBuf], sequence: u64, expected_payload: &[u8]) -> bool {
|
||||||
|
disk_paths.iter().all(|path| {
|
||||||
|
let root = path.join(META_BUCKET);
|
||||||
|
COMMITTED_PAYLOAD_RELS
|
||||||
|
.into_iter()
|
||||||
|
.zip(COMMITTED_MANIFEST_RELS)
|
||||||
|
.any(|(payload_rel, manifest_rel)| {
|
||||||
|
let Ok(payload) = std::fs::read(root.join(payload_rel)) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if payload != expected_payload {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Ok(manifest) = std::fs::read(root.join(manifest_rel)) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if manifest.len() != COMMITTED_MANIFEST_LEN || &manifest[..8] != COMMITTED_MAGIC || manifest[8] != 1 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Ok(recorded_sequence) = <[u8; 8]>::try_from(&manifest[25..33]).map(u64::from_le_bytes) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Ok(recorded_len) = <[u8; 8]>::try_from(&manifest[33..41]).map(u64::from_le_bytes) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Ok(expected_len) = u64::try_from(expected_payload.len()) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if recorded_sequence != sequence || recorded_len != expected_len {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let payload_digest: [u8; 32] = Sha256::digest(expected_payload).into();
|
||||||
|
let manifest_digest: [u8; 32] = Sha256::digest(&manifest[..COMMITTED_MANIFEST_LEN - 32]).into();
|
||||||
|
payload_digest.as_slice() == &manifest[41..73] && manifest_digest.as_slice() == &manifest[73..]
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
|
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
|
||||||
where
|
where
|
||||||
F: FnMut() -> Fut,
|
F: FnMut() -> Fut,
|
||||||
@@ -255,11 +323,12 @@ async fn decode_failure_intent_maps_to_urgent_mrf_heal_request() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// A journal left behind by a previous process must be replayed into the
|
/// A journal left behind by a previous process must be replayed into the
|
||||||
/// manager queue and then removed, and a torn tail must not block replay of
|
/// manager queue, and a torn tail must not block replay of the intact records.
|
||||||
/// the intact records.
|
/// The partial-write record keeps the legacy journal as the durable anchor
|
||||||
|
/// until an exact verified repair proof can discharge it.
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn journal_replay_arms_intents_and_deletes_the_file() {
|
async fn journal_replay_arms_intents_and_retains_unproven_partial_write_anchor() {
|
||||||
let (disk_paths, storage) = heal_env().await;
|
let (disk_paths, storage) = heal_env().await;
|
||||||
|
|
||||||
// The journal reader resolves disks through the process-local disk map;
|
// The journal reader resolves disks through the process-local disk map;
|
||||||
@@ -285,14 +354,14 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
|
|||||||
assert!(
|
assert!(
|
||||||
disk_paths
|
disk_paths
|
||||||
.iter()
|
.iter()
|
||||||
.all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
|
.all(|path| Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
|
||||||
"the journal file must be removed after a successful replay"
|
"partial-write replay must retain the legacy journal until durable proof"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
disk_paths
|
disk_paths
|
||||||
.iter()
|
.iter()
|
||||||
.all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
|
.all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
|
||||||
"the authoritative journal file must also be removed after replay"
|
"missing authoritative journal remains absent"
|
||||||
);
|
);
|
||||||
|
|
||||||
let snapshot = manager.operations_snapshot().await;
|
let snapshot = manager.operations_snapshot().await;
|
||||||
@@ -300,6 +369,76 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
|
|||||||
assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
|
assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A committed checkpoint published by the new two-slot writer is the
|
||||||
|
/// authoritative startup snapshot. Legacy mirrors are fallback-only and must
|
||||||
|
/// not be merged with or preferred over the committed epoch.
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
#[serial]
|
||||||
|
async fn committed_snapshot_replay_takes_precedence_over_stale_legacy_mirror() {
|
||||||
|
let (disk_paths, storage) = heal_env().await;
|
||||||
|
register_local_disks(&disk_paths, "mrf-committed-replay-test").await;
|
||||||
|
|
||||||
|
let committed = scoped_journal_record(3, "committed-bucket", "committed-object", Some([9u8; 16]), 0, 0, 0);
|
||||||
|
let stale_legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0);
|
||||||
|
write_committed_snapshot_to_disks(&disk_paths, 7, &committed);
|
||||||
|
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy);
|
||||||
|
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy);
|
||||||
|
|
||||||
|
let manager = make_manager(storage);
|
||||||
|
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||||
|
assert_eq!(replayed, 1, "only the committed snapshot epoch may replay");
|
||||||
|
|
||||||
|
let snapshot = manager.operations_snapshot().await;
|
||||||
|
assert_eq!(snapshot.queued_by_source.mrf, 1);
|
||||||
|
assert_eq!(
|
||||||
|
snapshot.queued_by_priority.normal, 1,
|
||||||
|
"the committed partial-write record must replay instead of the stale legacy decode-failure"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
snapshot.queued_by_priority.urgent, 0,
|
||||||
|
"stale legacy decode-failure records must not be mixed into committed replay"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
journal_exists_on_all_disks(&disk_paths, COMMITTED_MANIFEST_REL),
|
||||||
|
"the committed checkpoint remains until the accepted partial-write has proof"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A damaged committed checkpoint is ambiguous: replay must not fall back to
|
||||||
|
/// older legacy bytes or delete any recovery anchor until another process can
|
||||||
|
/// publish a valid successor.
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
#[serial]
|
||||||
|
async fn damaged_committed_snapshot_blocks_legacy_fallback_and_retains_anchors() {
|
||||||
|
let (disk_paths, storage) = heal_env().await;
|
||||||
|
register_local_disks(&disk_paths, "mrf-damaged-committed-replay-test").await;
|
||||||
|
|
||||||
|
let committed = scoped_journal_record(3, "damaged-committed-bucket", "committed-object", Some([8u8; 16]), 0, 0, 0);
|
||||||
|
let stale_legacy = journal_record(1, "damaged-legacy-bucket", "legacy-object", None, 0);
|
||||||
|
write_journal_path_to_disks(&disk_paths, COMMITTED_PAYLOAD_REL, &committed);
|
||||||
|
let mut manifest = committed_manifest(uuid::Uuid::new_v4(), 9, &committed);
|
||||||
|
manifest[25] ^= 1;
|
||||||
|
write_journal_path_to_disks(&disk_paths, COMMITTED_MANIFEST_REL, &manifest);
|
||||||
|
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy);
|
||||||
|
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy);
|
||||||
|
|
||||||
|
let manager = make_manager(storage);
|
||||||
|
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||||
|
assert_eq!(replayed, 0, "damaged committed state must fail closed");
|
||||||
|
assert_eq!(
|
||||||
|
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||||
|
0,
|
||||||
|
"stale legacy bytes must not be replayed when committed state is ambiguous"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
journal_exists_on_all_disks(&disk_paths, COMMITTED_MANIFEST_REL)
|
||||||
|
&& journal_exists_on_all_disks(&disk_paths, COMMITTED_PAYLOAD_REL)
|
||||||
|
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy)
|
||||||
|
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &stale_legacy),
|
||||||
|
"all recovery anchors must remain after a fail-closed committed read"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// A canonical snapshot and its compatibility mirror may differ after a
|
/// A canonical snapshot and its compatibility mirror may differ after a
|
||||||
/// partial flush. Replay must choose the complete canonical epoch instead of
|
/// partial flush. Replay must choose the complete canonical epoch instead of
|
||||||
/// combining records that never coexisted in memory.
|
/// combining records that never coexisted in memory.
|
||||||
@@ -322,21 +461,24 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
|||||||
assert_eq!(snapshot.queued_by_source.mrf, 1);
|
assert_eq!(snapshot.queued_by_source.mrf, 1);
|
||||||
assert!(
|
assert!(
|
||||||
disk_paths.iter().all(|path| {
|
disk_paths.iter().all(|path| {
|
||||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||||
}),
|
}),
|
||||||
"replay cleanup must remove both journal paths"
|
"accepted replay responsibilities remain anchored until a verified repair proof"
|
||||||
);
|
);
|
||||||
|
|
||||||
// A scoped-only snapshot is valid during a rollout where no legacy
|
// A scoped-only snapshot is valid during a rollout where no legacy
|
||||||
// compatibility mirror was written. Missing legacy files must not leave
|
// compatibility mirror was written. Missing legacy files must not leave
|
||||||
// the runtime in a permanent cleanup-retry state.
|
// the runtime in a permanent cleanup-retry state.
|
||||||
|
let (disk_paths, storage) = heal_env().await;
|
||||||
|
register_local_disks(&disk_paths, "mrf-scoped-authoritative-test").await;
|
||||||
|
let manager = make_manager(storage);
|
||||||
let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0);
|
let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0);
|
||||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only);
|
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only);
|
||||||
assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1);
|
assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1);
|
||||||
assert!(disk_paths.iter().all(|path| {
|
assert!(disk_paths.iter().all(|path| {
|
||||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||||
}));
|
}));
|
||||||
|
|
||||||
let scoped_v2 = scoped_journal_record(1, "scoped-v2-bucket", "scoped-v2-object", None, 0, 3, 7);
|
let scoped_v2 = scoped_journal_record(1, "scoped-v2-bucket", "scoped-v2-object", None, 0, 3, 7);
|
||||||
@@ -350,12 +492,12 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
|||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
manager.operations_snapshot().await.queued_by_source.mrf,
|
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||||
3,
|
2,
|
||||||
"only the three authoritative/scoped-only epochs should have reached the manager"
|
"only the scoped-only and scoped-v2 authoritative epochs should have reached the manager"
|
||||||
);
|
);
|
||||||
assert!(disk_paths.iter().all(|path| {
|
assert!(disk_paths.iter().all(|path| {
|
||||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -400,10 +542,13 @@ async fn authoritative_journal_replay_preserves_kind_and_scope_identity() {
|
|||||||
snapshot.queued_by_priority.urgent, 1,
|
snapshot.queued_by_priority.urgent, 1,
|
||||||
"decode-failure repair must not merge with object repair responsibility"
|
"decode-failure repair must not merge with object repair responsibility"
|
||||||
);
|
);
|
||||||
assert!(disk_paths.iter().all(|path| {
|
assert!(
|
||||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
disk_paths.iter().all(|path| {
|
||||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||||
}));
|
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||||
|
}),
|
||||||
|
"partial-write responsibilities keep both replay anchors until proof"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// If replay reaches a full heal-manager queue, the old journal remains the
|
/// If replay reaches a full heal-manager queue, the old journal remains the
|
||||||
@@ -524,13 +669,14 @@ fn mrf_successor_flush_child_process_fixture() {
|
|||||||
let expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
let expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||||
let flushed = wait_until(Duration::from_secs(10), || async {
|
let flushed = wait_until(Duration::from_secs(10), || async {
|
||||||
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
||||||
|
&& committed_checkpoint_matches_on_all_disks(&disk_paths, 2, &expected_successor)
|
||||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
||||||
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor)
|
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor)
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
assert!(
|
assert!(
|
||||||
flushed,
|
flushed,
|
||||||
"child process must publish the pending successor snapshot before the delete phase"
|
"child process must publish the committed pending successor before the delete phase"
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
std::process::exit(78);
|
std::process::exit(78);
|
||||||
@@ -571,13 +717,14 @@ fn mrf_successor_flush_waiting_child_process_fixture() {
|
|||||||
let expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
let expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||||
let flushed = wait_until(Duration::from_secs(10), || async {
|
let flushed = wait_until(Duration::from_secs(10), || async {
|
||||||
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
||||||
|
&& committed_checkpoint_matches_on_all_disks(&disk_paths, 2, &expected_successor)
|
||||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
||||||
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor)
|
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor)
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
assert!(
|
assert!(
|
||||||
flushed,
|
flushed,
|
||||||
"child process must publish the pending successor snapshot before it can be killed"
|
"child process must publish the committed pending successor before it can be killed"
|
||||||
);
|
);
|
||||||
std::fs::write(&ready_path, b"ready").expect("write ready marker");
|
std::fs::write(&ready_path, b"ready").expect("write ready marker");
|
||||||
loop {
|
loop {
|
||||||
@@ -698,10 +845,10 @@ async fn journal_replay_survives_successor_flush_before_delete() {
|
|||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
disk_paths.iter().all(|path| {
|
disk_paths.iter().all(|path| {
|
||||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||||
}),
|
}),
|
||||||
"a fully consumed successor snapshot may be deleted after restart replay"
|
"the accepted successor remains anchored until a verified repair proof"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -751,10 +898,10 @@ async fn journal_replay_survives_service_kill_after_successor_flush() {
|
|||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
disk_paths.iter().all(|path| {
|
disk_paths.iter().all(|path| {
|
||||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||||
}),
|
}),
|
||||||
"a fully consumed successor snapshot may be deleted after service-kill restart replay"
|
"the accepted successor remains anchored until a verified repair proof after service-kill restart"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -814,9 +961,9 @@ async fn journal_replay_survives_sigkill_after_authoritative_successor_fsync_bef
|
|||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
disk_paths.iter().all(|path| {
|
disk_paths.iter().all(|path| {
|
||||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||||
}),
|
}),
|
||||||
"a fully consumed authoritative successor may clean both epochs after restart replay"
|
"the accepted authoritative successor remains anchored until a verified repair proof"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -263,6 +263,14 @@ pub struct ScannerUsageRecoveryIntentResponse {
|
|||||||
pub mode: String,
|
pub mode: String,
|
||||||
pub intent_id: String,
|
pub intent_id: String,
|
||||||
pub state: String,
|
pub state: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub actor_sha256: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub idempotency_key_sha256: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub request_sha256: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub accepted_at_unix_secs: Option<u64>,
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||||
}
|
}
|
||||||
@@ -869,6 +877,10 @@ mod tests {
|
|||||||
"mode": "full-rebuild",
|
"mode": "full-rebuild",
|
||||||
"intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
"intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||||
"state": "accepted",
|
"state": "accepted",
|
||||||
|
"actor_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
|
||||||
|
"idempotency_key_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
|
||||||
|
"request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
|
||||||
|
"accepted_at_unix_secs": 7,
|
||||||
"future": {"worker": "pending"}
|
"future": {"worker": "pending"}
|
||||||
}))
|
}))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -877,7 +889,33 @@ mod tests {
|
|||||||
assert_eq!(intent.mode, "full-rebuild");
|
assert_eq!(intent.mode, "full-rebuild");
|
||||||
assert_eq!(intent.intent_id, "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef");
|
assert_eq!(intent.intent_id, "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef");
|
||||||
assert_eq!(intent.state, "accepted");
|
assert_eq!(intent.state, "accepted");
|
||||||
|
assert_eq!(
|
||||||
|
intent.actor_sha256.as_deref(),
|
||||||
|
Some("1111111111111111111111111111111111111111111111111111111111111111")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
intent.idempotency_key_sha256.as_deref(),
|
||||||
|
Some("2222222222222222222222222222222222222222222222222222222222222222")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
intent.request_sha256.as_deref(),
|
||||||
|
Some("3333333333333333333333333333333333333333333333333333333333333333")
|
||||||
|
);
|
||||||
|
assert_eq!(intent.accepted_at_unix_secs, Some(7));
|
||||||
assert_eq!(intent.extra["future"]["worker"], "pending");
|
assert_eq!(intent.extra["future"]["worker"], "pending");
|
||||||
|
|
||||||
|
let legacy_intent: ScannerUsageRecoveryIntentResponse = serde_json::from_value(json!({
|
||||||
|
"status": "accepted",
|
||||||
|
"action": "usage-full-rebuild",
|
||||||
|
"mode": "full-rebuild",
|
||||||
|
"intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||||
|
"state": "accepted"
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
assert!(legacy_intent.actor_sha256.is_none());
|
||||||
|
assert!(legacy_intent.idempotency_key_sha256.is_none());
|
||||||
|
assert!(legacy_intent.request_sha256.is_none());
|
||||||
|
assert!(legacy_intent.accepted_at_unix_secs.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -971,7 +1009,7 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn scanner_usage_async_reset_posts_explicit_intent_contract() {
|
async fn scanner_usage_async_reset_posts_explicit_intent_contract() {
|
||||||
let server = TestServer::spawn(
|
let server = TestServer::spawn(
|
||||||
r#"{"status":"accepted","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"accepted"}"#,
|
r#"{"status":"accepted","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"accepted","actor_sha256":"1111111111111111111111111111111111111111111111111111111111111111","idempotency_key_sha256":"2222222222222222222222222222222222222222222222222222222222222222","request_sha256":"3333333333333333333333333333333333333333333333333333333333333333","accepted_at_unix_secs":7}"#,
|
||||||
202,
|
202,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -985,6 +1023,7 @@ mod tests {
|
|||||||
assert_eq!(accepted.status, "accepted");
|
assert_eq!(accepted.status, "accepted");
|
||||||
assert_eq!(accepted.mode, "full-rebuild");
|
assert_eq!(accepted.mode, "full-rebuild");
|
||||||
assert_eq!(accepted.state, "accepted");
|
assert_eq!(accepted.state, "accepted");
|
||||||
|
assert_eq!(accepted.accepted_at_unix_secs, Some(7));
|
||||||
let request = server.recorded();
|
let request = server.recorded();
|
||||||
assert_eq!(request.method, "POST");
|
assert_eq!(request.method, "POST");
|
||||||
assert_eq!(request.path, "/rustfs/admin/v3/scanner/usage-state/reset");
|
assert_eq!(request.path, "/rustfs/admin/v3/scanner/usage-state/reset");
|
||||||
|
|||||||
@@ -97,9 +97,11 @@ pub use scanner::{
|
|||||||
pub use scanner_io::{
|
pub use scanner_io::{
|
||||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||||
record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot,
|
record_dirty_usage_bucket_from_producer, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||||
scanner_dirty_usage_state, scanner_maintenance_generation,
|
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||||
|
scanner_maintenance_generation,
|
||||||
};
|
};
|
||||||
|
pub use segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||||
|
|||||||
@@ -3,7 +3,8 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::segment_invalidation::{
|
use crate::segment_invalidation::{
|
||||||
MAX_SEGMENT_INVALIDATION_BYTES, MAX_SEGMENT_INVALIDATION_ENTRIES, SegmentInvalidationDomain, SegmentInvalidationEnvelope,
|
MAX_SEGMENT_INVALIDATION_BYTES, MAX_SEGMENT_INVALIDATION_ENTRIES, SegmentInvalidationDomain, SegmentInvalidationEnvelope,
|
||||||
SegmentInvalidationError, SegmentInvalidationProducer, SegmentInvalidationProof, admit_segment_invalidation,
|
SegmentInvalidationError, SegmentInvalidationProducer, SegmentInvalidationProducerIdentity, SegmentInvalidationProof,
|
||||||
|
admit_segment_invalidation, complete_segment_invalidation_producers,
|
||||||
};
|
};
|
||||||
use std::collections::BTreeSet;
|
use std::collections::BTreeSet;
|
||||||
|
|
||||||
@@ -11,7 +12,8 @@ const MAX_WALK_SAMPLES: usize = 32;
|
|||||||
const MAX_WALK_BYTES: usize = 1024;
|
const MAX_WALK_BYTES: usize = 1024;
|
||||||
|
|
||||||
fn segment_producers() -> BTreeSet<SegmentInvalidationProducer> {
|
fn segment_producers() -> BTreeSet<SegmentInvalidationProducer> {
|
||||||
SegmentInvalidationProducer::REQUIRED.into_iter().collect()
|
complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION)
|
||||||
|
.expect("fixture should enumerate the complete production producer matrix")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn segment_envelope() -> SegmentInvalidationEnvelope {
|
fn segment_envelope() -> SegmentInvalidationEnvelope {
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ use s3s::dto::{
|
|||||||
BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration, VersioningConfiguration,
|
BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration, VersioningConfiguration,
|
||||||
};
|
};
|
||||||
use sha2::{Digest as _, Sha256};
|
use sha2::{Digest as _, Sha256};
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
@@ -1224,8 +1224,9 @@ pub(crate) use cache::{
|
|||||||
pub use dirty_usage::{
|
pub use dirty_usage::{
|
||||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||||
record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot,
|
record_dirty_usage_bucket_from_producer, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||||
scanner_dirty_usage_state, scanner_maintenance_generation,
|
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||||
|
scanner_maintenance_generation,
|
||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||||
|
|||||||
@@ -22,6 +22,11 @@ pub(super) static DIRTY_USAGE_BUCKETS: LazyLock<StdMutex<DirtyUsageBuckets>> = L
|
|||||||
// matching scope.
|
// matching scope.
|
||||||
pub(super) static DIRTY_USAGE_BUCKET_SCOPES: LazyLock<StdMutex<DirtyUsageBucketScopes>> =
|
pub(super) static DIRTY_USAGE_BUCKET_SCOPES: LazyLock<StdMutex<DirtyUsageBucketScopes>> =
|
||||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||||
|
// Non-authoritative process-local producer coverage. Any future segment reuse
|
||||||
|
// activation must bind this to the exact generation window and durable proof.
|
||||||
|
pub(super) static DIRTY_USAGE_PRODUCER_IDENTITIES: LazyLock<
|
||||||
|
StdMutex<BTreeSet<crate::segment_invalidation::SegmentInvalidationProducerIdentity>>,
|
||||||
|
> = LazyLock::new(|| StdMutex::new(BTreeSet::new()));
|
||||||
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||||
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||||
pub(super) static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
pub(super) static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||||
@@ -153,6 +158,7 @@ fn apply_scoped_dirty_usage_ack(
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod scoped_dirty_usage_tests {
|
mod scoped_dirty_usage_tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
|
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
|
||||||
@@ -213,6 +219,30 @@ mod scoped_dirty_usage_tests {
|
|||||||
assert_eq!(scopes, original_scopes);
|
assert_eq!(scopes, original_scopes);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dirty_usage_tracks_known_segment_producer_identities_without_authorizing_unknown_sources() {
|
||||||
|
clear_dirty_usage_buckets_for_tests();
|
||||||
|
record_dirty_usage_object_from_producer("photos", "hot/object", SegmentInvalidationProducerIdentity::PutObject);
|
||||||
|
record_dirty_usage_object_from_producer("photos", "archive/object", SegmentInvalidationProducerIdentity::DeleteObject);
|
||||||
|
record_dirty_usage_bucket_from_producer("photos", SegmentInvalidationProducerIdentity::Unknown);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
dirty_usage_producer_identities_for_tests(),
|
||||||
|
BTreeSet::from([
|
||||||
|
SegmentInvalidationProducerIdentity::PutObject,
|
||||||
|
SegmentInvalidationProducerIdentity::DeleteObject
|
||||||
|
])
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
dirty_usage_bucket_scopes_for_tests().get("photos"),
|
||||||
|
Some(&DirtyUsageBucketScope::WholeBucket),
|
||||||
|
"an unknown producer keeps the bucket dirty but must not count as producer coverage"
|
||||||
|
);
|
||||||
|
|
||||||
|
clear_dirty_usage_buckets_for_tests();
|
||||||
|
assert!(dirty_usage_producer_identities_for_tests().is_empty());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||||
@@ -225,6 +255,13 @@ fn dirty_usage_bucket_scopes() -> MutexGuard<'static, DirtyUsageBucketScopes> {
|
|||||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn dirty_usage_producer_identities()
|
||||||
|
-> MutexGuard<'static, BTreeSet<crate::segment_invalidation::SegmentInvalidationProducerIdentity>> {
|
||||||
|
DIRTY_USAGE_PRODUCER_IDENTITIES
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn usize_to_u64_saturated(value: usize) -> u64 {
|
pub(super) fn usize_to_u64_saturated(value: usize) -> u64 {
|
||||||
u64::try_from(value).unwrap_or(u64::MAX)
|
u64::try_from(value).unwrap_or(u64::MAX)
|
||||||
}
|
}
|
||||||
@@ -240,6 +277,22 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
record_dirty_usage_bucket_inner(bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_dirty_usage_bucket_from_producer(
|
||||||
|
bucket: &str,
|
||||||
|
producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity,
|
||||||
|
) {
|
||||||
|
if bucket.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
record_segment_invalidation_producer_identity(producer);
|
||||||
|
record_dirty_usage_bucket_inner(bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_dirty_usage_bucket_inner(bucket: &str) {
|
||||||
let pending_buckets = {
|
let pending_buckets = {
|
||||||
let mut dirty_buckets = dirty_usage_buckets();
|
let mut dirty_buckets = dirty_usage_buckets();
|
||||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||||
@@ -263,6 +316,23 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
|||||||
/// local: after restart or any unverified distributed path the scanner falls
|
/// local: after restart or any unverified distributed path the scanner falls
|
||||||
/// back to its ordinary bucket scan.
|
/// back to its ordinary bucket scan.
|
||||||
pub fn record_dirty_usage_object(bucket: &str, object: &str) {
|
pub fn record_dirty_usage_object(bucket: &str, object: &str) {
|
||||||
|
record_dirty_usage_object_inner(bucket, object);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_dirty_usage_object_from_producer(
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity,
|
||||||
|
) {
|
||||||
|
if bucket.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
record_segment_invalidation_producer_identity(producer);
|
||||||
|
record_dirty_usage_object_inner(bucket, object);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_dirty_usage_object_inner(bucket: &str, object: &str) {
|
||||||
let Some(top_level_entry) = dirty_usage_top_level_entry(object) else {
|
let Some(top_level_entry) = dirty_usage_top_level_entry(object) else {
|
||||||
record_dirty_usage_bucket(bucket);
|
record_dirty_usage_bucket(bucket);
|
||||||
return;
|
return;
|
||||||
@@ -296,6 +366,17 @@ pub fn record_dirty_usage_object(bucket: &str, object: &str) {
|
|||||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn record_segment_invalidation_producer_identity(producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity) {
|
||||||
|
if producer.producer().is_some() {
|
||||||
|
dirty_usage_producer_identities().insert(producer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn dirty_usage_producer_identities_for_tests() -> BTreeSet<crate::segment_invalidation::SegmentInvalidationProducerIdentity> {
|
||||||
|
dirty_usage_producer_identities().clone()
|
||||||
|
}
|
||||||
|
|
||||||
fn dirty_usage_top_level_entry(object: &str) -> Option<String> {
|
fn dirty_usage_top_level_entry(object: &str) -> Option<String> {
|
||||||
let (top_level_entry, _) = object.split_once('/').unwrap_or((object, ""));
|
let (top_level_entry, _) = object.split_once('/').unwrap_or((object, ""));
|
||||||
(!top_level_entry.is_empty()
|
(!top_level_entry.is_empty()
|
||||||
@@ -577,6 +658,7 @@ pub(super) fn dirty_usage_bucket_count() -> usize {
|
|||||||
pub(crate) fn clear_dirty_usage_buckets_for_tests() {
|
pub(crate) fn clear_dirty_usage_buckets_for_tests() {
|
||||||
dirty_usage_buckets().clear();
|
dirty_usage_buckets().clear();
|
||||||
dirty_usage_bucket_scopes().clear();
|
dirty_usage_bucket_scopes().clear();
|
||||||
|
dirty_usage_producer_identities().clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -16,15 +16,20 @@ use super::*;
|
|||||||
use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision};
|
use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision};
|
||||||
|
|
||||||
async fn create_cohort_bucket(store: &ECStore, bucket: &str) {
|
async fn create_cohort_bucket(store: &ECStore, bucket: &str) {
|
||||||
|
create_cohort_bucket_objects(store, bucket, 1).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_cohort_bucket_objects(store: &ECStore, bucket: &str, objects: usize) {
|
||||||
store
|
store
|
||||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||||
.await
|
.await
|
||||||
.expect("fixture bucket");
|
.expect("fixture bucket");
|
||||||
for set in store.all_set_disks() {
|
for set in store.all_set_disks() {
|
||||||
|
for index in 0..objects {
|
||||||
let mut reader = ScannerPutObjReader::from_vec(b"cohort".to_vec());
|
let mut reader = ScannerPutObjReader::from_vec(b"cohort".to_vec());
|
||||||
set.put_object(
|
set.put_object(
|
||||||
bucket,
|
bucket,
|
||||||
"initial",
|
&format!("object-{index:04}"),
|
||||||
&mut reader,
|
&mut reader,
|
||||||
&ScannerObjectOptions {
|
&ScannerObjectOptions {
|
||||||
no_lock: true,
|
no_lock: true,
|
||||||
@@ -34,6 +39,7 @@ async fn create_cohort_bucket(store: &ECStore, bucket: &str) {
|
|||||||
.await
|
.await
|
||||||
.expect("fixture object and all rename tails should persist");
|
.expect("fixture object and all rename tails should persist");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_cohort_cycle(
|
async fn run_cohort_cycle(
|
||||||
@@ -168,6 +174,92 @@ async fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
|||||||
clear_dirty_usage_buckets_for_tests();
|
clear_dirty_usage_buckets_for_tests();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
|
||||||
|
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||||
|
clear_dirty_usage_buckets_for_tests();
|
||||||
|
let flat = format!("a-flat-{}", Uuid::new_v4().simple());
|
||||||
|
let small = format!("z-small-{}", Uuid::new_v4().simple());
|
||||||
|
create_cohort_bucket_objects(&store, &flat, 6).await;
|
||||||
|
create_cohort_bucket(&store, &small).await;
|
||||||
|
let cohort = Arc::new(StdMutex::new(ScannerServiceCohort::default()));
|
||||||
|
|
||||||
|
let expected_flat = store
|
||||||
|
.all_set_disks()
|
||||||
|
.iter()
|
||||||
|
.map(|set| (DataUsageCacheSource::new(set.pool_index, set.set_index), flat.clone()))
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
let expected_small = store
|
||||||
|
.all_set_disks()
|
||||||
|
.iter()
|
||||||
|
.map(|set| (DataUsageCacheSource::new(set.pool_index, set.set_index), small.clone()))
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
|
||||||
|
let ctx = CancellationToken::new();
|
||||||
|
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||||
|
&ctx,
|
||||||
|
ScannerCycleBudgetConfig {
|
||||||
|
max_objects: Some(1),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let (result, usage) = run_cohort_cycle(&store, cohort.clone(), 1, budget.clone()).await;
|
||||||
|
assert_eq!(result.status, ScannerCycleStatus::Incomplete);
|
||||||
|
assert!(usage.is_none(), "wide-bucket budget exhaustion must not publish a partial aggregate");
|
||||||
|
assert!(budget.budget_elapsed());
|
||||||
|
let first_round_admitted = cohort
|
||||||
|
.lock()
|
||||||
|
.expect("cohort lock")
|
||||||
|
.admitted_members()
|
||||||
|
.into_iter()
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
assert!(
|
||||||
|
!first_round_admitted.is_disjoint(&expected_flat),
|
||||||
|
"the first fixed budget round should exercise the wide flat bucket"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
first_round_admitted.is_disjoint(&expected_small),
|
||||||
|
"a small bucket not yet reached by the real scanner must not be marked admitted"
|
||||||
|
);
|
||||||
|
|
||||||
|
for cycle in 2..=4 {
|
||||||
|
let ctx = CancellationToken::new();
|
||||||
|
let (result, usage) = run_cohort_cycle(
|
||||||
|
&store,
|
||||||
|
cohort.clone(),
|
||||||
|
cycle,
|
||||||
|
ScannerCycleBudget::new_with_progress_tracking(
|
||||||
|
&ctx,
|
||||||
|
ScannerCycleBudgetConfig {
|
||||||
|
max_objects: Some(1),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(result.status, ScannerCycleStatus::Incomplete);
|
||||||
|
assert!(usage.is_none(), "mixed partial coverage still cannot publish the set root");
|
||||||
|
}
|
||||||
|
let admitted_after_budgeted_rounds = cohort
|
||||||
|
.lock()
|
||||||
|
.expect("cohort lock")
|
||||||
|
.admitted_members()
|
||||||
|
.into_iter()
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
assert!(
|
||||||
|
expected_small.is_subset(&admitted_after_budgeted_rounds),
|
||||||
|
"tracked small buckets must receive real execution opportunities within their fixed service-round bound"
|
||||||
|
);
|
||||||
|
|
||||||
|
let ctx = CancellationToken::new();
|
||||||
|
let (result, usage) =
|
||||||
|
run_cohort_cycle(&store, cohort, 5, ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default())).await;
|
||||||
|
assert_eq!(result.status, ScannerCycleStatus::Complete);
|
||||||
|
assert_eq!(usage.expect("final complete aggregate").objects_total_count, 14);
|
||||||
|
clear_dirty_usage_buckets_for_tests();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ pub enum SegmentInvalidationError {
|
|||||||
ByteLimit,
|
ByteLimit,
|
||||||
InvalidProof,
|
InvalidProof,
|
||||||
InvalidKey,
|
InvalidKey,
|
||||||
|
UnknownProducer,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||||
@@ -50,6 +51,74 @@ impl SegmentInvalidationProducer {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||||
|
pub enum SegmentInvalidationProducerIdentity {
|
||||||
|
PutObject,
|
||||||
|
DeleteObject,
|
||||||
|
DeleteMarker,
|
||||||
|
CompleteMultipartUpload,
|
||||||
|
Replication,
|
||||||
|
TierTransition,
|
||||||
|
TierExpiration,
|
||||||
|
DirectoryObject,
|
||||||
|
Unknown,
|
||||||
|
TestFixture,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SegmentInvalidationProducerIdentity {
|
||||||
|
pub const REQUIRED_PRODUCTION: [Self; 8] = [
|
||||||
|
Self::PutObject,
|
||||||
|
Self::DeleteObject,
|
||||||
|
Self::DeleteMarker,
|
||||||
|
Self::CompleteMultipartUpload,
|
||||||
|
Self::Replication,
|
||||||
|
Self::TierTransition,
|
||||||
|
Self::TierExpiration,
|
||||||
|
Self::DirectoryObject,
|
||||||
|
];
|
||||||
|
|
||||||
|
pub fn producer(self) -> Option<SegmentInvalidationProducer> {
|
||||||
|
match self {
|
||||||
|
Self::PutObject => Some(SegmentInvalidationProducer::Put),
|
||||||
|
Self::DeleteObject => Some(SegmentInvalidationProducer::Delete),
|
||||||
|
Self::DeleteMarker => Some(SegmentInvalidationProducer::DeleteMarker),
|
||||||
|
Self::CompleteMultipartUpload => Some(SegmentInvalidationProducer::Multipart),
|
||||||
|
Self::Replication => Some(SegmentInvalidationProducer::Replication),
|
||||||
|
Self::TierTransition | Self::TierExpiration => Some(SegmentInvalidationProducer::Tier),
|
||||||
|
Self::DirectoryObject => Some(SegmentInvalidationProducer::DirectoryObject),
|
||||||
|
Self::Unknown | Self::TestFixture => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn complete_segment_invalidation_producers<I>(
|
||||||
|
identities: I,
|
||||||
|
) -> Result<BTreeSet<SegmentInvalidationProducer>, SegmentInvalidationError>
|
||||||
|
where
|
||||||
|
I: IntoIterator<Item = SegmentInvalidationProducerIdentity>,
|
||||||
|
{
|
||||||
|
let mut covered_identities = BTreeSet::new();
|
||||||
|
let mut producers = BTreeSet::new();
|
||||||
|
for identity in identities {
|
||||||
|
let Some(producer) = identity.producer() else {
|
||||||
|
return Err(SegmentInvalidationError::UnknownProducer);
|
||||||
|
};
|
||||||
|
covered_identities.insert(identity);
|
||||||
|
producers.insert(producer);
|
||||||
|
}
|
||||||
|
if SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION
|
||||||
|
.iter()
|
||||||
|
.all(|identity| covered_identities.contains(identity))
|
||||||
|
&& SegmentInvalidationProducer::REQUIRED
|
||||||
|
.iter()
|
||||||
|
.all(|producer| producers.contains(producer))
|
||||||
|
{
|
||||||
|
Ok(producers)
|
||||||
|
} else {
|
||||||
|
Err(SegmentInvalidationError::InvalidProof)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
pub enum SegmentInvalidationDomain {
|
pub enum SegmentInvalidationDomain {
|
||||||
LocalSingleSet,
|
LocalSingleSet,
|
||||||
@@ -170,7 +239,8 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
fn producers() -> BTreeSet<SegmentInvalidationProducer> {
|
fn producers() -> BTreeSet<SegmentInvalidationProducer> {
|
||||||
SegmentInvalidationProducer::REQUIRED.into_iter().collect()
|
complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION)
|
||||||
|
.expect("production producer matrix should be complete")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn envelope() -> SegmentInvalidationEnvelope {
|
fn envelope() -> SegmentInvalidationEnvelope {
|
||||||
@@ -320,6 +390,63 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn segment_invalidation_producer_identities_must_be_known_and_complete() {
|
||||||
|
assert_eq!(
|
||||||
|
complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION),
|
||||||
|
Ok(SegmentInvalidationProducer::REQUIRED.into_iter().collect())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
complete_segment_invalidation_producers([
|
||||||
|
SegmentInvalidationProducerIdentity::PutObject,
|
||||||
|
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||||
|
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||||
|
SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||||
|
SegmentInvalidationProducerIdentity::Replication,
|
||||||
|
SegmentInvalidationProducerIdentity::TierTransition,
|
||||||
|
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||||
|
SegmentInvalidationProducerIdentity::Unknown,
|
||||||
|
]),
|
||||||
|
Err(SegmentInvalidationError::UnknownProducer)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
complete_segment_invalidation_producers([
|
||||||
|
SegmentInvalidationProducerIdentity::PutObject,
|
||||||
|
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||||
|
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||||
|
SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||||
|
SegmentInvalidationProducerIdentity::Replication,
|
||||||
|
SegmentInvalidationProducerIdentity::TierTransition,
|
||||||
|
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||||
|
SegmentInvalidationProducerIdentity::TestFixture,
|
||||||
|
]),
|
||||||
|
Err(SegmentInvalidationError::UnknownProducer)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
complete_segment_invalidation_producers([
|
||||||
|
SegmentInvalidationProducerIdentity::PutObject,
|
||||||
|
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||||
|
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||||
|
SegmentInvalidationProducerIdentity::Replication,
|
||||||
|
SegmentInvalidationProducerIdentity::TierTransition,
|
||||||
|
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||||
|
]),
|
||||||
|
Err(SegmentInvalidationError::InvalidProof)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
complete_segment_invalidation_producers([
|
||||||
|
SegmentInvalidationProducerIdentity::PutObject,
|
||||||
|
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||||
|
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||||
|
SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||||
|
SegmentInvalidationProducerIdentity::Replication,
|
||||||
|
SegmentInvalidationProducerIdentity::TierTransition,
|
||||||
|
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||||
|
]),
|
||||||
|
Err(SegmentInvalidationError::InvalidProof)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn segment_invalidation_entries_are_bounded_and_key_checked() {
|
fn segment_invalidation_entries_are_bounded_and_key_checked() {
|
||||||
let envelope = envelope();
|
let envelope = envelope();
|
||||||
|
|||||||
@@ -39,8 +39,8 @@ The `scanner` and `heal` subsystems are served by `GetConfigKVHandler` (`rustfs/
|
|||||||
The `--abba` mode runs five independent scenario cells: `cold-hot`, `fresh-hot`,
|
The `--abba` mode runs five independent scenario cells: `cold-hot`, `fresh-hot`,
|
||||||
`multi-hot-new`, `running-heal`, and `mrf-replay`. Each scenario runs at least
|
`multi-hot-new`, `running-heal`, and `mrf-replay`. Each scenario runs at least
|
||||||
three A1/B1/B2/A2 groups for both baseline/candidate with background work on,
|
three A1/B1/B2/A2 groups for both baseline/candidate with background work on,
|
||||||
and candidate-only background off/on. A measured leg lasts at least 900
|
and candidate-only background off/on. A measured release leg lasts at least 7200
|
||||||
seconds; the minimum matrix contains 120 legs (30 hours before setup/oracles).
|
seconds; the minimum matrix contains 120 legs (240 hours before setup/oracles).
|
||||||
The existing `performance-ab.yml` supplies the pattern for immutable build
|
The existing `performance-ab.yml` supplies the pattern for immutable build
|
||||||
provenance and failure propagation, but its short Warp workload is not this
|
provenance and failure propagation, but its short Warp workload is not this
|
||||||
scanner gate. No scheduled workflow starts this matrix automatically.
|
scanner gate. No scheduled workflow starts this matrix automatically.
|
||||||
@@ -63,9 +63,10 @@ The manifest has the following JSON contract (all fields are required):
|
|||||||
| Field | Value |
|
| Field | Value |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `schema`, `evidence` | `1`, and `measured` or `synthetic`. |
|
| `schema`, `evidence` | `1`, and `measured` or `synthetic`. |
|
||||||
| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 900..86400 seconds for measured runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. |
|
| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 7200..86400 seconds for measured release runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. |
|
||||||
| `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. |
|
| `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. |
|
||||||
| `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. |
|
| `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. |
|
||||||
|
| `release_evidence` | Required for `measured` runs. It binds the 3x4 EC8+4 topology, multi-pool/multi-set coverage, per-node metrics endpoints, same-window distributed sampling, process restart and crash-restart fault modes, mixed-version reader/writer/rollback participation, and allocation/flamegraph/RSS/save-frequency profile artifact requirements. Synthetic runs do not need this field and still cannot approve release evidence. |
|
||||||
| `oracles` | A map with all five scenario names. Each value contains positive integer `objects`, `versions`, `bytes`, and `sha256` of the independently prepared canonical object/version/content manifest. |
|
| `oracles` | A map with all five scenario names. Each value contains positive integer `objects`, `versions`, `bytes`, and `sha256` of the independently prepared canonical object/version/content manifest. |
|
||||||
| `expected_healed_objects` | A map with all five scenario names and independently seeded repair counts. Running-heal and MRF-replay require a positive count. |
|
| `expected_healed_objects` | A map with all five scenario names and independently seeded repair counts. Running-heal and MRF-replay require a positive count. |
|
||||||
|
|
||||||
@@ -75,6 +76,14 @@ object/version/content result. Fix the foreground arrival rate (offered load),
|
|||||||
cache preparation procedure, configuration, and hardware across every leg.
|
cache preparation procedure, configuration, and hardware across every leg.
|
||||||
Do not include credentials in the manifest, adapter output, or saved commands;
|
Do not include credentials in the manifest, adapter output, or saved commands;
|
||||||
the collector reads `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` from its environment.
|
the collector reads `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` from its environment.
|
||||||
|
The adapter must echo the measured run's `release_evidence` object in every
|
||||||
|
measurement response. A mismatch fails the cell because it means the deployment,
|
||||||
|
mixed-version set, crash mode, or profiler contract no longer matches the
|
||||||
|
operator-reviewed manifest. This echo is provenance binding only; it does not
|
||||||
|
replace the independent correctness oracle, distributed metrics samples, profile
|
||||||
|
artifacts, or ABBA comparison thresholds. The summary tool revalidates the same
|
||||||
|
manifest contract before it can print a measured PASS result, so hand-built or
|
||||||
|
trimmed reports without this provenance fail closed.
|
||||||
|
|
||||||
#### Deployment Adapter Contract
|
#### Deployment Adapter Contract
|
||||||
|
|
||||||
@@ -204,6 +213,14 @@ The command prints only `PASS scanner_heal_perf ...` for measured passing ABBA
|
|||||||
evidence, otherwise `FAIL scanner_heal_perf ...`. The JSON and Markdown outputs
|
evidence, otherwise `FAIL scanner_heal_perf ...`. The JSON and Markdown outputs
|
||||||
carry the key p99/throughput/P1/P2/cache-cost fields and artifact provenance
|
carry the key p99/throughput/P1/P2/cache-cost fields and artifact provenance
|
||||||
hashes; raw per-cell logs remain in the original artifact tree for audit.
|
hashes; raw per-cell logs remain in the original artifact tree for audit.
|
||||||
|
Failed or interrupted ABBA reports that contain only `status`, `performance`,
|
||||||
|
`completed_cells`, and `error` also summarize as `FAIL`; they do not become
|
||||||
|
performance evidence, and a missing comparison matrix is accepted only for a
|
||||||
|
non-passing report.
|
||||||
|
Measured passing reports must also retain the W10/W11 foreground-pressure,
|
||||||
|
heal-lock-wait, and heal-attempt-cost fields emitted by the ABBA evaluator. If
|
||||||
|
those fields are removed, empty, malformed, or length-mismatched, the quiet
|
||||||
|
summary fails closed instead of treating the report as performance evidence.
|
||||||
|
|
||||||
They cover the complete 120-cell schedule, data isolation, missing builds and
|
They cover the complete 120-cell schedule, data isolation, missing builds and
|
||||||
oracles, zero samples/requests, swallowed request errors, offered-load drift,
|
oracles, zero samples/requests, swallowed request errors, offered-load drift,
|
||||||
|
|||||||
+72
-12
@@ -148,11 +148,21 @@ and exact S3 content.
|
|||||||
This case is a **four-node, one-drive-per-node process-restart test**. It is not
|
This case is a **four-node, one-drive-per-node process-restart test**. It is not
|
||||||
power-loss validation, a 3x4 EC8+4 experiment, an all-version inventory, or proof
|
power-loss validation, a 3x4 EC8+4 experiment, an all-version inventory, or proof
|
||||||
of scanner enumeration, exact MRF disposition, legacy migration, or rollback.
|
of scanner enumeration, exact MRF disposition, legacy migration, or rollback.
|
||||||
The registry keeps all G01-G14/P1-P4 and R-E/R-D/R-L release requirements pending
|
The schema 2 registry separates the implemented single-set restart lane from
|
||||||
until their actual feature-specific oracles and required topologies exist.
|
structured release lanes for authority coverage, checkpoint/crash, status and
|
||||||
Missing cases cannot be supplied by synthetic W20 results. W20's bounded JSON
|
outcome, MRF responsibility, mixed-version rollback, scheduler pressure,
|
||||||
and file-hash helpers are reused; its ABBA performance contracts remain in
|
maintenance producers, and EC8+4 multi-set coverage. All G01-G14/P1-P4 and
|
||||||
|
R-E/R-D/R-L release requirements stay `pending` until their actual
|
||||||
|
feature-specific oracles, measurements and required topologies exist. Missing
|
||||||
|
cases cannot be supplied by synthetic W20 results. W20's bounded JSON and
|
||||||
|
file-hash helpers are reused; its ABBA performance contracts remain in
|
||||||
`docs/operations/scanner-benchmark-runbook.md`.
|
`docs/operations/scanner-benchmark-runbook.md`.
|
||||||
|
Measured ABBA manifests must also carry the runbook's `release_evidence`
|
||||||
|
contract. The runner rejects reports that cannot bind the exact 3x4 EC8+4
|
||||||
|
topology, multi-pool/multi-set shape, distributed same-window metrics endpoints,
|
||||||
|
restart/crash modes, mixed-version reader/writer/rollback participation, and
|
||||||
|
allocation/flamegraph/RSS/save-frequency profile artifact plan. Synthetic runs
|
||||||
|
and manifests missing that contract remain harness-only evidence.
|
||||||
|
|
||||||
### Recording One Case
|
### Recording One Case
|
||||||
|
|
||||||
@@ -232,14 +242,64 @@ For automation, `--check-scanner-heal-release "$RUN_DIR"` emits one compact
|
|||||||
JSON decision and exits nonzero while blocked. `verified_cases` contains only
|
JSON decision and exits nonzero while blocked. `verified_cases` contains only
|
||||||
cases that pass the complete receipt, build provenance, nextest/JUnit and real
|
cases that pass the complete receipt, build provenance, nextest/JUnit and real
|
||||||
oracle checks; `rejected_cases` names registered cases that do not, and
|
oracle checks; `rejected_cases` names registered cases that do not, and
|
||||||
`pending_gates` names the unimplemented release requirements. Approval requires
|
`pending_gates` names the unimplemented release requirements and
|
||||||
every registered case to verify, `pending_gates` to be empty, and a future
|
`pending_lanes` names the structured release lanes that still need real
|
||||||
registry schema capable of representing the complete release matrix. Schema 1
|
evidence. Schema 1 is deliberately marked `release_schema_capable: false`
|
||||||
is deliberately marked `release_schema_capable: false`: it models only the
|
because it models only the single-version, unversioned-object restart/crash
|
||||||
single-version, unversioned-object restart/crash cases and cannot represent
|
cases. Schema 2 can describe the wider release matrix, but approval still
|
||||||
mixed-version, rollback, EC8+4 or performance evidence. A focused run,
|
requires every registered case to verify and every required gate to leave
|
||||||
synthetic harness, compile-only result, skipped/retried test, ordinary CI
|
`pending` only after a future checker can bind it to real feature-specific
|
||||||
success, or removal of pending text therefore cannot become a release approval.
|
evidence. The current checker hard-rejects missing structured requirements and
|
||||||
|
pending gates mapped to an implemented lane, so clearing pending text cannot
|
||||||
|
become approval. A focused run, synthetic harness, compile-only result,
|
||||||
|
skipped/retried test, ordinary CI success, or unregistered mixed-version,
|
||||||
|
rollback, EC8+4 or performance claim therefore cannot become a release approval.
|
||||||
|
For high-risk rollback gates, `evidence_fields` records the specific proof
|
||||||
|
fields that a future real-evidence checker must bind before a pending gate can
|
||||||
|
move out of the blocked set. G03 keeps scoped ACK tied to durable root
|
||||||
|
publication, ACK request identity, participating peer capability snapshots, and
|
||||||
|
mixed-peer fallback oracles; G09 keeps mixed-version reader, writer, and rollback
|
||||||
|
payload evidence explicit. These fields are part of the release contract, not
|
||||||
|
evidence by themselves.
|
||||||
|
|
||||||
|
When the real release lanes have produced their dedicated artifacts, validate
|
||||||
|
the complete hard-gate bundle with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/python_bin.sh scripts/check_test_wiring.py \
|
||||||
|
--check-scanner-heal-release-bundle /path/to/release-evidence.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The bundle checker is intentionally stricter than the case checker. It requires
|
||||||
|
schema 2 registry metadata, `evidence: measured`, the current checkout revision,
|
||||||
|
all G01-G14/P1-P4/R-E/R-D/R-L gates, per-gate `status: pass`, lane identity,
|
||||||
|
relative artifact paths, matching SHA256 hashes, and non-empty summaries. It
|
||||||
|
also binds each evidence field to its own run provenance: `source_revision`,
|
||||||
|
`run_id`, `measurement_window_id`, timezone-qualified `started_at` and
|
||||||
|
`finished_at`, command arguments, and artifact format. The field
|
||||||
|
`source_revision` must match the bundle revision, and measured performance
|
||||||
|
duration cannot exceed the recorded run window.
|
||||||
|
|
||||||
|
The hard evidence shape remains claim-specific: mixed-version gates must name at
|
||||||
|
least two participating versions, crash/durable replay gates must include
|
||||||
|
crash-boundary evidence, G14 must record EC8+4 with at least three nodes and four
|
||||||
|
drives per node plus multi-set and multi-pool evidence, performance gates need
|
||||||
|
measured durations, P3's pressure run needs at least two hours, and P1 needs a
|
||||||
|
symbolized profile summary with resolved samples. Every G14 field and every
|
||||||
|
performance gate's fields must also share one `measurement_window_id`, so EC8+4,
|
||||||
|
multi-set/multi-pool, ABBA, throughput, and profiling artifacts cannot be
|
||||||
|
stitched together from unrelated runs. P1 `profile_evidence` must bind every
|
||||||
|
required profile artifact kind (`allocation-profile`, `flamegraph`,
|
||||||
|
`rss-samples`, and `save-frequency`) with a relative path, artifact format,
|
||||||
|
non-empty file, matching SHA256, and the same measurement window when a
|
||||||
|
per-artifact window is declared. Missing, synthetic, stale, tampered, undersized,
|
||||||
|
or topology-mismatched evidence returns a compact blocked or invalid JSON result
|
||||||
|
and a nonzero exit.
|
||||||
|
|
||||||
|
This command validates the evidence package; it does not create evidence. A
|
||||||
|
handwritten JSON file, a synthetic harness pass, a single focused case, or a
|
||||||
|
local unit fixture still cannot satisfy the distributed, mixed-version,
|
||||||
|
crash-restart, durable MRF replay, EC8+4, ABBA, or profiling gates.
|
||||||
|
|
||||||
Run parser/receipt regressions with
|
Run parser/receipt regressions with
|
||||||
`scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures
|
`scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures
|
||||||
|
|||||||
@@ -75,6 +75,10 @@ struct ScannerRecoveryIntentResponse {
|
|||||||
mode: String,
|
mode: String,
|
||||||
intent_id: String,
|
intent_id: String,
|
||||||
state: String,
|
state: String,
|
||||||
|
actor_sha256: String,
|
||||||
|
idempotency_key_sha256: String,
|
||||||
|
request_sha256: String,
|
||||||
|
accepted_at_unix_secs: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
@@ -332,6 +336,10 @@ fn scanner_recovery_intent_record_response(
|
|||||||
mode: record.mode,
|
mode: record.mode,
|
||||||
intent_id: record.intent_id,
|
intent_id: record.intent_id,
|
||||||
state: record.state,
|
state: record.state,
|
||||||
|
actor_sha256: record.actor_sha256,
|
||||||
|
idempotency_key_sha256: record.idempotency_key_sha256,
|
||||||
|
request_sha256: record.request_sha256,
|
||||||
|
accepted_at_unix_secs: record.accepted_at_unix_secs,
|
||||||
};
|
};
|
||||||
let body = serde_json::to_vec(&response).map_err(|err| {
|
let body = serde_json::to_vec(&response).map_err(|err| {
|
||||||
S3Error::with_message(
|
S3Error::with_message(
|
||||||
@@ -371,10 +379,7 @@ fn scanner_recovery_intent_accept_response(
|
|||||||
|
|
||||||
fn scanner_recovery_intent_executor_id(result: &rustfs_scanner::ScannerRecoveryIntentAcceptResult) -> Option<String> {
|
fn scanner_recovery_intent_executor_id(result: &rustfs_scanner::ScannerRecoveryIntentAcceptResult) -> Option<String> {
|
||||||
match result {
|
match result {
|
||||||
rustfs_scanner::ScannerRecoveryIntentAcceptResult::Accepted { record }
|
rustfs_scanner::ScannerRecoveryIntentAcceptResult::Accepted { record } if record.state == "accepted" => {
|
||||||
| rustfs_scanner::ScannerRecoveryIntentAcceptResult::Replayed { record }
|
|
||||||
if matches!(record.state.as_str(), "accepted" | "running") =>
|
|
||||||
{
|
|
||||||
Some(record.intent_id.clone())
|
Some(record.intent_id.clone())
|
||||||
}
|
}
|
||||||
_ => None,
|
_ => None,
|
||||||
@@ -673,7 +678,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scanner_recovery_intent_executor_only_starts_non_terminal_work() {
|
fn scanner_recovery_intent_executor_starts_only_newly_accepted_work() {
|
||||||
let mut record = rustfs_scanner::ScannerRecoveryIntentRecord {
|
let mut record = rustfs_scanner::ScannerRecoveryIntentRecord {
|
||||||
schema_version: 1,
|
schema_version: 1,
|
||||||
intent_id: "0".repeat(64),
|
intent_id: "0".repeat(64),
|
||||||
@@ -699,7 +704,16 @@ mod tests {
|
|||||||
record: record.clone(),
|
record: record.clone(),
|
||||||
})
|
})
|
||||||
.as_deref(),
|
.as_deref(),
|
||||||
Some(record.intent_id.as_str())
|
None,
|
||||||
|
"a lost-response retry must not start a duplicate executor"
|
||||||
|
);
|
||||||
|
record.state = "accepted".to_string();
|
||||||
|
assert!(
|
||||||
|
scanner_recovery_intent_executor_id(&rustfs_scanner::ScannerRecoveryIntentAcceptResult::Replayed {
|
||||||
|
record: record.clone(),
|
||||||
|
})
|
||||||
|
.is_none(),
|
||||||
|
"replayed accepted records remain durable for startup/control recovery instead of duplicating work"
|
||||||
);
|
);
|
||||||
record.state = "completed".to_string();
|
record.state = "completed".to_string();
|
||||||
assert!(
|
assert!(
|
||||||
|
|||||||
@@ -814,7 +814,11 @@ impl DefaultObjectUsecase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||||
|
&bucket,
|
||||||
|
&key,
|
||||||
|
rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject,
|
||||||
|
);
|
||||||
Ok::<_, S3Error>((oi, dest_versioned))
|
Ok::<_, S3Error>((oi, dest_versioned))
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1175,7 +1175,12 @@ impl DefaultObjectUsecase {
|
|||||||
let manager = get_capacity_manager();
|
let manager = get_capacity_manager();
|
||||||
manager.record_write_operation().await;
|
manager.record_write_operation().await;
|
||||||
let _ = helper.complete(&result);
|
let _ = helper.complete(&result);
|
||||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
let producer = if delete_marker && version_id_clone.is_none() {
|
||||||
|
rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteMarker
|
||||||
|
} else {
|
||||||
|
rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteObject
|
||||||
|
};
|
||||||
|
rustfs_scanner::record_dirty_usage_object_from_producer(&bucket, &key, producer);
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -689,7 +689,11 @@ impl DefaultObjectUsecase {
|
|||||||
schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await;
|
schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||||
|
&bucket,
|
||||||
|
&key,
|
||||||
|
rustfs_scanner::SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||||
|
);
|
||||||
Ok::<_, ApiError>(obj_info)
|
Ok::<_, ApiError>(obj_info)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2054,7 +2054,11 @@ impl DefaultObjectUsecase {
|
|||||||
schedule_object_replication(obj_info.clone(), store, dsc).await;
|
schedule_object_replication(obj_info.clone(), store, dsc).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||||
|
&bucket,
|
||||||
|
&key,
|
||||||
|
rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject,
|
||||||
|
);
|
||||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start);
|
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start);
|
||||||
|
|
||||||
let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||||
|
|||||||
@@ -3475,6 +3475,9 @@ mod tests {
|
|||||||
1,
|
1,
|
||||||
"post-admission response loss must leave exactly one canonical task"
|
"post-admission response loss must leave exactly one canonical task"
|
||||||
);
|
);
|
||||||
|
if let Some(cache) = super::HEAL_CONTROL_REPLAY_CACHE.get() {
|
||||||
|
cache.lock().await.clear();
|
||||||
|
}
|
||||||
|
|
||||||
let mut retry = connect_faulty_heal_control_client(
|
let mut retry = connect_faulty_heal_control_client(
|
||||||
Arc::clone(&manager),
|
Arc::clone(&manager),
|
||||||
|
|||||||
+776
-22
@@ -13,7 +13,7 @@ import tempfile
|
|||||||
import unittest
|
import unittest
|
||||||
import uuid
|
import uuid
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||||
@@ -23,10 +23,78 @@ try:
|
|||||||
except ModuleNotFoundError:
|
except ModuleNotFoundError:
|
||||||
import tomli as tomllib
|
import tomli as tomllib
|
||||||
|
|
||||||
from scanner_abba import MAX_JSON_BYTES, digest, number, read_json, require, sha, write_json
|
from scanner_abba import (
|
||||||
|
MAX_JSON_BYTES,
|
||||||
|
RELEASE_PROFILE_ARTIFACTS,
|
||||||
|
digest,
|
||||||
|
number,
|
||||||
|
read_json,
|
||||||
|
require,
|
||||||
|
sha,
|
||||||
|
write_json,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SCANNER_HEAL_REGISTRY_SCHEMA_MAX = 2
|
||||||
|
SCANNER_HEAL_RELEASE_REQUIRED_GATES = (
|
||||||
|
"G01", "G02", "G03", "G04", "G05", "G06", "G07", "G08", "G09", "G10", "G11", "G12", "G13", "G14",
|
||||||
|
"P1", "P2", "P3", "P4", "R-E", "R-D", "R-L",
|
||||||
|
)
|
||||||
|
SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS = {
|
||||||
|
"G03": (
|
||||||
|
"durable_root_publication_proof",
|
||||||
|
"scoped_ack_request_identity",
|
||||||
|
"participating_peer_capability_snapshot",
|
||||||
|
"mixed_peer_ack_fallback_oracle",
|
||||||
|
),
|
||||||
|
"G09": (
|
||||||
|
"mixed_version_reader_evidence",
|
||||||
|
"mixed_version_writer_evidence",
|
||||||
|
"rollback_payload_evidence",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS = {
|
||||||
|
"G01": ("root_authority_evidence", "quota_authority_evidence"),
|
||||||
|
"G02": ("bounded_checkpoint_oracle", "independent_version_inventory"),
|
||||||
|
"G03": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G03"],
|
||||||
|
"G04": ("cache_boundary_crash_evidence", "root_floor_intent_crash_evidence"),
|
||||||
|
"G05": ("per_object_outcome_oracle", "terminal_retention_bounds"),
|
||||||
|
"G06": ("concurrent_status_evidence", "legacy_client_compatibility", "truncation_behavior"),
|
||||||
|
"G07": ("mrf_responsibility_oracle", "commit_boundary_crash_matrix"),
|
||||||
|
"G08": ("mrf_capacity_evidence", "disk_full_matrix", "replica_loss_matrix"),
|
||||||
|
"G09": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G09"],
|
||||||
|
"G10": ("scheduler_bound_evidence", "pressure_recovery_evidence"),
|
||||||
|
"G11": ("maintenance_producer_matrix", "complete_producer_inventory"),
|
||||||
|
"G12": ("reset_quota_path_evidence", "settlement_quota_path_evidence"),
|
||||||
|
"G13": ("quorum_minus_one_matrix", "unknown_disk_remount_matrix", "object_lock_dry_run_grace_evidence"),
|
||||||
|
"G14": ("same_window_field_evidence", "ec8_4_evidence", "multi_set_evidence", "multi_pool_evidence"),
|
||||||
|
"P1": ("cold_walk_share_measurement", "foreground_latency_throughput_measurement", "profile_evidence"),
|
||||||
|
"P2": ("post_stop_convergence_measurement", "cold_segment_reuse_measurement"),
|
||||||
|
"P3": ("two_hour_pressure_measurement", "heal_capacity_measurement", "recovery_window_measurement"),
|
||||||
|
"P4": ("mrf_scale_measurement", "mrf_replay_cost_measurement", "retained_responsibility_evidence"),
|
||||||
|
"R-E": ("fixed_budget_restart_evidence", "enumeration_evidence", "classification_evidence"),
|
||||||
|
"R-D": ("manager_disposition_evidence", "event_disposition_evidence", "ledger_disposition_evidence", "grace_handling"),
|
||||||
|
"R-L": ("legacy_source_conflict_evidence", "migration_gap_evidence", "crash_safe_source_retirement_evidence"),
|
||||||
|
}
|
||||||
|
SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES = {
|
||||||
|
("G03", "durable_root_publication_proof"): "durable-root-publication",
|
||||||
|
("G03", "scoped_ack_request_identity"): "scoped-ack-request",
|
||||||
|
("G03", "participating_peer_capability_snapshot"): "peer-capability-snapshot",
|
||||||
|
("G03", "mixed_peer_ack_fallback_oracle"): "mixed-peer-ack-fallback",
|
||||||
|
("G09", "mixed_version_reader_evidence"): "mixed-version-reader",
|
||||||
|
("G09", "mixed_version_writer_evidence"): "mixed-version-writer",
|
||||||
|
("G09", "rollback_payload_evidence"): "rollback-payload",
|
||||||
|
("R-L", "legacy_source_conflict_evidence"): "legacy-source-conflict",
|
||||||
|
("R-L", "migration_gap_evidence"): "migration-gap",
|
||||||
|
("R-L", "crash_safe_source_retirement_evidence"): "crash-safe-source-retirement",
|
||||||
|
}
|
||||||
|
SCANNER_HEAL_RELEASE_MRF_DURABLE_REPLAY_FIELDS = {
|
||||||
|
("G07", "mrf_responsibility_oracle"),
|
||||||
|
("G07", "commit_boundary_crash_matrix"),
|
||||||
|
("P4", "mrf_replay_cost_measurement"),
|
||||||
|
("P4", "retained_responsibility_evidence"),
|
||||||
|
}
|
||||||
SCHEDULED_ALERT_WORKFLOWS = tuple(
|
SCHEDULED_ALERT_WORKFLOWS = tuple(
|
||||||
item["workflow"]
|
item["workflow"]
|
||||||
for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text())
|
for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text())
|
||||||
@@ -886,14 +954,38 @@ def evidence_integer(value: object, name: str, minimum: int, maximum: int) -> in
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def evidence_string(value: object, name: str, pattern: str | None = None) -> str:
|
||||||
|
require(isinstance(value, str) and value.strip(), f"invalid string {name}")
|
||||||
|
if pattern is not None:
|
||||||
|
require(re.fullmatch(pattern, value) is not None, f"invalid string {name}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def evidence_timestamp(value: object, name: str) -> datetime:
|
||||||
|
text = evidence_string(value, name)
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||||
|
except ValueError as error:
|
||||||
|
raise ValueError(f"invalid timestamp {name}") from error
|
||||||
|
require(parsed.tzinfo is not None, f"{name} must include timezone")
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
def scanner_heal_registry_schema(registry: dict[str, object]) -> int:
|
||||||
|
return evidence_integer(registry.get("schema"), "registry schema", 1, SCANNER_HEAL_REGISTRY_SCHEMA_MAX)
|
||||||
|
|
||||||
|
|
||||||
def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]:
|
def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]:
|
||||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
evidence_integer(registry.get("schema"), "registry schema", 1, 1)
|
scanner_heal_registry_schema(registry)
|
||||||
cases = registry.get("cases")
|
cases = registry.get("cases")
|
||||||
require(isinstance(cases, dict) and cases, "invalid scanner/heal registry")
|
require(isinstance(cases, dict) and cases, "invalid scanner/heal registry")
|
||||||
names = set()
|
names = set()
|
||||||
for case_id, requirement in cases.items():
|
for case_id, requirement in cases.items():
|
||||||
require(isinstance(case_id, str) and case_id, "invalid scanner/heal case identity")
|
require(isinstance(case_id, str) and case_id, "invalid scanner/heal case identity")
|
||||||
|
lane = requirement.get("lane")
|
||||||
|
require(isinstance(lane, str) and re.fullmatch(r"[a-z0-9-]+", lane) is not None,
|
||||||
|
f"invalid nextest profile lane for {case_id}")
|
||||||
oracle = requirement.get("oracle")
|
oracle = requirement.get("oracle")
|
||||||
require(isinstance(oracle, str) and oracle.endswith(".json"), f"invalid oracle for {case_id}")
|
require(isinstance(oracle, str) and oracle.endswith(".json"), f"invalid oracle for {case_id}")
|
||||||
path = Path(oracle)
|
path = Path(oracle)
|
||||||
@@ -902,10 +994,103 @@ def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]:
|
|||||||
f"invalid evidence for {case_id}")
|
f"invalid evidence for {case_id}")
|
||||||
require(type(requirement.get("unclean_shutdown_marker")) is bool,
|
require(type(requirement.get("unclean_shutdown_marker")) is bool,
|
||||||
f"invalid unclean-shutdown marker expectation for {case_id}")
|
f"invalid unclean-shutdown marker expectation for {case_id}")
|
||||||
|
if "erasure" in requirement:
|
||||||
|
erasure = requirement["erasure"]
|
||||||
|
require(isinstance(erasure, dict), f"invalid erasure expectation for {case_id}")
|
||||||
|
data_blocks = evidence_integer(erasure.get("data_blocks"), f"{case_id} data_blocks", 1, 16)
|
||||||
|
parity_blocks = evidence_integer(erasure.get("parity_blocks"), f"{case_id} parity_blocks", 1, 16)
|
||||||
|
require(data_blocks >= parity_blocks, f"invalid erasure geometry for {case_id}")
|
||||||
|
require(data_blocks + parity_blocks == requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"],
|
||||||
|
f"erasure geometry differs from topology for {case_id}")
|
||||||
|
if "erasure_set_drive_count" in requirement:
|
||||||
|
erasure_set_drive_count = evidence_integer(requirement.get("erasure_set_drive_count"),
|
||||||
|
f"{case_id} erasure_set_drive_count", 1, 64)
|
||||||
|
require(erasure_set_drive_count == requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"],
|
||||||
|
f"erasure set drive count differs from topology for {case_id}")
|
||||||
names.add(oracle)
|
names.add(oracle)
|
||||||
return tuple(sorted(names))
|
return tuple(sorted(names))
|
||||||
|
|
||||||
|
|
||||||
|
def scanner_heal_release_requirements(registry: dict[str, object]) -> tuple[dict[str, dict[str, object]], bool, list[str]]:
|
||||||
|
schema = scanner_heal_registry_schema(registry)
|
||||||
|
if schema == 1:
|
||||||
|
pending = registry.get("release_pending")
|
||||||
|
require(isinstance(pending, dict), "invalid scanner/heal release requirements")
|
||||||
|
requirements = {}
|
||||||
|
for gate, reason in pending.items():
|
||||||
|
require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None,
|
||||||
|
"invalid scanner/heal release gate")
|
||||||
|
require(isinstance(reason, str) and reason.strip(), f"missing release requirement for {gate}")
|
||||||
|
requirements[gate] = {"gate": gate, "status": "pending", "lane": "schema-1-pending",
|
||||||
|
"description": reason, "requires": [reason]}
|
||||||
|
return requirements, False, ["schema-1-pending"] if requirements else []
|
||||||
|
|
||||||
|
lanes = registry.get("release_lanes")
|
||||||
|
cases = registry.get("cases")
|
||||||
|
require(isinstance(cases, dict) and cases, "invalid scanner/heal registry")
|
||||||
|
require(isinstance(lanes, dict) and lanes, "invalid scanner/heal release lanes")
|
||||||
|
lane_statuses = {}
|
||||||
|
lane_gates = {}
|
||||||
|
for lane_id, lane in lanes.items():
|
||||||
|
require(isinstance(lane_id, str) and re.fullmatch(r"[a-z0-9-]+", lane_id) is not None,
|
||||||
|
"invalid scanner/heal release lane")
|
||||||
|
require(isinstance(lane, dict), f"invalid release lane {lane_id}")
|
||||||
|
status = lane.get("status")
|
||||||
|
require(status in ("implemented", "pending"), f"invalid release lane status for {lane_id}")
|
||||||
|
lane_statuses[lane_id] = status
|
||||||
|
if status == "implemented":
|
||||||
|
lane_cases = lane.get("cases")
|
||||||
|
require(isinstance(lane_cases, list) and lane_cases and all(isinstance(case, str) and case for case in lane_cases),
|
||||||
|
f"implemented release lane {lane_id} has no cases")
|
||||||
|
require(all(case in cases for case in lane_cases), f"implemented release lane {lane_id} has unknown cases")
|
||||||
|
else:
|
||||||
|
gates = lane.get("gates")
|
||||||
|
require(isinstance(gates, list) and gates and all(isinstance(gate, str) and gate for gate in gates),
|
||||||
|
f"pending release lane {lane_id} has no gates")
|
||||||
|
lane_gates[lane_id] = set(gates)
|
||||||
|
|
||||||
|
raw_requirements = registry.get("release_requirements")
|
||||||
|
require(isinstance(raw_requirements, list) and raw_requirements, "invalid scanner/heal release requirements")
|
||||||
|
requirements: dict[str, dict[str, object]] = {}
|
||||||
|
for item in raw_requirements:
|
||||||
|
require(isinstance(item, dict), "invalid scanner/heal release requirement")
|
||||||
|
gate = item.get("gate")
|
||||||
|
require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None,
|
||||||
|
"invalid scanner/heal release gate")
|
||||||
|
require(gate not in requirements, f"duplicate scanner/heal release gate {gate}")
|
||||||
|
status = item.get("status")
|
||||||
|
require(status == "pending", f"release gate {gate} must stay pending until real evidence is registered")
|
||||||
|
lane = item.get("lane")
|
||||||
|
require(isinstance(lane, str) and lane in lane_statuses, f"unknown release lane for {gate}")
|
||||||
|
require(lane_statuses[lane] == "pending", f"pending release gate {gate} mapped to non-pending lane {lane}")
|
||||||
|
description = item.get("description")
|
||||||
|
require(isinstance(description, str) and description.strip(), f"missing release requirement for {gate}")
|
||||||
|
requires = item.get("requires")
|
||||||
|
require(isinstance(requires, list) and requires and
|
||||||
|
all(isinstance(requirement, str) and requirement.strip() for requirement in requires),
|
||||||
|
f"missing concrete evidence requirements for {gate}")
|
||||||
|
evidence_fields = item.get("evidence_fields", [])
|
||||||
|
require(isinstance(evidence_fields, list) and
|
||||||
|
all(isinstance(field, str) and re.fullmatch(r"[a-z0-9][a-z0-9_]*", field) is not None
|
||||||
|
for field in evidence_fields),
|
||||||
|
f"invalid evidence fields for release gate {gate}")
|
||||||
|
required_fields = set(SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS.get(gate, ()))
|
||||||
|
missing_fields = sorted(required_fields - set(evidence_fields))
|
||||||
|
require(not missing_fields,
|
||||||
|
f"release gate {gate} missing required evidence fields: {', '.join(missing_fields)}")
|
||||||
|
requirements[gate] = item
|
||||||
|
|
||||||
|
missing = sorted(set(SCANNER_HEAL_RELEASE_REQUIRED_GATES) - set(requirements))
|
||||||
|
require(not missing, f"missing scanner/heal release requirements: {', '.join(missing)}")
|
||||||
|
for lane, gates in lane_gates.items():
|
||||||
|
unknown = sorted(gates - set(requirements))
|
||||||
|
require(not unknown, f"pending release lane {lane} has unknown gates: {', '.join(unknown)}")
|
||||||
|
mapped = {gate for gate, requirement in requirements.items() if requirement["lane"] == lane}
|
||||||
|
require(gates == mapped, f"pending release lane {lane} gates do not match release requirements")
|
||||||
|
pending_lanes = sorted(lane for lane, status in lane_statuses.items() if status == "pending")
|
||||||
|
return requirements, True, pending_lanes
|
||||||
|
|
||||||
|
|
||||||
def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_binary: Path) -> None:
|
def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_binary: Path) -> None:
|
||||||
"""Record an existing build; this command never builds or runs a test."""
|
"""Record an existing build; this command never builds or runs a test."""
|
||||||
require(not directory.exists(), "scanner/heal run directory must be new")
|
require(not directory.exists(), "scanner/heal run directory must be new")
|
||||||
@@ -969,7 +1154,7 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
|||||||
"""Validate one actual case, or fail the release while required lanes are pending."""
|
"""Validate one actual case, or fail the release while required lanes are pending."""
|
||||||
try:
|
try:
|
||||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
evidence_integer(registry.get("schema"), "registry schema", 1, 1)
|
scanner_heal_registry_schema(registry)
|
||||||
require(registry.get("cases"), "invalid scanner/heal registry")
|
require(registry.get("cases"), "invalid scanner/heal registry")
|
||||||
selected = registry["cases"] if case_id == "release" else {case_id: registry["cases"][case_id]}
|
selected = registry["cases"] if case_id == "release" else {case_id: registry["cases"][case_id]}
|
||||||
run = read_json(directory / "run.json")
|
run = read_json(directory / "run.json")
|
||||||
@@ -1050,6 +1235,19 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
|||||||
require(oracle.get("topology") == requirement["topology"], "oracle topology mismatch")
|
require(oracle.get("topology") == requirement["topology"], "oracle topology mismatch")
|
||||||
for key in ("nodes", "drives_per_node"):
|
for key in ("nodes", "drives_per_node"):
|
||||||
evidence_integer(oracle["topology"][key], f"observed {key}", 1, 16)
|
evidence_integer(oracle["topology"][key], f"observed {key}", 1, 16)
|
||||||
|
expected_erasure = requirement.get("erasure")
|
||||||
|
if expected_erasure is not None:
|
||||||
|
require(isinstance(expected_erasure, dict), "invalid erasure expectation")
|
||||||
|
expected_data_blocks = evidence_integer(expected_erasure.get("data_blocks"), "expected EC data blocks", 1, 16)
|
||||||
|
expected_parity_blocks = evidence_integer(expected_erasure.get("parity_blocks"), "expected EC parity blocks", 1, 16)
|
||||||
|
require(
|
||||||
|
expected_data_blocks + expected_parity_blocks
|
||||||
|
== oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"],
|
||||||
|
"expected EC geometry differs from topology",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
expected_data_blocks = None
|
||||||
|
expected_parity_blocks = None
|
||||||
evidence_integer(oracle.get("pid_before"), "pid_before", 1, 2**32 - 1)
|
evidence_integer(oracle.get("pid_before"), "pid_before", 1, 2**32 - 1)
|
||||||
evidence_integer(oracle.get("pid_after"), "pid_after", 1, 2**32 - 1)
|
evidence_integer(oracle.get("pid_after"), "pid_after", 1, 2**32 - 1)
|
||||||
require(oracle["pid_before"] != oracle["pid_after"], "no process restart witnessed")
|
require(oracle["pid_before"] != oracle["pid_after"], "no process restart witnessed")
|
||||||
@@ -1073,6 +1271,9 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
|||||||
parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16)
|
parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16)
|
||||||
require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"],
|
require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"],
|
||||||
"EC geometry differs from this case's single set")
|
"EC geometry differs from this case's single set")
|
||||||
|
if expected_data_blocks is not None:
|
||||||
|
require(data == expected_data_blocks and parity == expected_parity_blocks,
|
||||||
|
"EC data/parity geometry differs from the required case")
|
||||||
evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity)
|
evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity)
|
||||||
require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata")
|
require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata")
|
||||||
parts = physical["expected_part_numbers"]
|
parts = physical["expected_part_numbers"]
|
||||||
@@ -1088,7 +1289,10 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
|||||||
require(all(keys == sorted(obj["key"] for obj in objects) for keys in node_listings),
|
require(all(keys == sorted(obj["key"] for obj in objects) for keys in node_listings),
|
||||||
"S3 listing differs from object oracle")
|
"S3 listing differs from object oracle")
|
||||||
if case_id == "release":
|
if case_id == "release":
|
||||||
errors.extend(f"pending {gate}: {reason}" for gate, reason in registry["release_pending"].items())
|
release_requirements, _, _ = scanner_heal_release_requirements(registry)
|
||||||
|
errors.extend(
|
||||||
|
f"pending {gate}: {requirement['description']}" for gate, requirement in release_requirements.items()
|
||||||
|
)
|
||||||
return errors
|
return errors
|
||||||
except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error:
|
except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error:
|
||||||
return [f"scanner/heal evidence rejected: {error}"]
|
return [f"scanner/heal evidence rejected: {error}"]
|
||||||
@@ -1097,15 +1301,10 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
|||||||
def scanner_heal_release_status(root: Path, directory: Path) -> dict[str, object]:
|
def scanner_heal_release_status(root: Path, directory: Path) -> dict[str, object]:
|
||||||
"""Return a compact release decision without weakening case validation."""
|
"""Return a compact release decision without weakening case validation."""
|
||||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
evidence_integer(registry.get("schema"), "registry schema", 1, 1)
|
scanner_heal_registry_schema(registry)
|
||||||
cases = registry.get("cases")
|
cases = registry.get("cases")
|
||||||
require(isinstance(cases, dict) and cases, "invalid scanner/heal registry")
|
require(isinstance(cases, dict) and cases, "invalid scanner/heal registry")
|
||||||
pending = registry.get("release_pending")
|
release_requirements, release_schema_capable, pending_lanes = scanner_heal_release_requirements(registry)
|
||||||
require(isinstance(pending, dict), "invalid scanner/heal release requirements")
|
|
||||||
for gate, reason in pending.items():
|
|
||||||
require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None,
|
|
||||||
"invalid scanner/heal release gate")
|
|
||||||
require(isinstance(reason, str) and reason.strip(), f"missing release requirement for {gate}")
|
|
||||||
|
|
||||||
verified_cases = []
|
verified_cases = []
|
||||||
rejected_cases = []
|
rejected_cases = []
|
||||||
@@ -1119,10 +1318,206 @@ def scanner_heal_release_status(root: Path, directory: Path) -> dict[str, object
|
|||||||
"schema": 1,
|
"schema": 1,
|
||||||
"decision": "blocked",
|
"decision": "blocked",
|
||||||
"release_approved": False,
|
"release_approved": False,
|
||||||
"release_schema_capable": False,
|
"release_schema_capable": release_schema_capable,
|
||||||
"verified_cases": verified_cases,
|
"verified_cases": verified_cases,
|
||||||
"rejected_cases": rejected_cases,
|
"rejected_cases": rejected_cases,
|
||||||
"pending_gates": sorted(pending),
|
"pending_gates": sorted(release_requirements),
|
||||||
|
"pending_lanes": pending_lanes,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def release_bundle_artifact_path(bundle_path: Path, raw_path: object, gate: str, field: str) -> Path:
|
||||||
|
require(isinstance(raw_path, str) and raw_path.strip(), f"{gate}.{field} missing artifact")
|
||||||
|
path = Path(raw_path)
|
||||||
|
require(not path.is_absolute() and ".." not in path.parts, f"{gate}.{field} artifact path escapes bundle directory")
|
||||||
|
resolved = (bundle_path.parent / path).resolve()
|
||||||
|
require(resolved.is_relative_to(bundle_path.parent.resolve()), f"{gate}.{field} artifact path escapes bundle directory")
|
||||||
|
require(resolved.is_file(), f"{gate}.{field} artifact is missing")
|
||||||
|
require(resolved.stat().st_size > 0, f"{gate}.{field} artifact is empty")
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def validate_release_bundle_artifact(bundle_path: Path, source_revision: str, gate: str, field: str,
|
||||||
|
evidence: dict[str, object]) -> str:
|
||||||
|
require(evidence.get("evidence_type") == "measured", f"{gate}.{field} must be measured evidence")
|
||||||
|
require(evidence.get("source_revision") == source_revision, f"{gate}.{field} source revision mismatch")
|
||||||
|
run_id = evidence_string(evidence.get("run_id"), f"{gate}.{field}.run_id", r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}")
|
||||||
|
window_id = evidence_string(evidence.get("measurement_window_id"), f"{gate}.{field}.measurement_window_id",
|
||||||
|
r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}")
|
||||||
|
require(window_id != run_id, f"{gate}.{field} must separate run and measurement-window identities")
|
||||||
|
started = evidence_timestamp(evidence.get("started_at"), f"{gate}.{field}.started_at")
|
||||||
|
finished = evidence_timestamp(evidence.get("finished_at"), f"{gate}.{field}.finished_at")
|
||||||
|
require(started <= finished, f"{gate}.{field} evidence timestamps are inverted")
|
||||||
|
command = evidence.get("command")
|
||||||
|
require(isinstance(command, list) and command and
|
||||||
|
all(isinstance(part, str) and part.strip() for part in command),
|
||||||
|
f"{gate}.{field} missing command provenance")
|
||||||
|
evidence_string(evidence.get("artifact_format"), f"{gate}.{field}.artifact_format",
|
||||||
|
r"[A-Za-z0-9][A-Za-z0-9._+:-]{1,63}")
|
||||||
|
artifact = release_bundle_artifact_path(bundle_path, evidence.get("artifact"), gate, field)
|
||||||
|
require(sha(evidence.get("sha256")) and digest(artifact) == evidence["sha256"], f"{gate}.{field} artifact hash mismatch")
|
||||||
|
summary = evidence.get("summary")
|
||||||
|
require(isinstance(summary, str) and summary.strip(), f"{gate}.{field} missing human summary")
|
||||||
|
if gate.startswith("P"):
|
||||||
|
duration = evidence_integer(evidence.get("duration_seconds"), f"{gate}.{field}.duration_seconds", 1, 86400)
|
||||||
|
require((finished - started).total_seconds() + 1 >= duration, f"{gate}.{field} duration exceeds run window")
|
||||||
|
require(duration >= 900, f"{gate}.{field} requires at least 900 seconds")
|
||||||
|
if gate == "P3" and field == "two_hour_pressure_measurement":
|
||||||
|
require(duration >= 7200, f"{gate}.{field} requires at least two hours")
|
||||||
|
elif "duration_seconds" in evidence:
|
||||||
|
evidence_integer(evidence.get("duration_seconds"), f"{gate}.{field}.duration_seconds", 1, 86400)
|
||||||
|
if gate in ("G03", "G09", "R-L"):
|
||||||
|
versions = evidence.get("versions")
|
||||||
|
require(isinstance(versions, list) and
|
||||||
|
len(set(versions)) >= 2 and
|
||||||
|
all(isinstance(version, str) and re.fullmatch(r"[0-9a-f]{40}", version) is not None
|
||||||
|
for version in versions),
|
||||||
|
f"{gate}.{field} requires mixed-version evidence")
|
||||||
|
require(source_revision in versions, f"{gate}.{field} versions omit tested source revision")
|
||||||
|
expected_role = SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES[(gate, field)]
|
||||||
|
require(evidence.get("mixed_version_role") == expected_role,
|
||||||
|
f"{gate}.{field} mixed-version role must be {expected_role}")
|
||||||
|
if gate in ("G04", "G07", "R-E", "R-L"):
|
||||||
|
crash_points = evidence.get("crash_points")
|
||||||
|
require(isinstance(crash_points, list) and crash_points,
|
||||||
|
f"{gate}.{field} requires crash-boundary evidence")
|
||||||
|
if (gate, field) in SCANNER_HEAL_RELEASE_MRF_DURABLE_REPLAY_FIELDS:
|
||||||
|
evidence_integer(evidence.get("replayed_records"), f"{gate}.{field}.replayed_records", 1, 2**63 - 1)
|
||||||
|
require(evidence.get("responsibility_anchor_retained") is True,
|
||||||
|
f"{gate}.{field} requires retained MRF responsibility anchors")
|
||||||
|
require(evidence.get("successor_snapshot_published") is True,
|
||||||
|
f"{gate}.{field} requires successor snapshot publication evidence")
|
||||||
|
if gate == "G14":
|
||||||
|
if field == "same_window_field_evidence":
|
||||||
|
required_fields = set(SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS["G14"]) - {field}
|
||||||
|
same_window_fields = evidence.get("same_window_fields")
|
||||||
|
require(isinstance(same_window_fields, list) and
|
||||||
|
len(set(same_window_fields)) == len(same_window_fields) and
|
||||||
|
all(isinstance(item, str) and item in required_fields for item in same_window_fields),
|
||||||
|
"G14.same_window_field_evidence requires named G14 field coverage")
|
||||||
|
missing_same_window_fields = sorted(required_fields - set(same_window_fields))
|
||||||
|
require(not missing_same_window_fields,
|
||||||
|
"G14.same_window_field_evidence missing fields: " + ", ".join(missing_same_window_fields))
|
||||||
|
if field == "ec8_4_evidence":
|
||||||
|
topology = evidence.get("topology")
|
||||||
|
require(isinstance(topology, dict), "G14.ec8_4_evidence missing topology")
|
||||||
|
require(topology.get("erasure") == "EC8+4", "G14.ec8_4_evidence must record EC8+4")
|
||||||
|
require(evidence_integer(topology.get("nodes"), "G14 topology nodes", 3, 64) >= 3,
|
||||||
|
"G14.ec8_4_evidence requires at least three nodes")
|
||||||
|
require(evidence_integer(topology.get("drives_per_node"), "G14 topology drives", 4, 64) >= 4,
|
||||||
|
"G14.ec8_4_evidence requires at least four drives per node")
|
||||||
|
if field == "multi_set_evidence":
|
||||||
|
evidence_integer(evidence.get("sets"), "G14 multi_set_evidence.sets", 2, 1024)
|
||||||
|
if field == "multi_pool_evidence":
|
||||||
|
evidence_integer(evidence.get("pools"), "G14 multi_pool_evidence.pools", 2, 1024)
|
||||||
|
if field == "profile_evidence":
|
||||||
|
evidence_integer(evidence.get("resolved_samples"), f"{gate}.{field}.resolved_samples", 1, 2**63 - 1)
|
||||||
|
profile_artifacts = evidence.get("profile_artifacts")
|
||||||
|
require(isinstance(profile_artifacts, dict), f"{gate}.{field} missing profile artifacts")
|
||||||
|
missing_artifacts = sorted(set(RELEASE_PROFILE_ARTIFACTS) - set(profile_artifacts))
|
||||||
|
require(not missing_artifacts,
|
||||||
|
f"{gate}.{field} missing profile artifacts: {', '.join(missing_artifacts)}")
|
||||||
|
unknown_artifacts = sorted(set(profile_artifacts) - set(RELEASE_PROFILE_ARTIFACTS))
|
||||||
|
require(not unknown_artifacts,
|
||||||
|
f"{gate}.{field} unknown profile artifacts: {', '.join(unknown_artifacts)}")
|
||||||
|
for artifact_kind in RELEASE_PROFILE_ARTIFACTS:
|
||||||
|
item = profile_artifacts[artifact_kind]
|
||||||
|
require(isinstance(item, dict), f"{gate}.{field}.{artifact_kind} must be an object")
|
||||||
|
artifact_field = f"{field}.{artifact_kind}"
|
||||||
|
artifact_path = release_bundle_artifact_path(bundle_path, item.get("artifact"), gate, artifact_field)
|
||||||
|
require(sha(item.get("sha256")) and digest(artifact_path) == item["sha256"],
|
||||||
|
f"{gate}.{artifact_field} artifact hash mismatch")
|
||||||
|
evidence_string(item.get("artifact_format"), f"{gate}.{artifact_field}.artifact_format",
|
||||||
|
r"[A-Za-z0-9][A-Za-z0-9._+:-]{1,63}")
|
||||||
|
if "measurement_window_id" in item:
|
||||||
|
require(item["measurement_window_id"] == window_id,
|
||||||
|
f"{gate}.{artifact_field} measurement window mismatch")
|
||||||
|
if "resolved_samples" in item:
|
||||||
|
evidence_integer(item.get("resolved_samples"), f"{gate}.{artifact_field}.resolved_samples",
|
||||||
|
0, 2**63 - 1)
|
||||||
|
return window_id
|
||||||
|
|
||||||
|
|
||||||
|
def validate_release_bundle_gate_windows(gate: str, field_windows: dict[str, str]) -> None:
|
||||||
|
if gate == "G14" or gate.startswith("P"):
|
||||||
|
windows = sorted(set(field_windows.values()))
|
||||||
|
require(len(windows) == 1, f"{gate} evidence fields must share one measurement window")
|
||||||
|
|
||||||
|
|
||||||
|
def scanner_heal_release_bundle_status(root: Path, bundle_path: Path) -> dict[str, object]:
|
||||||
|
"""Validate a complete hard-gate evidence bundle without accepting synthetic claims."""
|
||||||
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
|
requirements, release_schema_capable, pending_lanes = scanner_heal_release_requirements(registry)
|
||||||
|
require(release_schema_capable, "scanner/heal release bundle requires schema 2 registry")
|
||||||
|
bundle_path = bundle_path.resolve()
|
||||||
|
bundle = read_json(bundle_path)
|
||||||
|
require(bundle.get("schema") == 1, "unsupported scanner/heal release evidence bundle schema")
|
||||||
|
require(bundle.get("evidence") == "measured", "scanner/heal release evidence bundle must be measured")
|
||||||
|
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip()
|
||||||
|
source_revision = bundle.get("source_revision")
|
||||||
|
require(isinstance(source_revision, str) and re.fullmatch(r"[0-9a-f]{40}", source_revision) is not None and
|
||||||
|
source_revision == revision,
|
||||||
|
"scanner/heal release evidence source revision mismatch")
|
||||||
|
raw_gates = bundle.get("gates")
|
||||||
|
require(isinstance(raw_gates, dict), "scanner/heal release evidence bundle missing gates")
|
||||||
|
|
||||||
|
verified: list[str] = []
|
||||||
|
rejected: dict[str, list[str]] = {}
|
||||||
|
for gate, requirement in requirements.items():
|
||||||
|
gate_errors: list[str] = []
|
||||||
|
gate_evidence = raw_gates.get(gate)
|
||||||
|
if not isinstance(gate_evidence, dict):
|
||||||
|
rejected[gate] = ["missing gate evidence"]
|
||||||
|
continue
|
||||||
|
if gate_evidence.get("status") != "pass":
|
||||||
|
gate_errors.append("gate status must be pass")
|
||||||
|
if gate_evidence.get("lane") != requirement["lane"]:
|
||||||
|
gate_errors.append("gate lane mismatch")
|
||||||
|
if gate_evidence.get("evidence_type") != "measured":
|
||||||
|
gate_errors.append("gate evidence type must be measured")
|
||||||
|
fields = gate_evidence.get("evidence_fields")
|
||||||
|
if not isinstance(fields, dict):
|
||||||
|
gate_errors.append("missing gate evidence fields")
|
||||||
|
fields = {}
|
||||||
|
required_fields = tuple(SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS[gate])
|
||||||
|
missing_fields = [field for field in required_fields if field not in fields]
|
||||||
|
if missing_fields:
|
||||||
|
gate_errors.append(f"missing required fields: {', '.join(missing_fields)}")
|
||||||
|
field_windows: dict[str, str] = {}
|
||||||
|
for field in required_fields:
|
||||||
|
if field not in fields:
|
||||||
|
continue
|
||||||
|
evidence = fields[field]
|
||||||
|
if not isinstance(evidence, dict):
|
||||||
|
gate_errors.append(f"{field} must be an object")
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
field_windows[field] = validate_release_bundle_artifact(bundle_path, source_revision, gate, field, evidence)
|
||||||
|
except (OSError, KeyError, TypeError, ValueError) as error:
|
||||||
|
gate_errors.append(str(error))
|
||||||
|
if not gate_errors:
|
||||||
|
try:
|
||||||
|
validate_release_bundle_gate_windows(gate, field_windows)
|
||||||
|
except ValueError as error:
|
||||||
|
gate_errors.append(str(error))
|
||||||
|
if gate_errors:
|
||||||
|
rejected[gate] = gate_errors
|
||||||
|
else:
|
||||||
|
verified.append(gate)
|
||||||
|
|
||||||
|
unknown = sorted(set(raw_gates) - set(requirements))
|
||||||
|
if unknown:
|
||||||
|
rejected["unknown"] = [f"unknown gates: {', '.join(unknown)}"]
|
||||||
|
approved = not rejected and sorted(verified) == sorted(requirements)
|
||||||
|
return {
|
||||||
|
"schema": 1,
|
||||||
|
"decision": "approved" if approved else "blocked",
|
||||||
|
"release_approved": approved,
|
||||||
|
"release_schema_capable": release_schema_capable,
|
||||||
|
"verified_gates": sorted(verified),
|
||||||
|
"rejected_gates": rejected,
|
||||||
|
"pending_gates": [] if approved else sorted(gate for gate in requirements if gate not in verified),
|
||||||
|
"pending_lanes": [] if approved else pending_lanes,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1310,16 +1705,31 @@ class SelfTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
+ "</testsuite></testsuites>"
|
+ "</testsuite></testsuites>"
|
||||||
)
|
)
|
||||||
|
def oracle_objects(requirement: dict[str, object]) -> list[dict[str, object]]:
|
||||||
|
topology = requirement["topology"]
|
||||||
|
total_blocks = topology["nodes"] * topology["drives_per_node"]
|
||||||
|
erasure = requirement.get("erasure")
|
||||||
|
if erasure is None:
|
||||||
|
parity_blocks = 4 if total_blocks == 12 else total_blocks // 2
|
||||||
|
data_blocks = total_blocks - parity_blocks
|
||||||
|
else:
|
||||||
|
data_blocks = erasure["data_blocks"]
|
||||||
|
parity_blocks = erasure["parity_blocks"]
|
||||||
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
|
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
|
||||||
"erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1],
|
"erasure_index": 1, "data_blocks": data_blocks, "parity_blocks": parity_blocks,
|
||||||
|
"expected_part_numbers": [1],
|
||||||
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
|
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
|
||||||
"inline_data_fingerprint": None}
|
"inline_data_fingerprint": None}
|
||||||
obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
|
obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
|
||||||
"expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
|
"expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
|
||||||
"expected_physical": physical, "physical": physical}
|
"expected_physical": physical, "physical": physical}
|
||||||
objects = [dict(obj, key=f"object-{index}") for index in range(9)]
|
count = requirement.get("min_objects", 9)
|
||||||
|
objects = [dict(obj, key=f"object-{index}") for index in range(count)]
|
||||||
objects[-1] = dict(objects[-1], expected_physical=None)
|
objects[-1] = dict(objects[-1], expected_physical=None)
|
||||||
|
return objects
|
||||||
|
|
||||||
for case_id, requirement in requirements.items():
|
for case_id, requirement in requirements.items():
|
||||||
|
objects = oracle_objects(requirement)
|
||||||
write_json(run_dir / requirement["oracle"], {
|
write_json(run_dir / requirement["oracle"], {
|
||||||
"schema": 1, "evidence": requirement["evidence"], "case": case_id,
|
"schema": 1, "evidence": requirement["evidence"], "case": case_id,
|
||||||
"run_id": "a" * 32, "source_revision": "b" * 40,
|
"run_id": "a" * 32, "source_revision": "b" * 40,
|
||||||
@@ -1328,11 +1738,245 @@ class SelfTests(unittest.TestCase):
|
|||||||
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
|
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
|
||||||
"topology": requirement["topology"], "pid_before": 10, "pid_after": 11,
|
"topology": requirement["topology"], "pid_before": 10, "pid_after": 11,
|
||||||
"unclean_shutdown_marker": requirement["unclean_shutdown_marker"],
|
"unclean_shutdown_marker": requirement["unclean_shutdown_marker"],
|
||||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
|
"objects": objects, "node_listings": [[item["key"] for item in objects]] * requirement["topology"]["nodes"],
|
||||||
})
|
})
|
||||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||||
return root, run_dir
|
return root, run_dir
|
||||||
|
|
||||||
|
def test_scanner_heal_case_lane_is_required_for_runner_profile(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, _ = self.scanner_heal_fixture(Path(tmp))
|
||||||
|
registry_path = root / ".config/scanner-heal-required-tests.json"
|
||||||
|
registry = read_json(registry_path)
|
||||||
|
del registry["cases"]["ec84-target-drive-restart"]["lane"]
|
||||||
|
write_json(registry_path, registry)
|
||||||
|
with self.assertRaisesRegex(ValueError, "invalid nextest profile lane for ec84-target-drive-restart"):
|
||||||
|
scanner_heal_oracle_names(root)
|
||||||
|
|
||||||
|
def scanner_heal_release_bundle_fixture(self, directory: Path) -> tuple[Path, Path]:
|
||||||
|
"""Parser fixtures only; the bundle is not runtime evidence."""
|
||||||
|
root, _ = self.scanner_heal_fixture(directory)
|
||||||
|
bundle_dir = directory / "bundle"
|
||||||
|
artifact_dir = bundle_dir / "artifacts"
|
||||||
|
artifact_dir.mkdir(parents=True)
|
||||||
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
|
requirements, _, _ = scanner_heal_release_requirements(registry)
|
||||||
|
gates = {}
|
||||||
|
source_revision = "b" * 40
|
||||||
|
started = datetime(2026, 9, 8, 0, 0, tzinfo=timezone.utc)
|
||||||
|
for gate, requirement in requirements.items():
|
||||||
|
fields = {}
|
||||||
|
for field in SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS[gate]:
|
||||||
|
artifact = artifact_dir / f"{gate}-{field}.json"
|
||||||
|
write_json(artifact, {"gate": gate, "field": field, "fixture": True})
|
||||||
|
duration = 60
|
||||||
|
evidence = {
|
||||||
|
"artifact": artifact.relative_to(bundle_dir).as_posix(),
|
||||||
|
"sha256": digest(artifact),
|
||||||
|
"evidence_type": "measured",
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"run_id": f"{gate.lower()}-{field.replace('_', '-')}-run",
|
||||||
|
"measurement_window_id": f"{gate.lower()}-window",
|
||||||
|
"started_at": started.isoformat().replace("+00:00", "Z"),
|
||||||
|
"command": ["cargo", "nextest", "run", requirement["description"]],
|
||||||
|
"artifact_format": "json",
|
||||||
|
"summary": f"parser fixture for {gate}.{field}",
|
||||||
|
}
|
||||||
|
if gate.startswith("P"):
|
||||||
|
duration = 900
|
||||||
|
evidence["duration_seconds"] = duration
|
||||||
|
if gate == "P3" and field == "two_hour_pressure_measurement":
|
||||||
|
duration = 7200
|
||||||
|
evidence["duration_seconds"] = duration
|
||||||
|
evidence["finished_at"] = (started + timedelta(seconds=duration)).isoformat().replace("+00:00", "Z")
|
||||||
|
if gate in ("G03", "G09", "R-L"):
|
||||||
|
evidence["versions"] = ["a" * 40, source_revision]
|
||||||
|
evidence["mixed_version_role"] = SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES[(gate, field)]
|
||||||
|
if gate in ("G04", "G07", "R-E", "R-L"):
|
||||||
|
evidence["crash_points"] = ["before-commit"]
|
||||||
|
if (gate, field) in SCANNER_HEAL_RELEASE_MRF_DURABLE_REPLAY_FIELDS:
|
||||||
|
evidence["replayed_records"] = 2
|
||||||
|
evidence["responsibility_anchor_retained"] = True
|
||||||
|
evidence["successor_snapshot_published"] = True
|
||||||
|
if gate == "G14" and field == "ec8_4_evidence":
|
||||||
|
evidence["topology"] = {"erasure": "EC8+4", "nodes": 3, "drives_per_node": 4}
|
||||||
|
if gate == "G14" and field == "same_window_field_evidence":
|
||||||
|
evidence["same_window_fields"] = [
|
||||||
|
item
|
||||||
|
for item in SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS["G14"]
|
||||||
|
if item != "same_window_field_evidence"
|
||||||
|
]
|
||||||
|
if gate == "G14" and field == "multi_set_evidence":
|
||||||
|
evidence["sets"] = 2
|
||||||
|
if gate == "G14" and field == "multi_pool_evidence":
|
||||||
|
evidence["pools"] = 2
|
||||||
|
if field == "profile_evidence":
|
||||||
|
evidence["resolved_samples"] = 1
|
||||||
|
artifacts = {}
|
||||||
|
for artifact_kind in RELEASE_PROFILE_ARTIFACTS:
|
||||||
|
artifact = artifact_dir / f"{gate}-{field}-{artifact_kind}.json"
|
||||||
|
write_json(artifact, {"gate": gate, "field": field, "artifact": artifact_kind})
|
||||||
|
artifacts[artifact_kind] = {
|
||||||
|
"artifact": artifact.relative_to(bundle_dir).as_posix(),
|
||||||
|
"sha256": digest(artifact),
|
||||||
|
"artifact_format": "json",
|
||||||
|
}
|
||||||
|
evidence["profile_artifacts"] = artifacts
|
||||||
|
fields[field] = evidence
|
||||||
|
gates[gate] = {
|
||||||
|
"status": "pass",
|
||||||
|
"lane": requirement["lane"],
|
||||||
|
"evidence_type": "measured",
|
||||||
|
"evidence_fields": fields,
|
||||||
|
}
|
||||||
|
bundle = bundle_dir / "release-evidence.json"
|
||||||
|
write_json(bundle, {"schema": 1, "evidence": "measured", "source_revision": source_revision, "gates": gates})
|
||||||
|
return root, bundle
|
||||||
|
|
||||||
|
def test_scanner_heal_release_bundle_accepts_complete_measured_evidence(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||||
|
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||||
|
status = scanner_heal_release_bundle_status(root, bundle)
|
||||||
|
self.assertEqual(status["decision"], "approved")
|
||||||
|
self.assertTrue(status["release_approved"])
|
||||||
|
self.assertEqual(len(status["verified_gates"]), len(SCANNER_HEAL_RELEASE_REQUIRED_GATES))
|
||||||
|
self.assertEqual(status["pending_gates"], [])
|
||||||
|
self.assertEqual(status["pending_lanes"], [])
|
||||||
|
|
||||||
|
def test_scanner_heal_release_bundle_rejects_synthetic_or_missing_fields(self) -> None:
|
||||||
|
for fault in ("synthetic", "missing-field", "hash", "empty-artifact"):
|
||||||
|
with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||||
|
data = read_json(bundle)
|
||||||
|
if fault == "synthetic":
|
||||||
|
data["evidence"] = "synthetic"
|
||||||
|
elif fault == "missing-field":
|
||||||
|
del data["gates"]["G09"]["evidence_fields"]["rollback_payload_evidence"]
|
||||||
|
elif fault == "hash":
|
||||||
|
artifact = bundle.parent / data["gates"]["G01"]["evidence_fields"]["root_authority_evidence"]["artifact"]
|
||||||
|
artifact.write_text(artifact.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
||||||
|
else:
|
||||||
|
artifact = bundle.parent / data["gates"]["G01"]["evidence_fields"]["root_authority_evidence"]["artifact"]
|
||||||
|
artifact.write_text("", encoding="utf-8")
|
||||||
|
write_json(bundle, data)
|
||||||
|
|
||||||
|
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||||
|
if fault == "synthetic":
|
||||||
|
with self.assertRaisesRegex(ValueError, "must be measured"):
|
||||||
|
scanner_heal_release_bundle_status(root, bundle)
|
||||||
|
else:
|
||||||
|
status = scanner_heal_release_bundle_status(root, bundle)
|
||||||
|
if fault != "synthetic":
|
||||||
|
self.assertEqual(status["decision"], "blocked")
|
||||||
|
self.assertFalse(status["release_approved"])
|
||||||
|
self.assertTrue(status["rejected_gates"])
|
||||||
|
|
||||||
|
def test_scanner_heal_release_bundle_enforces_topology_duration_profile_and_versions(self) -> None:
|
||||||
|
for fault, gate, field, mutation, expected in (
|
||||||
|
("topology", "G14", "ec8_4_evidence", lambda item: item.update({"topology": {"erasure": "EC4+2", "nodes": 2, "drives_per_node": 3}}), "EC8+4"),
|
||||||
|
("missing-duration", "P1", "cold_walk_share_measurement", lambda item: item.pop("duration_seconds"), "duration_seconds"),
|
||||||
|
("duration", "P3", "two_hour_pressure_measurement", lambda item: item.update({"duration_seconds": 7199}), "two hours"),
|
||||||
|
("profile", "P1", "profile_evidence", lambda item: item.pop("resolved_samples"), "resolved_samples"),
|
||||||
|
(
|
||||||
|
"profile-artifact",
|
||||||
|
"P1",
|
||||||
|
"profile_evidence",
|
||||||
|
lambda item: item["profile_artifacts"].pop("flamegraph"),
|
||||||
|
"missing profile artifacts",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"profile-artifact-hash",
|
||||||
|
"P1",
|
||||||
|
"profile_evidence",
|
||||||
|
lambda item: item["profile_artifacts"]["rss-samples"].update({"sha256": "0" * 64}),
|
||||||
|
"artifact hash mismatch",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"profile-artifact-window",
|
||||||
|
"P1",
|
||||||
|
"profile_evidence",
|
||||||
|
lambda item: item["profile_artifacts"]["save-frequency"].update(
|
||||||
|
{"measurement_window_id": "p1-different-window"}
|
||||||
|
),
|
||||||
|
"measurement window mismatch",
|
||||||
|
),
|
||||||
|
("versions", "G09", "mixed_version_reader_evidence", lambda item: item.update({"versions": [1, 2]}), "mixed-version"),
|
||||||
|
("stale-versions", "G09", "mixed_version_writer_evidence", lambda item: item.update({"versions": ["a" * 40, "c" * 40]}), "tested source revision"),
|
||||||
|
("mrf-records", "G07", "mrf_responsibility_oracle", lambda item: item.pop("replayed_records"), "replayed_records"),
|
||||||
|
("mrf-anchor", "G07", "commit_boundary_crash_matrix", lambda item: item.update({"responsibility_anchor_retained": False}), "retained MRF responsibility anchors"),
|
||||||
|
("mrf-successor", "P4", "retained_responsibility_evidence", lambda item: item.pop("successor_snapshot_published"), "successor snapshot"),
|
||||||
|
("same-window-fields", "G14", "same_window_field_evidence", lambda item: item.update({"same_window_fields": ["ec8_4_evidence", "multi_set_evidence"]}), "missing fields"),
|
||||||
|
):
|
||||||
|
with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||||
|
data = read_json(bundle)
|
||||||
|
mutation(data["gates"][gate]["evidence_fields"][field])
|
||||||
|
write_json(bundle, data)
|
||||||
|
|
||||||
|
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||||
|
status = scanner_heal_release_bundle_status(root, bundle)
|
||||||
|
self.assertEqual(status["decision"], "blocked")
|
||||||
|
self.assertFalse(status["release_approved"])
|
||||||
|
self.assertTrue(any(expected in error for error in status["rejected_gates"][gate]))
|
||||||
|
|
||||||
|
def test_scanner_heal_release_bundle_requires_mixed_version_field_roles(self) -> None:
|
||||||
|
for gate, field, wrong_role in (
|
||||||
|
("G03", "mixed_peer_ack_fallback_oracle", "mixed-version-reader"),
|
||||||
|
("G09", "mixed_version_reader_evidence", "mixed-version-writer"),
|
||||||
|
("G09", "mixed_version_writer_evidence", "mixed-version-reader"),
|
||||||
|
("G09", "rollback_payload_evidence", "mixed-version-reader"),
|
||||||
|
("R-L", "crash_safe_source_retirement_evidence", "migration-gap"),
|
||||||
|
):
|
||||||
|
with self.subTest(gate=gate, field=field), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||||
|
data = read_json(bundle)
|
||||||
|
data["gates"][gate]["evidence_fields"][field]["mixed_version_role"] = wrong_role
|
||||||
|
write_json(bundle, data)
|
||||||
|
|
||||||
|
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||||
|
status = scanner_heal_release_bundle_status(root, bundle)
|
||||||
|
self.assertEqual(status["decision"], "blocked")
|
||||||
|
self.assertFalse(status["release_approved"])
|
||||||
|
self.assertTrue(any("mixed-version role" in error for error in status["rejected_gates"][gate]))
|
||||||
|
|
||||||
|
def test_scanner_heal_release_bundle_requires_field_provenance(self) -> None:
|
||||||
|
for fault, mutation, expected in (
|
||||||
|
("source", lambda item: item.update({"source_revision": "c" * 40}), "source revision mismatch"),
|
||||||
|
("run-id", lambda item: item.pop("run_id"), "run_id"),
|
||||||
|
("window-id", lambda item: item.update({"measurement_window_id": item["run_id"]}), "separate run"),
|
||||||
|
("started-at", lambda item: item.update({"started_at": "not-a-time"}), "timestamp"),
|
||||||
|
("finished-at", lambda item: item.update({"finished_at": "2026-09-07T00:00:00Z"}), "timestamps are inverted"),
|
||||||
|
("command", lambda item: item.update({"command": []}), "command provenance"),
|
||||||
|
("artifact-format", lambda item: item.pop("artifact_format"), "artifact_format"),
|
||||||
|
):
|
||||||
|
with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||||
|
data = read_json(bundle)
|
||||||
|
mutation(data["gates"]["G01"]["evidence_fields"]["root_authority_evidence"])
|
||||||
|
write_json(bundle, data)
|
||||||
|
|
||||||
|
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||||
|
status = scanner_heal_release_bundle_status(root, bundle)
|
||||||
|
self.assertEqual(status["decision"], "blocked")
|
||||||
|
self.assertFalse(status["release_approved"])
|
||||||
|
self.assertTrue(any(expected in error for error in status["rejected_gates"]["G01"]), fault)
|
||||||
|
|
||||||
|
def test_scanner_heal_release_bundle_requires_same_gate_measurement_window(self) -> None:
|
||||||
|
for gate, field in (("G14", "multi_pool_evidence"), ("P1", "profile_evidence"), ("P3", "heal_capacity_measurement")):
|
||||||
|
with self.subTest(gate=gate), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp))
|
||||||
|
data = read_json(bundle)
|
||||||
|
data["gates"][gate]["evidence_fields"][field]["measurement_window_id"] = f"{gate.lower()}-different-window"
|
||||||
|
write_json(bundle, data)
|
||||||
|
|
||||||
|
with mock.patch("subprocess.check_output", return_value="b" * 40):
|
||||||
|
status = scanner_heal_release_bundle_status(root, bundle)
|
||||||
|
self.assertEqual(status["decision"], "blocked")
|
||||||
|
self.assertFalse(status["release_approved"])
|
||||||
|
self.assertTrue(any("must share one measurement window" in error
|
||||||
|
for error in status["rejected_gates"][gate]))
|
||||||
|
|
||||||
def test_scanner_heal_case_does_not_approve_pending_release(self) -> None:
|
def test_scanner_heal_case_does_not_approve_pending_release(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||||
@@ -1346,14 +1990,21 @@ class SelfTests(unittest.TestCase):
|
|||||||
status = scanner_heal_release_status(root, run_dir)
|
status = scanner_heal_release_status(root, run_dir)
|
||||||
self.assertEqual(status["decision"], "blocked")
|
self.assertEqual(status["decision"], "blocked")
|
||||||
self.assertFalse(status["release_approved"])
|
self.assertFalse(status["release_approved"])
|
||||||
|
self.assertTrue(status["release_schema_capable"])
|
||||||
self.assertEqual(status["rejected_cases"], [])
|
self.assertEqual(status["rejected_cases"], [])
|
||||||
self.assertEqual(len(status["pending_gates"]), 21)
|
self.assertEqual(len(status["pending_gates"]), 21)
|
||||||
|
self.assertIn("mixed-version-rollback", status["pending_lanes"])
|
||||||
|
self.assertIn("ec8-4-multiset", status["pending_lanes"])
|
||||||
|
self.assertIn("scheduler-pressure", status["pending_lanes"])
|
||||||
|
|
||||||
def test_scanner_heal_case_only_schema_cannot_approve_release(self) -> None:
|
def test_scanner_heal_case_only_schema_cannot_approve_release(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
|
registry["schema"] = 1
|
||||||
registry["release_pending"] = {}
|
registry["release_pending"] = {}
|
||||||
|
registry.pop("release_lanes")
|
||||||
|
registry.pop("release_requirements")
|
||||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||||
|
|
||||||
status = scanner_heal_release_status(root, run_dir)
|
status = scanner_heal_release_status(root, run_dir)
|
||||||
@@ -1363,11 +2014,99 @@ class SelfTests(unittest.TestCase):
|
|||||||
self.assertEqual(status["rejected_cases"], [])
|
self.assertEqual(status["rejected_cases"], [])
|
||||||
self.assertEqual(status["pending_gates"], [])
|
self.assertEqual(status["pending_gates"], [])
|
||||||
|
|
||||||
|
def test_scanner_heal_release_requirements_cannot_be_cleared(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||||
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
|
registry["release_requirements"] = []
|
||||||
|
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ValueError, "invalid scanner/heal release requirements"):
|
||||||
|
scanner_heal_release_status(root, run_dir)
|
||||||
|
|
||||||
|
def test_scanner_heal_release_matrix_lanes_remain_pending_without_real_evidence(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||||
|
|
||||||
|
errors = check_scanner_heal_evidence(root, run_dir, "release")
|
||||||
|
for gate, text in (
|
||||||
|
("G03", "Exact scoped ACK"),
|
||||||
|
("G09", "mixed-version reader/writer"),
|
||||||
|
("G14", "3x4 EC8+4"),
|
||||||
|
("P3", "two-hour pressure/heal capacity"),
|
||||||
|
("R-L", "Legacy source conflicts"),
|
||||||
|
):
|
||||||
|
self.assertTrue(any(error.startswith(f"pending {gate}:") and text in error for error in errors), gate)
|
||||||
|
|
||||||
|
status = scanner_heal_release_status(root, run_dir)
|
||||||
|
self.assertFalse(status["release_approved"])
|
||||||
|
self.assertIn("mixed-version-rollback", status["pending_lanes"])
|
||||||
|
self.assertIn("ec8-4-multiset", status["pending_lanes"])
|
||||||
|
self.assertIn("scheduler-pressure", status["pending_lanes"])
|
||||||
|
requirements, _, _ = scanner_heal_release_requirements(read_json(root / ".config/scanner-heal-required-tests.json"))
|
||||||
|
self.assertIn("durable_root_publication_proof", requirements["G03"]["evidence_fields"])
|
||||||
|
self.assertIn("mixed_version_writer_evidence", requirements["G09"]["evidence_fields"])
|
||||||
|
|
||||||
|
def test_scanner_heal_required_evidence_fields_cannot_be_removed(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||||
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
|
for gate in ("G03", "G09"):
|
||||||
|
for requirement in registry["release_requirements"]:
|
||||||
|
if requirement["gate"] == gate:
|
||||||
|
requirement["evidence_fields"] = []
|
||||||
|
break
|
||||||
|
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||||
|
|
||||||
|
with self.subTest(gate=gate):
|
||||||
|
with self.assertRaisesRegex(ValueError, f"release gate {gate} missing required evidence fields"):
|
||||||
|
scanner_heal_release_status(root, run_dir)
|
||||||
|
|
||||||
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
|
for requirement in registry["release_requirements"]:
|
||||||
|
if requirement["gate"] == gate:
|
||||||
|
requirement["evidence_fields"] = list(SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS[gate])
|
||||||
|
break
|
||||||
|
|
||||||
|
def test_scanner_heal_ec84_case_rejects_wrong_erasure_geometry(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||||
|
self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4"), [])
|
||||||
|
|
||||||
|
path = run_dir / "background-target-crash-ec8-4.json"
|
||||||
|
oracle = read_json(path)
|
||||||
|
oracle["objects"][0]["physical"]["data_blocks"] = 10
|
||||||
|
oracle["objects"][0]["physical"]["parity_blocks"] = 2
|
||||||
|
oracle["objects"][0]["expected_physical"]["data_blocks"] = 10
|
||||||
|
oracle["objects"][0]["expected_physical"]["parity_blocks"] = 2
|
||||||
|
write_json(path, oracle)
|
||||||
|
(run_dir / "execution.json").unlink()
|
||||||
|
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||||
|
|
||||||
|
errors = check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4")
|
||||||
|
self.assertTrue(
|
||||||
|
any("EC data/parity geometry differs from the required case" in error for error in errors),
|
||||||
|
errors,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_scanner_heal_pending_gate_cannot_map_to_implemented_lane(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||||
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
|
registry["release_requirements"][0]["lane"] = "single-set-restart"
|
||||||
|
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ValueError, "mapped to non-pending lane"):
|
||||||
|
scanner_heal_release_status(root, run_dir)
|
||||||
|
|
||||||
def test_scanner_heal_release_status_rejects_synthetic_case(self) -> None:
|
def test_scanner_heal_release_status_rejects_synthetic_case(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
|
registry["schema"] = 1
|
||||||
registry["release_pending"] = {}
|
registry["release_pending"] = {}
|
||||||
|
registry.pop("release_lanes")
|
||||||
|
registry.pop("release_requirements")
|
||||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||||
path = run_dir / "background-target-crash.json"
|
path = run_dir / "background-target-crash.json"
|
||||||
oracle = read_json(path)
|
oracle = read_json(path)
|
||||||
@@ -1379,6 +2118,7 @@ class SelfTests(unittest.TestCase):
|
|||||||
status = scanner_heal_release_status(root, run_dir)
|
status = scanner_heal_release_status(root, run_dir)
|
||||||
self.assertEqual(status["decision"], "blocked")
|
self.assertEqual(status["decision"], "blocked")
|
||||||
self.assertFalse(status["release_approved"])
|
self.assertFalse(status["release_approved"])
|
||||||
|
self.assertFalse(status["release_schema_capable"])
|
||||||
self.assertEqual(status["rejected_cases"], ["background-target-crash"])
|
self.assertEqual(status["rejected_cases"], ["background-target-crash"])
|
||||||
self.assertEqual(status["pending_gates"], [])
|
self.assertEqual(status["pending_gates"], [])
|
||||||
|
|
||||||
@@ -1386,9 +2126,14 @@ class SelfTests(unittest.TestCase):
|
|||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||||
|
registry["schema"] = 1
|
||||||
registry["release_pending"] = {}
|
registry["release_pending"] = {}
|
||||||
|
registry.pop("release_lanes")
|
||||||
|
registry.pop("release_requirements")
|
||||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||||
(run_dir / "background-target-crash.json").unlink()
|
for case_id, requirement in registry["cases"].items():
|
||||||
|
if case_id != "background-target-restart":
|
||||||
|
(run_dir / requirement["oracle"]).unlink()
|
||||||
(run_dir / "execution.json").unlink()
|
(run_dir / "execution.json").unlink()
|
||||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||||
|
|
||||||
@@ -1396,7 +2141,7 @@ class SelfTests(unittest.TestCase):
|
|||||||
self.assertEqual(status["decision"], "blocked")
|
self.assertEqual(status["decision"], "blocked")
|
||||||
self.assertFalse(status["release_approved"])
|
self.assertFalse(status["release_approved"])
|
||||||
self.assertEqual(status["verified_cases"], ["background-target-restart"])
|
self.assertEqual(status["verified_cases"], ["background-target-restart"])
|
||||||
self.assertEqual(status["rejected_cases"], ["background-target-crash"])
|
self.assertEqual(status["rejected_cases"], sorted(set(registry["cases"]) - {"background-target-restart"}))
|
||||||
|
|
||||||
def test_scanner_heal_finish_collects_oracles_from_registry(self) -> None:
|
def test_scanner_heal_finish_collects_oracles_from_registry(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
@@ -2246,7 +2991,7 @@ def main() -> int:
|
|||||||
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
|
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
|
||||||
return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
|
return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
|
||||||
if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"],
|
if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"],
|
||||||
["--check-scanner-heal-release"]):
|
["--check-scanner-heal-release"], ["--check-scanner-heal-release-bundle"]):
|
||||||
try:
|
try:
|
||||||
if len(sys.argv) == 5 and sys.argv[1] == "--begin-scanner-heal":
|
if len(sys.argv) == 5 and sys.argv[1] == "--begin-scanner-heal":
|
||||||
begin_scanner_heal_receipt(ROOT, Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4]))
|
begin_scanner_heal_receipt(ROOT, Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4]))
|
||||||
@@ -2270,7 +3015,16 @@ def main() -> int:
|
|||||||
return 2
|
return 2
|
||||||
print(json.dumps(status, sort_keys=True, separators=(",", ":")))
|
print(json.dumps(status, sort_keys=True, separators=(",", ":")))
|
||||||
return 0 if status["release_approved"] else 1
|
return 0 if status["release_approved"] else 1
|
||||||
raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, --check-scanner-heal DIR CASE|release, or --check-scanner-heal-release DIR")
|
if len(sys.argv) == 3 and sys.argv[1] == "--check-scanner-heal-release-bundle":
|
||||||
|
try:
|
||||||
|
status = scanner_heal_release_bundle_status(ROOT, Path(sys.argv[2]))
|
||||||
|
except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error:
|
||||||
|
print(json.dumps({"schema": 1, "decision": "invalid", "release_approved": False,
|
||||||
|
"error": str(error)}, sort_keys=True, separators=(",", ":")))
|
||||||
|
return 2
|
||||||
|
print(json.dumps(status, sort_keys=True, separators=(",", ":")))
|
||||||
|
return 0 if status["release_approved"] else 1
|
||||||
|
raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, --check-scanner-heal DIR CASE|release, --check-scanner-heal-release DIR, or --check-scanner-heal-release-bundle FILE")
|
||||||
except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error:
|
except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error:
|
||||||
print(f"ERROR: {error}", file=sys.stderr)
|
print(f"ERROR: {error}", file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
@@ -2297,7 +3051,7 @@ def main() -> int:
|
|||||||
if sys.argv[1:]:
|
if sys.argv[1:]:
|
||||||
print(
|
print(
|
||||||
"usage: check_test_wiring.py [--self-test | --check-core LISTING | --check-profile PROFILE LISTING | "
|
"usage: check_test_wiring.py [--self-test | --check-core LISTING | --check-profile PROFILE LISTING | "
|
||||||
"--update-profile PROFILE LISTING PLATFORM]",
|
"--update-profile PROFILE LISTING PLATFORM | --check-scanner-heal-release-bundle FILE]",
|
||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
return 2
|
return 2
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ set -euo pipefail
|
|||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}"
|
PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}"
|
||||||
PROFILE="e2e-nightly"
|
PROFILE=""
|
||||||
CASE_ID="background-target-crash"
|
CASE_ID="background-target-crash"
|
||||||
RUN_DIR=""
|
RUN_DIR=""
|
||||||
PLAN_ONLY=0
|
PLAN_ONLY=0
|
||||||
@@ -18,7 +18,7 @@ then validate the produced receipt, nextest listing, JUnit, and case oracle.
|
|||||||
|
|
||||||
Options:
|
Options:
|
||||||
--case CASE Registry case to run (default: background-target-crash)
|
--case CASE Registry case to run (default: background-target-crash)
|
||||||
--profile PROFILE Nextest profile to use (default: e2e-nightly)
|
--profile PROFILE Nextest profile to use (default: registry lane)
|
||||||
--run-dir DIR New evidence directory (default: target/scanner-heal-evidence/CASE-TIMESTAMP)
|
--run-dir DIR New evidence directory (default: target/scanner-heal-evidence/CASE-TIMESTAMP)
|
||||||
--plan-only Validate registry selection and print the exact filter without running cargo
|
--plan-only Validate registry selection and print the exact filter without running cargo
|
||||||
--self-test Run lightweight CLI/registry checks without building Rust
|
--self-test Run lightweight CLI/registry checks without building Rust
|
||||||
@@ -31,6 +31,18 @@ port allocator when the default 20000..30000 test range is unavailable.
|
|||||||
USAGE
|
USAGE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case_ids() {
|
||||||
|
"$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" <<'PY'
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
registry = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||||
|
for case_id in sorted(registry["cases"]):
|
||||||
|
print(case_id)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
case_field() {
|
case_field() {
|
||||||
local case_id="$1"
|
local case_id="$1"
|
||||||
local field="$2"
|
local field="$2"
|
||||||
@@ -62,6 +74,40 @@ print("test(/^" + re.escape(case["name"]) + "$/)")
|
|||||||
PY
|
PY
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runtime_profile_for() {
|
||||||
|
local case_id="$1"
|
||||||
|
case "$case_id" in
|
||||||
|
background-target-crash|background-target-restart)
|
||||||
|
echo "background-4x1"
|
||||||
|
;;
|
||||||
|
background-target-crash-ec8-4|background-target-restart-ec8-4)
|
||||||
|
echo "background-ec8-4"
|
||||||
|
;;
|
||||||
|
ec84-target-drive-restart)
|
||||||
|
echo "distributed-ec8-4"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "default"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
apply_runtime_profile() {
|
||||||
|
local case_id="$1"
|
||||||
|
case "$(runtime_profile_for "$case_id")" in
|
||||||
|
background-4x1)
|
||||||
|
export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}"
|
||||||
|
export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-16777216}"
|
||||||
|
export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-120}"
|
||||||
|
;;
|
||||||
|
background-ec8-4)
|
||||||
|
export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-32}"
|
||||||
|
export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-8388608}"
|
||||||
|
export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-180}"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
test_binary_from_listing() {
|
test_binary_from_listing() {
|
||||||
local listing="$1"
|
local listing="$1"
|
||||||
local case_id="$2"
|
local case_id="$2"
|
||||||
@@ -103,26 +149,32 @@ import sys
|
|||||||
status = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
status = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||||
if status.get("decision") != "blocked" or status.get("release_approved") is not False:
|
if status.get("decision") != "blocked" or status.get("release_approved") is not False:
|
||||||
raise SystemExit("release status did not record a blocked decision")
|
raise SystemExit("release status did not record a blocked decision")
|
||||||
if status.get("release_schema_capable") is not False:
|
if not status.get("pending_gates"):
|
||||||
raise SystemExit("case-only evidence schema unexpectedly became release-capable")
|
raise SystemExit("release status did not retain pending gates")
|
||||||
PY
|
PY
|
||||||
}
|
}
|
||||||
|
|
||||||
run_self_test() {
|
run_self_test() {
|
||||||
local filter
|
|
||||||
filter="$(test_filter_for background-target-crash)"
|
|
||||||
case "$filter" in
|
|
||||||
*background_target_crash*) ;;
|
|
||||||
*)
|
|
||||||
echo "self-test failed: crash case filter missing" >&2
|
|
||||||
return 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
if "$0" --case release --plan-only >/dev/null 2>&1; then
|
if "$0" --case release --plan-only >/dev/null 2>&1; then
|
||||||
echo "self-test failed: release pseudo-case must not be runnable" >&2
|
echo "self-test failed: release pseudo-case must not be runnable" >&2
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
"$0" --case background-target-crash --plan-only >/dev/null
|
local case_id
|
||||||
|
while IFS= read -r case_id; do
|
||||||
|
local expected_filter expected_profile plan
|
||||||
|
expected_filter="$(test_filter_for "$case_id")"
|
||||||
|
expected_profile="$(case_field "$case_id" lane)"
|
||||||
|
expected_runtime_profile="$(runtime_profile_for "$case_id")"
|
||||||
|
plan="$("$0" --case "$case_id" --plan-only)"
|
||||||
|
if [[ "$plan" != *"case=$case_id"* ]] ||
|
||||||
|
[[ "$plan" != *"profile=$expected_profile"* ]] ||
|
||||||
|
[[ "$plan" != *"runtime_profile=$expected_runtime_profile"* ]] ||
|
||||||
|
[[ "$plan" != *"filter=$expected_filter"* ]] ||
|
||||||
|
[[ "$plan" != *"run_dir=$ROOT/target/scanner-heal-evidence/$case_id-"* ]]; then
|
||||||
|
echo "self-test failed: registry case plan mismatch for $case_id" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done < <(case_ids)
|
||||||
}
|
}
|
||||||
|
|
||||||
while [[ $# -gt 0 ]]; do
|
while [[ $# -gt 0 ]]; do
|
||||||
@@ -166,13 +218,10 @@ fi
|
|||||||
|
|
||||||
case_field "$CASE_ID" name >/dev/null
|
case_field "$CASE_ID" name >/dev/null
|
||||||
TEST_FILTER="$(test_filter_for "$CASE_ID")"
|
TEST_FILTER="$(test_filter_for "$CASE_ID")"
|
||||||
case "$CASE_ID" in
|
if [[ -z "$PROFILE" ]]; then
|
||||||
background-target-crash|background-target-restart)
|
PROFILE="$(case_field "$CASE_ID" lane)"
|
||||||
export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}"
|
fi
|
||||||
export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-16777216}"
|
apply_runtime_profile "$CASE_ID"
|
||||||
export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-120}"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
if [[ -z "$RUN_DIR" ]]; then
|
if [[ -z "$RUN_DIR" ]]; then
|
||||||
RUN_DIR="$ROOT/target/scanner-heal-evidence/${CASE_ID}-$(date -u +%Y%m%dT%H%M%SZ)"
|
RUN_DIR="$ROOT/target/scanner-heal-evidence/${CASE_ID}-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||||
elif [[ "$RUN_DIR" != /* ]]; then
|
elif [[ "$RUN_DIR" != /* ]]; then
|
||||||
@@ -182,6 +231,7 @@ fi
|
|||||||
if [[ "$PLAN_ONLY" == 1 ]]; then
|
if [[ "$PLAN_ONLY" == 1 ]]; then
|
||||||
echo "case=$CASE_ID"
|
echo "case=$CASE_ID"
|
||||||
echo "profile=$PROFILE"
|
echo "profile=$PROFILE"
|
||||||
|
echo "runtime_profile=$(runtime_profile_for "$CASE_ID")"
|
||||||
echo "filter=$TEST_FILTER"
|
echo "filter=$TEST_FILTER"
|
||||||
echo "run_dir=$RUN_DIR"
|
echo "run_dir=$RUN_DIR"
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
+124
-1
@@ -29,6 +29,17 @@ METRICS = (
|
|||||||
)
|
)
|
||||||
REPEATABILITY_LIMIT = Decimal("0.05")
|
REPEATABILITY_LIMIT = Decimal("0.05")
|
||||||
P2_WORK_MULTIPLE_LIMIT = Decimal("1.2")
|
P2_WORK_MULTIPLE_LIMIT = Decimal("1.2")
|
||||||
|
RELEASE_PROFILE_ARTIFACTS = (
|
||||||
|
"allocation-profile",
|
||||||
|
"flamegraph",
|
||||||
|
"rss-samples",
|
||||||
|
"save-frequency",
|
||||||
|
)
|
||||||
|
MIN_MEASURED_RELEASE_DURATION_SECONDS = 7200
|
||||||
|
RELEASE_FAULT_MODES = (
|
||||||
|
"process-restart",
|
||||||
|
"process-crash-restart",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def require(condition, message):
|
def require(condition, message):
|
||||||
@@ -120,7 +131,7 @@ def validate_manifest(manifest):
|
|||||||
number(fixed.get("offered_load_ops"), "offered load", 1)
|
number(fixed.get("offered_load_ops"), "offered load", 1)
|
||||||
require(type(manifest.get("rounds")) is int and 3 <= manifest["rounds"] <= 10,
|
require(type(manifest.get("rounds")) is int and 3 <= manifest["rounds"] <= 10,
|
||||||
"rounds must be 3..10")
|
"rounds must be 3..10")
|
||||||
minimum = 900 if manifest["evidence"] == "measured" else 1
|
minimum = MIN_MEASURED_RELEASE_DURATION_SECONDS if manifest["evidence"] == "measured" else 1
|
||||||
require(type(manifest.get("duration_seconds")) is int and
|
require(type(manifest.get("duration_seconds")) is int and
|
||||||
minimum <= manifest["duration_seconds"] <= 86400, "invalid duration_seconds")
|
minimum <= manifest["duration_seconds"] <= 86400, "invalid duration_seconds")
|
||||||
number(manifest.get("min_free_bytes"), "min_free_bytes", 1)
|
number(manifest.get("min_free_bytes"), "min_free_bytes", 1)
|
||||||
@@ -140,6 +151,108 @@ def validate_manifest(manifest):
|
|||||||
number(manifest["expected_healed_objects"].get(scenario), f"{scenario} expected repairs")
|
number(manifest["expected_healed_objects"].get(scenario), f"{scenario} expected repairs")
|
||||||
if scenario in ("running-heal", "mrf-replay"):
|
if scenario in ("running-heal", "mrf-replay"):
|
||||||
require(manifest["expected_healed_objects"][scenario] > 0, f"{scenario} requires repairs")
|
require(manifest["expected_healed_objects"][scenario] > 0, f"{scenario} requires repairs")
|
||||||
|
validate_release_evidence_manifest(manifest)
|
||||||
|
|
||||||
|
|
||||||
|
def release_evidence_integer(value, name, minimum=1, maximum=1024):
|
||||||
|
require(type(value) is int and minimum <= value <= maximum, f"invalid release_evidence.{name}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def release_evidence_string(value, name):
|
||||||
|
require(isinstance(value, str) and value.strip(), f"missing release_evidence.{name}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def release_evidence_bool(value, name):
|
||||||
|
require(type(value) is bool, f"invalid release_evidence.{name}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def release_evidence_true(value, name):
|
||||||
|
release_evidence_bool(value, name)
|
||||||
|
require(value is True, f"missing release_evidence.{name}")
|
||||||
|
|
||||||
|
|
||||||
|
def release_evidence_exact_strings(value, expected, name):
|
||||||
|
require(isinstance(value, list) and all(isinstance(item, str) and item.strip() for item in value),
|
||||||
|
f"invalid release_evidence.{name}")
|
||||||
|
observed = set(value)
|
||||||
|
require(len(observed) == len(value), f"duplicate release_evidence.{name}")
|
||||||
|
missing = sorted(set(expected) - observed)
|
||||||
|
require(not missing, f"missing release_evidence.{name}: {', '.join(missing)}")
|
||||||
|
unknown = sorted(observed - set(expected))
|
||||||
|
require(not unknown, f"unknown release_evidence.{name}: {', '.join(unknown)}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def validate_release_evidence_manifest(manifest):
|
||||||
|
if manifest["evidence"] != "measured":
|
||||||
|
return
|
||||||
|
|
||||||
|
evidence = manifest.get("release_evidence")
|
||||||
|
require(isinstance(evidence, dict), "missing release_evidence for measured ABBA")
|
||||||
|
|
||||||
|
topology = evidence.get("topology")
|
||||||
|
require(isinstance(topology, dict), "missing release_evidence.topology")
|
||||||
|
nodes = release_evidence_integer(topology.get("nodes"), "topology.nodes", 3, 64)
|
||||||
|
drives = release_evidence_integer(topology.get("drives_per_node"), "topology.drives_per_node", 1, 64)
|
||||||
|
set_size = release_evidence_integer(topology.get("erasure_set_size"), "topology.erasure_set_size", 12, 12)
|
||||||
|
data = release_evidence_integer(topology.get("erasure_data_blocks"), "topology.erasure_data_blocks", 8, 8)
|
||||||
|
parity = release_evidence_integer(topology.get("erasure_parity_blocks"), "topology.erasure_parity_blocks", 4, 4)
|
||||||
|
require(data + parity == set_size, "release_evidence.topology must be EC8+4")
|
||||||
|
require(nodes * drives >= set_size, "release_evidence.topology cannot host one EC8+4 set")
|
||||||
|
pools = release_evidence_integer(topology.get("pools"), "topology.pools", 1)
|
||||||
|
sets_total = release_evidence_integer(topology.get("sets_total"), "topology.sets_total", 1)
|
||||||
|
sampled_pools = release_evidence_integer(topology.get("sampled_pools"), "topology.sampled_pools", 2)
|
||||||
|
sampled_sets = release_evidence_integer(topology.get("sampled_sets"), "topology.sampled_sets", 2)
|
||||||
|
require(sampled_pools <= pools, "release_evidence.topology sampled pools exceed total pools")
|
||||||
|
require(sampled_sets <= sets_total, "release_evidence.topology sampled sets exceed total sets")
|
||||||
|
|
||||||
|
distributed = evidence.get("distributed")
|
||||||
|
require(isinstance(distributed, dict), "missing release_evidence.distributed")
|
||||||
|
endpoints = distributed.get("metrics_endpoints")
|
||||||
|
require(isinstance(endpoints, list) and len(endpoints) >= nodes, "missing release_evidence.distributed.metrics_endpoints")
|
||||||
|
require(
|
||||||
|
all(isinstance(endpoint, str) and endpoint.strip() for endpoint in endpoints)
|
||||||
|
and len(set(endpoints)) == len(endpoints),
|
||||||
|
"invalid release_evidence.distributed.metrics_endpoints",
|
||||||
|
)
|
||||||
|
release_evidence_string(distributed.get("failure_domain"), "distributed.failure_domain")
|
||||||
|
release_evidence_true(distributed.get("same_window_sampling"), "distributed.same_window_sampling")
|
||||||
|
|
||||||
|
crash = evidence.get("crash_restart")
|
||||||
|
require(isinstance(crash, dict), "missing release_evidence.crash_restart")
|
||||||
|
release_evidence_exact_strings(crash.get("fault_modes"), RELEASE_FAULT_MODES, "crash_restart.fault_modes")
|
||||||
|
release_evidence_true(crash.get("unclean_shutdown_marker"), "crash_restart.unclean_shutdown_marker")
|
||||||
|
|
||||||
|
mixed = evidence.get("mixed_version")
|
||||||
|
require(isinstance(mixed, dict), "missing release_evidence.mixed_version")
|
||||||
|
baseline_revision = manifest["baseline"]["revision"]
|
||||||
|
candidate_revision = manifest["candidate"]["revision"]
|
||||||
|
require(baseline_revision != candidate_revision,
|
||||||
|
"release_evidence.mixed_version requires distinct baseline and candidate revisions")
|
||||||
|
require(manifest["baseline"]["sha256"] != manifest["candidate"]["sha256"],
|
||||||
|
"release_evidence.mixed_version requires distinct baseline and candidate binaries")
|
||||||
|
revisions = mixed.get("participating_revisions")
|
||||||
|
require(
|
||||||
|
isinstance(revisions, list)
|
||||||
|
and len(set(revisions)) >= 2
|
||||||
|
and all(isinstance(revision, str) and len(revision) == 40 and all(c in "0123456789abcdef" for c in revision)
|
||||||
|
for revision in revisions),
|
||||||
|
"invalid release_evidence.mixed_version.participating_revisions",
|
||||||
|
)
|
||||||
|
for revision in (baseline_revision, candidate_revision):
|
||||||
|
require(revision in revisions, "release_evidence.mixed_version omits tested build revision")
|
||||||
|
for key in ("reader", "writer", "rollback_payload"):
|
||||||
|
require(mixed.get(key) is True, f"missing release_evidence.mixed_version.{key}")
|
||||||
|
|
||||||
|
profile = evidence.get("profile")
|
||||||
|
require(isinstance(profile, dict), "missing release_evidence.profile")
|
||||||
|
release_evidence_exact_strings(profile.get("required_artifacts"), RELEASE_PROFILE_ARTIFACTS,
|
||||||
|
"profile.required_artifacts")
|
||||||
|
for key in ("collector_config_sha256", "profiler_config_sha256"):
|
||||||
|
require(sha(profile.get(key)), f"invalid release_evidence.profile.{key}")
|
||||||
|
|
||||||
|
|
||||||
class OwnedCommand:
|
class OwnedCommand:
|
||||||
@@ -254,6 +367,8 @@ def validate_result(result, request, expected):
|
|||||||
require(result.get("build") == request["build"], "deployed build provenance mismatch")
|
require(result.get("build") == request["build"], "deployed build provenance mismatch")
|
||||||
require(result.get("data_dir") == request["data_dir"], "adapter data isolation mismatch")
|
require(result.get("data_dir") == request["data_dir"], "adapter data isolation mismatch")
|
||||||
require(result.get("background") == request["background"], "background mode mismatch")
|
require(result.get("background") == request["background"], "background mode mismatch")
|
||||||
|
if request["evidence"] == "measured":
|
||||||
|
require(result.get("release_evidence") == request["release_evidence"], "release evidence provenance mismatch")
|
||||||
require(type(result.get("sample_count")) is int and 1 <= result["sample_count"] <= 3600,
|
require(type(result.get("sample_count")) is int and 1 <= result["sample_count"] <= 3600,
|
||||||
"sample_count must be 1..3600")
|
"sample_count must be 1..3600")
|
||||||
number(result.get("elapsed_seconds"), "elapsed_seconds", request["duration_seconds"])
|
number(result.get("elapsed_seconds"), "elapsed_seconds", request["duration_seconds"])
|
||||||
@@ -445,6 +560,9 @@ def collect_live(prepared, request, request_path, adapter):
|
|||||||
connection = prepared["collector"]
|
connection = prepared["collector"]
|
||||||
require(set(connection) == {"alias", "endpoint", "metrics_endpoints"}, "invalid collector connection")
|
require(set(connection) == {"alias", "endpoint", "metrics_endpoints"}, "invalid collector connection")
|
||||||
require(all(isinstance(value, str) and value for value in connection.values()), "missing collector endpoint")
|
require(all(isinstance(value, str) and value for value in connection.values()), "missing collector endpoint")
|
||||||
|
expected_metrics_endpoints = None
|
||||||
|
if request.get("evidence") == "measured":
|
||||||
|
expected_metrics_endpoints = request["release_evidence"]["distributed"]["metrics_endpoints"]
|
||||||
output = request_path.parent / "telemetry"
|
output = request_path.parent / "telemetry"
|
||||||
args = ["bash", str(collector), "--alias", connection["alias"], "--endpoint", connection["endpoint"],
|
args = ["bash", str(collector), "--alias", connection["alias"], "--endpoint", connection["endpoint"],
|
||||||
"--metrics-endpoints", connection["metrics_endpoints"], "--deployment", "distributed",
|
"--metrics-endpoints", connection["metrics_endpoints"], "--deployment", "distributed",
|
||||||
@@ -470,6 +588,9 @@ def collect_live(prepared, request, request_path, adapter):
|
|||||||
require(isinstance(status.get("healOperations"), dict) and status["healOperations"], "invalid heal status response")
|
require(isinstance(status.get("healOperations"), dict) and status["healOperations"], "invalid heal status response")
|
||||||
metrics = list((output / "metrics").glob("admin-metrics.*.ndjson"))
|
metrics = list((output / "metrics").glob("admin-metrics.*.ndjson"))
|
||||||
endpoints = [endpoint for endpoint in connection["metrics_endpoints"].split(",") if endpoint]
|
endpoints = [endpoint for endpoint in connection["metrics_endpoints"].split(",") if endpoint]
|
||||||
|
if expected_metrics_endpoints is not None:
|
||||||
|
require(endpoints == expected_metrics_endpoints,
|
||||||
|
"collector metrics endpoints do not match release evidence")
|
||||||
require(metrics and len(metrics) == len(endpoints) * len(samples), "missing distributed metrics samples")
|
require(metrics and len(metrics) == len(endpoints) * len(samples), "missing distributed metrics samples")
|
||||||
for sample in metrics:
|
for sample in metrics:
|
||||||
# The collector requests n=1, so each file contains one final JSON record.
|
# The collector requests n=1, so each file contains one final JSON record.
|
||||||
@@ -518,6 +639,8 @@ def run(manifest, adapter, output, data_root):
|
|||||||
"duration_seconds": manifest["duration_seconds"], "data_dir": str(data_dir),
|
"duration_seconds": manifest["duration_seconds"], "data_dir": str(data_dir),
|
||||||
"expected_healed_objects": manifest["expected_healed_objects"][scenario],
|
"expected_healed_objects": manifest["expected_healed_objects"][scenario],
|
||||||
"expected_oracle": manifest["oracles"][scenario]}
|
"expected_oracle": manifest["oracles"][scenario]}
|
||||||
|
if manifest["evidence"] == "measured":
|
||||||
|
request["release_evidence"] = manifest["release_evidence"]
|
||||||
require(digest(Path(request["build"]["binary"])) == request["build"]["sha256"], "binary changed during run")
|
require(digest(Path(request["build"]["binary"])) == request["build"]["sha256"], "binary changed during run")
|
||||||
require(digest(adapter) == manifest["adapter_sha256"], "adapter changed during run")
|
require(digest(adapter) == manifest["adapter_sha256"], "adapter changed during run")
|
||||||
require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space")
|
require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space")
|
||||||
|
|||||||
@@ -12,6 +12,13 @@ from pathlib import Path
|
|||||||
import sys
|
import sys
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from scanner_abba import (
|
||||||
|
LEGS,
|
||||||
|
MIN_MEASURED_RELEASE_DURATION_SECONDS,
|
||||||
|
SCENARIOS,
|
||||||
|
validate_release_evidence_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
MAX_JSON_BYTES = 1024 * 1024
|
MAX_JSON_BYTES = 1024 * 1024
|
||||||
CACHE_COST_PREFIX = "CACHE_COST "
|
CACHE_COST_PREFIX = "CACHE_COST "
|
||||||
PASS_STATES = {"pass"}
|
PASS_STATES = {"pass"}
|
||||||
@@ -77,12 +84,92 @@ def max_decimal(values: list[Decimal | None]) -> Decimal | None:
|
|||||||
return max(present)
|
return max(present)
|
||||||
|
|
||||||
|
|
||||||
|
def require_metric_series(value: Any, name: str, minimum: Decimal | None = None,
|
||||||
|
maximum: Decimal | None = None) -> list[Decimal | None]:
|
||||||
|
require(isinstance(value, list) and value, f"missing performance evidence field: {name}")
|
||||||
|
parsed = [maybe_number(item, name) for item in value]
|
||||||
|
for item in parsed:
|
||||||
|
if item is None:
|
||||||
|
continue
|
||||||
|
if minimum is not None:
|
||||||
|
require(item >= minimum, f"{name} below minimum")
|
||||||
|
if maximum is not None:
|
||||||
|
require(item <= maximum, f"{name} above maximum")
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
def require_measured_comparison_evidence(comparison: dict[str, Any], index: int) -> None:
|
||||||
|
w10_w11 = comparison.get("w10_w11")
|
||||||
|
require(isinstance(w10_w11, dict), f"comparison {index} missing W10/W11 evidence")
|
||||||
|
pressure = require_metric_series(
|
||||||
|
w10_w11.get("foreground_pressure_high_sample_ratios"),
|
||||||
|
f"comparison {index} foreground_pressure_high_sample_ratios",
|
||||||
|
Decimal("0"),
|
||||||
|
Decimal("1"),
|
||||||
|
)
|
||||||
|
lock_wait = require_metric_series(
|
||||||
|
w10_w11.get("heal_lock_wait_p99_ms"),
|
||||||
|
f"comparison {index} heal_lock_wait_p99_ms",
|
||||||
|
Decimal("0"),
|
||||||
|
)
|
||||||
|
attempt_cost = require_metric_series(
|
||||||
|
w10_w11.get("attempt_cost_per_healed_object"),
|
||||||
|
f"comparison {index} attempt_cost_per_healed_object",
|
||||||
|
Decimal("0"),
|
||||||
|
)
|
||||||
|
require(len(pressure) == len(lock_wait) == len(attempt_cost),
|
||||||
|
f"comparison {index} W10/W11 evidence length mismatch")
|
||||||
|
candidate_attempt_cost = maybe_number(
|
||||||
|
w10_w11.get("candidate_attempt_cost_per_healed_object"),
|
||||||
|
f"comparison {index} candidate_attempt_cost_per_healed_object",
|
||||||
|
)
|
||||||
|
require(candidate_attempt_cost is None or candidate_attempt_cost >= 0,
|
||||||
|
f"comparison {index} candidate attempt cost below minimum")
|
||||||
|
|
||||||
|
|
||||||
|
def require_complete_abba_matrix(manifest: dict[str, Any], report: dict[str, Any], comparisons: list[dict[str, Any]]) -> None:
|
||||||
|
require(report.get("evidence") == manifest.get("evidence"), "manifest/report evidence mismatch")
|
||||||
|
require(type(manifest.get("duration_seconds")) is int and
|
||||||
|
manifest["duration_seconds"] >= MIN_MEASURED_RELEASE_DURATION_SECONDS,
|
||||||
|
"measured ABBA duration_seconds requires at least two hours")
|
||||||
|
rounds = manifest.get("rounds")
|
||||||
|
require(type(rounds) is int and 3 <= rounds <= 10, "invalid manifest.rounds")
|
||||||
|
expected_cells = len(SCENARIOS) * 2 * rounds * len(LEGS)
|
||||||
|
require(
|
||||||
|
report.get("cells") == expected_cells,
|
||||||
|
f"ABBA matrix cell count mismatch: expected {expected_cells}, got {report.get('cells')}",
|
||||||
|
)
|
||||||
|
expected_keys = {
|
||||||
|
(scenario, comparison, round_id)
|
||||||
|
for scenario in SCENARIOS
|
||||||
|
for comparison in ("build", "background")
|
||||||
|
for round_id in range(1, rounds + 1)
|
||||||
|
}
|
||||||
|
observed_keys = []
|
||||||
|
for index, comparison in enumerate(comparisons):
|
||||||
|
key = (comparison.get("scenario"), comparison.get("comparison"), comparison.get("round"))
|
||||||
|
require(key in expected_keys, f"comparison {index} is outside the ABBA matrix")
|
||||||
|
require(comparison.get("status") in PASS_STATES, f"comparison {index} did not pass")
|
||||||
|
observed_keys.append(key)
|
||||||
|
observed_set = set(observed_keys)
|
||||||
|
require(len(observed_keys) == len(observed_set), "duplicate ABBA matrix comparison")
|
||||||
|
missing = sorted(expected_keys - observed_set)
|
||||||
|
require(not missing, f"missing ABBA matrix comparison: {missing[0] if missing else ''}")
|
||||||
|
|
||||||
|
|
||||||
def summarize_abba(abba_dir: Path) -> dict[str, Any]:
|
def summarize_abba(abba_dir: Path) -> dict[str, Any]:
|
||||||
manifest_path = abba_dir / "manifest.json"
|
manifest_path = abba_dir / "manifest.json"
|
||||||
report_path = abba_dir / "report.json"
|
report_path = abba_dir / "report.json"
|
||||||
manifest = read_json(manifest_path)
|
manifest = read_json(manifest_path)
|
||||||
report = read_json(report_path)
|
report = read_json(report_path)
|
||||||
|
report_state = report.get("status")
|
||||||
|
performance_state = report.get("performance")
|
||||||
|
require(isinstance(report_state, str) and report_state, "report.status missing")
|
||||||
|
require(isinstance(performance_state, str) and performance_state, "report.performance missing")
|
||||||
comparisons = report.get("comparisons")
|
comparisons = report.get("comparisons")
|
||||||
|
if comparisons is None:
|
||||||
|
require(report_state not in PASS_STATES, "passing report requires comparisons")
|
||||||
|
comparisons = []
|
||||||
require(isinstance(comparisons, list), "report.comparisons must be a list")
|
require(isinstance(comparisons, list), "report.comparisons must be a list")
|
||||||
|
|
||||||
counts = Counter()
|
counts = Counter()
|
||||||
@@ -115,15 +202,18 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]:
|
|||||||
if isinstance(p2, list):
|
if isinstance(p2, list):
|
||||||
p2_values.extend(maybe_number(value, "p2_post_stop_work_multiple") for value in p2)
|
p2_values.extend(maybe_number(value, "p2_post_stop_work_multiple") for value in p2)
|
||||||
|
|
||||||
report_state = report.get("status")
|
|
||||||
performance_state = report.get("performance")
|
|
||||||
require(isinstance(report_state, str) and report_state, "report.status missing")
|
|
||||||
require(isinstance(performance_state, str) and performance_state, "report.performance missing")
|
|
||||||
measured = report.get("evidence") == "measured"
|
measured = report.get("evidence") == "measured"
|
||||||
passed = report_state in PASS_STATES and performance_state in PASS_STATES and measured
|
passed = report_state in PASS_STATES and performance_state in PASS_STATES and measured
|
||||||
|
if passed:
|
||||||
|
require_complete_abba_matrix(manifest, report, comparisons)
|
||||||
|
validate_release_evidence_manifest({**manifest, "evidence": "measured"})
|
||||||
|
for index, comparison in enumerate(comparisons):
|
||||||
|
require_measured_comparison_evidence(comparison, index)
|
||||||
gate_state = "pass" if passed else "fail"
|
gate_state = "pass" if passed else "fail"
|
||||||
if report_state == "synthetic_validated":
|
if report_state == "synthetic_validated":
|
||||||
reason = "synthetic evidence validates the harness only; measured performance remains pending"
|
reason = "synthetic evidence validates the harness only; measured performance remains pending"
|
||||||
|
elif report_state in FAIL_STATES and isinstance(report.get("error"), str) and report["error"]:
|
||||||
|
reason = f"ABBA report status is {report_state}: {report['error']}"
|
||||||
elif report_state not in PASS_STATES:
|
elif report_state not in PASS_STATES:
|
||||||
reason = f"ABBA report status is {report_state}"
|
reason = f"ABBA report status is {report_state}"
|
||||||
elif performance_state not in PASS_STATES:
|
elif performance_state not in PASS_STATES:
|
||||||
@@ -142,6 +232,8 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]:
|
|||||||
"performance": performance_state,
|
"performance": performance_state,
|
||||||
"evidence": report.get("evidence"),
|
"evidence": report.get("evidence"),
|
||||||
"cells": report.get("cells", 0),
|
"cells": report.get("cells", 0),
|
||||||
|
"completed_cells": report.get("completed_cells"),
|
||||||
|
"error": report.get("error"),
|
||||||
"comparisons_total": len(comparisons),
|
"comparisons_total": len(comparisons),
|
||||||
"comparison_status_counts": dict(sorted(counts.items())),
|
"comparison_status_counts": dict(sorted(counts.items())),
|
||||||
"worst_p99_regression": None if not p99_regressions else float(max(p99_regressions)),
|
"worst_p99_regression": None if not p99_regressions else float(max(p99_regressions)),
|
||||||
@@ -164,6 +256,7 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]:
|
|||||||
"durability": fixed.get("durability"),
|
"durability": fixed.get("durability"),
|
||||||
"topology": fixed.get("topology"),
|
"topology": fixed.get("topology"),
|
||||||
"offered_load_ops": fixed.get("offered_load_ops"),
|
"offered_load_ops": fixed.get("offered_load_ops"),
|
||||||
|
"release_evidence": manifest.get("release_evidence"),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -247,6 +340,10 @@ def markdown(summary: dict[str, Any]) -> str:
|
|||||||
f"- worst_throughput_loss: {pct(throughput)}",
|
f"- worst_throughput_loss: {pct(throughput)}",
|
||||||
f"- p2_worst_post_stop_work_multiple: {ratio(p2)}",
|
f"- p2_worst_post_stop_work_multiple: {ratio(p2)}",
|
||||||
]
|
]
|
||||||
|
if abba.get("completed_cells") is not None:
|
||||||
|
lines.append(f"- completed_cells: {abba['completed_cells']}")
|
||||||
|
if abba.get("error"):
|
||||||
|
lines.append(f"- error: {abba['error']}")
|
||||||
if summary.get("cache_cost") is not None:
|
if summary.get("cache_cost") is not None:
|
||||||
cache = summary["cache_cost"]
|
cache = summary["cache_cost"]
|
||||||
lines.extend([
|
lines.extend([
|
||||||
|
|||||||
@@ -49,6 +49,8 @@ def fake_adapter():
|
|||||||
if fault == "measure-exit":
|
if fault == "measure-exit":
|
||||||
return 42
|
return 42
|
||||||
result = {key: request[key] for key in ("evidence", "fixed", "build", "data_dir", "background")}
|
result = {key: request[key] for key in ("evidence", "fixed", "build", "data_dir", "background")}
|
||||||
|
if request["evidence"] == "measured":
|
||||||
|
result["release_evidence"] = copy.deepcopy(request["release_evidence"])
|
||||||
result.update({"sample_count": 10, "elapsed_seconds": request["duration_seconds"],
|
result.update({"sample_count": 10, "elapsed_seconds": request["duration_seconds"],
|
||||||
"metrics": dict.fromkeys(harness.METRICS, 10)})
|
"metrics": dict.fromkeys(harness.METRICS, 10)})
|
||||||
baseline = request["comparison"] == "build" and request["leg"].startswith("A")
|
baseline = request["comparison"] == "build" and request["leg"].startswith("A")
|
||||||
@@ -163,6 +165,52 @@ class ScannerAbbaTest(unittest.TestCase):
|
|||||||
build = {"binary": str(self.binary), "sha256": harness.digest(self.binary), "revision": "a" * 40}
|
build = {"binary": str(self.binary), "sha256": harness.digest(self.binary), "revision": "a" * 40}
|
||||||
self.manifest.update(baseline=build.copy(), candidate=build.copy())
|
self.manifest.update(baseline=build.copy(), candidate=build.copy())
|
||||||
|
|
||||||
|
def measured_manifest(self):
|
||||||
|
manifest = copy.deepcopy(self.manifest)
|
||||||
|
manifest.update(evidence="measured", duration_seconds=harness.MIN_MEASURED_RELEASE_DURATION_SECONDS)
|
||||||
|
candidate_binary = self.root / "candidate-python"
|
||||||
|
candidate_binary.write_bytes(self.binary.read_bytes() + b"\n")
|
||||||
|
candidate_binary.chmod(0o755)
|
||||||
|
manifest["candidate"] = {
|
||||||
|
"binary": str(candidate_binary),
|
||||||
|
"sha256": harness.digest(candidate_binary),
|
||||||
|
"revision": "b" * 40,
|
||||||
|
}
|
||||||
|
manifest["release_evidence"] = {
|
||||||
|
"topology": {
|
||||||
|
"nodes": 3,
|
||||||
|
"drives_per_node": 4,
|
||||||
|
"pools": 2,
|
||||||
|
"sets_total": 2,
|
||||||
|
"sampled_pools": 2,
|
||||||
|
"sampled_sets": 2,
|
||||||
|
"erasure_set_size": 12,
|
||||||
|
"erasure_data_blocks": 8,
|
||||||
|
"erasure_parity_blocks": 4,
|
||||||
|
},
|
||||||
|
"distributed": {
|
||||||
|
"metrics_endpoints": ["https://node-1:9000", "https://node-2:9000", "https://node-3:9000"],
|
||||||
|
"failure_domain": "three-node-localhost-lab",
|
||||||
|
"same_window_sampling": True,
|
||||||
|
},
|
||||||
|
"crash_restart": {
|
||||||
|
"fault_modes": ["process-restart", "process-crash-restart"],
|
||||||
|
"unclean_shutdown_marker": True,
|
||||||
|
},
|
||||||
|
"mixed_version": {
|
||||||
|
"participating_revisions": ["a" * 40, "b" * 40],
|
||||||
|
"reader": True,
|
||||||
|
"writer": True,
|
||||||
|
"rollback_payload": True,
|
||||||
|
},
|
||||||
|
"profile": {
|
||||||
|
"required_artifacts": ["allocation-profile", "flamegraph", "rss-samples", "save-frequency"],
|
||||||
|
"collector_config_sha256": "4" * 64,
|
||||||
|
"profiler_config_sha256": "5" * 64,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return manifest
|
||||||
|
|
||||||
def run_harness(self, fault=""):
|
def run_harness(self, fault=""):
|
||||||
with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": fault}), contextlib.redirect_stdout(io.StringIO()):
|
with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": fault}), contextlib.redirect_stdout(io.StringIO()):
|
||||||
return harness.run(copy.deepcopy(self.manifest), self.adapter, self.root / "out", self.root / "data")
|
return harness.run(copy.deepcopy(self.manifest), self.adapter, self.root / "out", self.root / "data")
|
||||||
@@ -440,6 +488,34 @@ class ScannerAbbaTest(unittest.TestCase):
|
|||||||
|
|
||||||
process.finish.assert_called_once_with(terminate=True)
|
process.finish.assert_called_once_with(terminate=True)
|
||||||
|
|
||||||
|
def test_live_collector_binds_release_evidence_metrics_endpoints(self):
|
||||||
|
telemetry = self.root / "telemetry"
|
||||||
|
for name in ("status", "heal", "metrics"):
|
||||||
|
(telemetry / name).mkdir(parents=True)
|
||||||
|
(telemetry / "scanner-summary.csv").write_text("timestamp\n")
|
||||||
|
for index in range(16):
|
||||||
|
harness.write_json(telemetry / f"status/scanner-status.{index}.json", {"metrics": {"objects": 10}})
|
||||||
|
for node in ("node-a", "node-b"):
|
||||||
|
harness.write_json(telemetry / f"heal/background-heal-status.{node}.{index}.json",
|
||||||
|
{"healOperations": {"queueLength": 0}})
|
||||||
|
harness.write_json(telemetry / f"metrics/admin-metrics.{node}.{index}.ndjson",
|
||||||
|
{"errors": [], "final": True,
|
||||||
|
"by_host": {f"{node}:9000": {"scanner": {"objects": 10}}}})
|
||||||
|
prepared = {"collector": {"alias": "test", "endpoint": "http://node-a:9000",
|
||||||
|
"metrics_endpoints": "http://node-a:9000,http://node-b:9000"}}
|
||||||
|
request = {
|
||||||
|
"duration_seconds": 900,
|
||||||
|
"evidence": "measured",
|
||||||
|
"release_evidence": self.measured_manifest()["release_evidence"],
|
||||||
|
}
|
||||||
|
process = Mock(pid=123, wait=Mock(return_value=0))
|
||||||
|
with patch.object(harness, "OwnedCommand", return_value=process), \
|
||||||
|
patch.object(harness, "invoke", return_value={"sample_count": 10}), \
|
||||||
|
patch.object(harness.time, "monotonic", side_effect=(0, 900)):
|
||||||
|
with self.assertRaisesRegex(ValueError, "collector metrics endpoints"):
|
||||||
|
harness.collect_live(prepared, request, self.root / "request.json", self.adapter)
|
||||||
|
process.finish.assert_called_once_with(terminate=True)
|
||||||
|
|
||||||
def test_unstable_p1_work_control_is_inconclusive(self):
|
def test_unstable_p1_work_control_is_inconclusive(self):
|
||||||
with patch.object(harness, "SCENARIOS", ("cold-hot",)):
|
with patch.object(harness, "SCENARIOS", ("cold-hot",)):
|
||||||
self.assertEqual(self.run_harness("unstable-p1-control"), 3)
|
self.assertEqual(self.run_harness("unstable-p1-control"), 3)
|
||||||
@@ -458,11 +534,107 @@ class ScannerAbbaTest(unittest.TestCase):
|
|||||||
self.manifest["evidence"] = "measured"
|
self.manifest["evidence"] = "measured"
|
||||||
with self.assertRaisesRegex(ValueError, "duration_seconds"):
|
with self.assertRaisesRegex(ValueError, "duration_seconds"):
|
||||||
harness.validate_manifest(self.manifest)
|
harness.validate_manifest(self.manifest)
|
||||||
self.manifest["duration_seconds"] = 900
|
self.manifest["duration_seconds"] = harness.MIN_MEASURED_RELEASE_DURATION_SECONDS
|
||||||
self.manifest["rounds"] = 2
|
self.manifest["rounds"] = 2
|
||||||
with self.assertRaisesRegex(ValueError, "rounds"):
|
with self.assertRaisesRegex(ValueError, "rounds"):
|
||||||
harness.validate_manifest(self.manifest)
|
harness.validate_manifest(self.manifest)
|
||||||
|
|
||||||
|
def test_measured_manifest_requires_release_evidence_contract(self):
|
||||||
|
harness.validate_manifest(self.measured_manifest())
|
||||||
|
faults = {
|
||||||
|
"missing root": lambda manifest: manifest.pop("release_evidence"),
|
||||||
|
"single-set": lambda manifest: manifest["release_evidence"]["topology"].update(sets_total=1),
|
||||||
|
"unsampled-set": lambda manifest: manifest["release_evidence"]["topology"].update(sampled_sets=1),
|
||||||
|
"wrong geometry": lambda manifest: manifest["release_evidence"]["topology"].update(erasure_set_size=11),
|
||||||
|
"duplicate endpoint": lambda manifest: manifest["release_evidence"]["distributed"].update(
|
||||||
|
metrics_endpoints=["https://node-1:9000", "https://node-1:9000", "https://node-3:9000"],
|
||||||
|
),
|
||||||
|
"split sampling": lambda manifest: manifest["release_evidence"]["distributed"].update(
|
||||||
|
same_window_sampling=False,
|
||||||
|
),
|
||||||
|
"missing crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update(
|
||||||
|
fault_modes=["process-restart"],
|
||||||
|
),
|
||||||
|
"unknown crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update(
|
||||||
|
fault_modes=["process-restart", "process-crash-restart", "kernel-panic"],
|
||||||
|
),
|
||||||
|
"duplicate crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update(
|
||||||
|
fault_modes=["process-restart", "process-restart", "process-crash-restart"],
|
||||||
|
),
|
||||||
|
"clean crash marker": lambda manifest: manifest["release_evidence"]["crash_restart"].update(
|
||||||
|
unclean_shutdown_marker=False,
|
||||||
|
),
|
||||||
|
"mixed version false": lambda manifest: manifest["release_evidence"]["mixed_version"].update(writer=False),
|
||||||
|
"missing candidate": lambda manifest: manifest["release_evidence"]["mixed_version"].update(
|
||||||
|
participating_revisions=["a" * 40, "c" * 40],
|
||||||
|
),
|
||||||
|
"same mixed revision": lambda manifest: manifest["candidate"].update(
|
||||||
|
revision=manifest["baseline"]["revision"],
|
||||||
|
),
|
||||||
|
"same mixed binary": lambda manifest: manifest["candidate"].update(
|
||||||
|
binary=manifest["baseline"]["binary"],
|
||||||
|
sha256=manifest["baseline"]["sha256"],
|
||||||
|
),
|
||||||
|
"missing profile": lambda manifest: manifest["release_evidence"]["profile"].update(
|
||||||
|
required_artifacts=["allocation-profile", "flamegraph", "rss-samples"],
|
||||||
|
),
|
||||||
|
"unknown profile": lambda manifest: manifest["release_evidence"]["profile"].update(
|
||||||
|
required_artifacts=["allocation-profile", "flamegraph", "rss-samples", "save-frequency", "heapdump"],
|
||||||
|
),
|
||||||
|
"duplicate profile": lambda manifest: manifest["release_evidence"]["profile"].update(
|
||||||
|
required_artifacts=[
|
||||||
|
"allocation-profile", "flamegraph", "rss-samples", "save-frequency", "flamegraph",
|
||||||
|
],
|
||||||
|
),
|
||||||
|
"bad profile hash": lambda manifest: manifest["release_evidence"]["profile"].update(
|
||||||
|
profiler_config_sha256="not-a-sha",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
for name, mutate in faults.items():
|
||||||
|
with self.subTest(fault=name):
|
||||||
|
manifest = self.measured_manifest()
|
||||||
|
mutate(manifest)
|
||||||
|
with self.assertRaisesRegex(ValueError, "release_evidence"):
|
||||||
|
harness.validate_manifest(manifest)
|
||||||
|
|
||||||
|
def test_measured_result_must_echo_release_evidence(self):
|
||||||
|
manifest = self.measured_manifest()
|
||||||
|
request = {
|
||||||
|
"schema": 1,
|
||||||
|
"scenario": "cold-hot",
|
||||||
|
"comparison": "build",
|
||||||
|
"round": 1,
|
||||||
|
"leg": "B1",
|
||||||
|
"background": "on",
|
||||||
|
"build": manifest["candidate"],
|
||||||
|
"evidence": manifest["evidence"],
|
||||||
|
"fixed": manifest["fixed"],
|
||||||
|
"release_evidence": manifest["release_evidence"],
|
||||||
|
"duration_seconds": manifest["duration_seconds"],
|
||||||
|
"data_dir": str(self.root / "data"),
|
||||||
|
"expected_healed_objects": manifest["expected_healed_objects"]["cold-hot"],
|
||||||
|
}
|
||||||
|
metrics = dict.fromkeys(harness.METRICS, 10)
|
||||||
|
metrics.update(p99_ms=10, throughput_ops=100, errors=0, requests=100,
|
||||||
|
walk_objects=100, cold_walk_objects=20, healed_objects=10)
|
||||||
|
result = {
|
||||||
|
"evidence": request["evidence"],
|
||||||
|
"fixed": request["fixed"],
|
||||||
|
"build": request["build"],
|
||||||
|
"data_dir": request["data_dir"],
|
||||||
|
"background": request["background"],
|
||||||
|
"release_evidence": request["release_evidence"],
|
||||||
|
"sample_count": 10,
|
||||||
|
"elapsed_seconds": request["duration_seconds"],
|
||||||
|
"metrics": metrics,
|
||||||
|
"oracle": manifest["oracles"]["cold-hot"],
|
||||||
|
}
|
||||||
|
harness.validate_result(result, request, manifest["oracles"]["cold-hot"])
|
||||||
|
result["release_evidence"] = copy.deepcopy(result["release_evidence"])
|
||||||
|
result["release_evidence"]["profile"]["required_artifacts"].remove("flamegraph")
|
||||||
|
with self.assertRaisesRegex(ValueError, "release evidence provenance mismatch"):
|
||||||
|
harness.validate_result(result, request, manifest["oracles"]["cold-hot"])
|
||||||
|
|
||||||
def test_existing_data_preserved(self):
|
def test_existing_data_preserved(self):
|
||||||
(self.root / "data").mkdir()
|
(self.root / "data").mkdir()
|
||||||
marker = self.root / "data/keep"
|
marker = self.root / "data/keep"
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
import contextlib
|
import contextlib
|
||||||
import hashlib
|
import hashlib
|
||||||
import io
|
import io
|
||||||
@@ -33,6 +34,10 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
|
|||||||
self.abba = self.root / "abba"
|
self.abba = self.root / "abba"
|
||||||
self.abba.mkdir()
|
self.abba.mkdir()
|
||||||
self.manifest = {
|
self.manifest = {
|
||||||
|
"schema": 1,
|
||||||
|
"evidence": "measured",
|
||||||
|
"rounds": 3,
|
||||||
|
"duration_seconds": summary.MIN_MEASURED_RELEASE_DURATION_SECONDS,
|
||||||
"fixed": {
|
"fixed": {
|
||||||
"config_sha256": "1" * 64,
|
"config_sha256": "1" * 64,
|
||||||
"dataset_sha256": "2" * 64,
|
"dataset_sha256": "2" * 64,
|
||||||
@@ -45,6 +50,39 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
|
|||||||
"candidate": {"revision": "b" * 40, "sha256": "4" * 64},
|
"candidate": {"revision": "b" * 40, "sha256": "4" * 64},
|
||||||
"adapter_sha256": "5" * 64,
|
"adapter_sha256": "5" * 64,
|
||||||
"collector_sha256": "6" * 64,
|
"collector_sha256": "6" * 64,
|
||||||
|
"release_evidence": {
|
||||||
|
"topology": {
|
||||||
|
"nodes": 3,
|
||||||
|
"drives_per_node": 4,
|
||||||
|
"pools": 2,
|
||||||
|
"sets_total": 2,
|
||||||
|
"sampled_pools": 2,
|
||||||
|
"sampled_sets": 2,
|
||||||
|
"erasure_set_size": 12,
|
||||||
|
"erasure_data_blocks": 8,
|
||||||
|
"erasure_parity_blocks": 4,
|
||||||
|
},
|
||||||
|
"distributed": {
|
||||||
|
"metrics_endpoints": ["https://node-1:9000", "https://node-2:9000", "https://node-3:9000"],
|
||||||
|
"failure_domain": "three-node-localhost-lab",
|
||||||
|
"same_window_sampling": True,
|
||||||
|
},
|
||||||
|
"crash_restart": {
|
||||||
|
"fault_modes": ["process-restart", "process-crash-restart"],
|
||||||
|
"unclean_shutdown_marker": True,
|
||||||
|
},
|
||||||
|
"mixed_version": {
|
||||||
|
"participating_revisions": ["a" * 40, "b" * 40],
|
||||||
|
"reader": True,
|
||||||
|
"writer": True,
|
||||||
|
"rollback_payload": True,
|
||||||
|
},
|
||||||
|
"profile": {
|
||||||
|
"required_artifacts": ["allocation-profile", "flamegraph", "rss-samples", "save-frequency"],
|
||||||
|
"collector_config_sha256": "7" * 64,
|
||||||
|
"profiler_config_sha256": "8" * 64,
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
self.comparison = {
|
self.comparison = {
|
||||||
"scenario": "cold-hot",
|
"scenario": "cold-hot",
|
||||||
@@ -55,16 +93,32 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
|
|||||||
"throughput_change": -0.01,
|
"throughput_change": -0.01,
|
||||||
"p1": {"required_reduction": 0.8, "observed_reduction": 0.82, "repeatability_drift": 0.01},
|
"p1": {"required_reduction": 0.8, "observed_reduction": 0.82, "repeatability_drift": 0.01},
|
||||||
"p2_post_stop_work_multiples": [None, 1.1, 1.0, None],
|
"p2_post_stop_work_multiples": [None, 1.1, 1.0, None],
|
||||||
|
"w10_w11": {
|
||||||
|
"foreground_pressure_high_sample_ratios": [0.0, 0.25, 0.25, 0.0],
|
||||||
|
"heal_lock_wait_p99_ms": [12.0, 8.0, 9.0, 13.0],
|
||||||
|
"attempt_cost_per_healed_object": [None, 1.2, 1.3, None],
|
||||||
|
"candidate_attempt_cost_per_healed_object": 1.3,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
self.report = {
|
self.report = {
|
||||||
"status": "pass",
|
"status": "pass",
|
||||||
"performance": "pass",
|
"performance": "pass",
|
||||||
"evidence": "measured",
|
"evidence": "measured",
|
||||||
"cells": 120,
|
"cells": 120,
|
||||||
"comparisons": [self.comparison],
|
"comparisons": self.full_comparisons(),
|
||||||
}
|
}
|
||||||
self.write_inputs()
|
self.write_inputs()
|
||||||
|
|
||||||
|
def full_comparisons(self):
|
||||||
|
comparisons = []
|
||||||
|
for scenario in summary.SCENARIOS:
|
||||||
|
for comparison in ("build", "background"):
|
||||||
|
for round_id in range(1, 4):
|
||||||
|
row = copy.deepcopy(self.comparison)
|
||||||
|
row.update(scenario=scenario, comparison=comparison, round=round_id)
|
||||||
|
comparisons.append(row)
|
||||||
|
return comparisons
|
||||||
|
|
||||||
def write_inputs(self):
|
def write_inputs(self):
|
||||||
(self.abba / "manifest.json").write_text(json.dumps(self.manifest), encoding="utf-8")
|
(self.abba / "manifest.json").write_text(json.dumps(self.manifest), encoding="utf-8")
|
||||||
(self.abba / "report.json").write_text(json.dumps(self.report), encoding="utf-8")
|
(self.abba / "report.json").write_text(json.dumps(self.report), encoding="utf-8")
|
||||||
@@ -112,6 +166,120 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
|
|||||||
self.assertEqual(result["verdict"], "FAIL")
|
self.assertEqual(result["verdict"], "FAIL")
|
||||||
self.assertIn("synthetic evidence", result["reason"])
|
self.assertIn("synthetic evidence", result["reason"])
|
||||||
|
|
||||||
|
def test_failed_abba_report_without_comparisons_writes_fail_closed_summary(self):
|
||||||
|
self.report = {
|
||||||
|
"status": "failed",
|
||||||
|
"performance": "pending",
|
||||||
|
"completed_cells": 7,
|
||||||
|
"error": "collector failed",
|
||||||
|
}
|
||||||
|
self.write_inputs()
|
||||||
|
args = type("Args", (), {
|
||||||
|
"abba_dir": self.abba,
|
||||||
|
"cache_cost_log": None,
|
||||||
|
"require_cache_cost": False,
|
||||||
|
"json_out": None,
|
||||||
|
"markdown_out": None,
|
||||||
|
})
|
||||||
|
result = summary.build_summary(args)
|
||||||
|
self.assertEqual(result["verdict"], "FAIL")
|
||||||
|
self.assertEqual(result["abba"]["completed_cells"], 7)
|
||||||
|
self.assertEqual(result["abba"]["comparisons_total"], 0)
|
||||||
|
self.assertIn("collector failed", result["reason"])
|
||||||
|
self.assertIn("- completed_cells: 7", summary.markdown(result))
|
||||||
|
self.assertIn("- error: collector failed", summary.markdown(result))
|
||||||
|
|
||||||
|
def test_passing_abba_report_requires_comparisons(self):
|
||||||
|
del self.report["comparisons"]
|
||||||
|
self.write_inputs()
|
||||||
|
args = type("Args", (), {
|
||||||
|
"abba_dir": self.abba,
|
||||||
|
"cache_cost_log": None,
|
||||||
|
"require_cache_cost": False,
|
||||||
|
"json_out": None,
|
||||||
|
"markdown_out": None,
|
||||||
|
})
|
||||||
|
with self.assertRaisesRegex(ValueError, "passing report requires comparisons"):
|
||||||
|
summary.build_summary(args)
|
||||||
|
|
||||||
|
def test_passing_abba_report_requires_complete_matrix(self):
|
||||||
|
cases = {
|
||||||
|
"trimmed": lambda: self.report["comparisons"].pop(),
|
||||||
|
"duplicate": lambda: self.report["comparisons"].__setitem__(1, copy.deepcopy(self.report["comparisons"][0])),
|
||||||
|
"bad cells": lambda: self.report.update(cells=119),
|
||||||
|
"bad evidence": lambda: self.manifest.update(evidence="synthetic"),
|
||||||
|
"outside": lambda: self.report["comparisons"][0].update(round=99),
|
||||||
|
"failed comparison": lambda: self.report["comparisons"][0].update(status="inconclusive"),
|
||||||
|
}
|
||||||
|
for name, mutate in cases.items():
|
||||||
|
with self.subTest(fault=name):
|
||||||
|
self.setUp()
|
||||||
|
mutate()
|
||||||
|
self.write_inputs()
|
||||||
|
args = type("Args", (), {
|
||||||
|
"abba_dir": self.abba,
|
||||||
|
"cache_cost_log": None,
|
||||||
|
"require_cache_cost": False,
|
||||||
|
"json_out": None,
|
||||||
|
"markdown_out": None,
|
||||||
|
})
|
||||||
|
with self.assertRaisesRegex(ValueError, "ABBA matrix|manifest/report evidence|comparison"):
|
||||||
|
summary.build_summary(args)
|
||||||
|
|
||||||
|
def test_passing_measured_report_requires_two_hour_window(self):
|
||||||
|
self.manifest["duration_seconds"] = summary.MIN_MEASURED_RELEASE_DURATION_SECONDS - 1
|
||||||
|
self.write_inputs()
|
||||||
|
args = type("Args", (), {
|
||||||
|
"abba_dir": self.abba,
|
||||||
|
"cache_cost_log": None,
|
||||||
|
"require_cache_cost": False,
|
||||||
|
"json_out": None,
|
||||||
|
"markdown_out": None,
|
||||||
|
})
|
||||||
|
with self.assertRaisesRegex(ValueError, "two hours"):
|
||||||
|
summary.build_summary(args)
|
||||||
|
|
||||||
|
def test_passing_abba_report_requires_w10_w11_evidence(self):
|
||||||
|
for fault in ("missing", "pressure", "lock", "attempt", "length", "range"):
|
||||||
|
with self.subTest(fault=fault):
|
||||||
|
self.setUp()
|
||||||
|
target = self.report["comparisons"][0]
|
||||||
|
if fault == "missing":
|
||||||
|
del target["w10_w11"]
|
||||||
|
elif fault == "pressure":
|
||||||
|
del target["w10_w11"]["foreground_pressure_high_sample_ratios"]
|
||||||
|
elif fault == "lock":
|
||||||
|
del target["w10_w11"]["heal_lock_wait_p99_ms"]
|
||||||
|
elif fault == "attempt":
|
||||||
|
del target["w10_w11"]["attempt_cost_per_healed_object"]
|
||||||
|
elif fault == "length":
|
||||||
|
target["w10_w11"]["attempt_cost_per_healed_object"] = [None]
|
||||||
|
else:
|
||||||
|
target["w10_w11"]["foreground_pressure_high_sample_ratios"] = [1.5, 0.0, 0.0, 0.0]
|
||||||
|
self.write_inputs()
|
||||||
|
args = type("Args", (), {
|
||||||
|
"abba_dir": self.abba,
|
||||||
|
"cache_cost_log": None,
|
||||||
|
"require_cache_cost": False,
|
||||||
|
"json_out": None,
|
||||||
|
"markdown_out": None,
|
||||||
|
})
|
||||||
|
with self.assertRaisesRegex(ValueError, "W10/W11|performance evidence|length mismatch|above maximum"):
|
||||||
|
summary.build_summary(args)
|
||||||
|
|
||||||
|
def test_passing_measured_report_requires_release_evidence_manifest(self):
|
||||||
|
del self.manifest["release_evidence"]
|
||||||
|
self.write_inputs()
|
||||||
|
args = type("Args", (), {
|
||||||
|
"abba_dir": self.abba,
|
||||||
|
"cache_cost_log": None,
|
||||||
|
"require_cache_cost": False,
|
||||||
|
"json_out": None,
|
||||||
|
"markdown_out": None,
|
||||||
|
})
|
||||||
|
with self.assertRaisesRegex(ValueError, "release_evidence"):
|
||||||
|
summary.build_summary(args)
|
||||||
|
|
||||||
def test_requires_cache_profile_when_requested(self):
|
def test_requires_cache_profile_when_requested(self):
|
||||||
args = type("Args", (), {
|
args = type("Args", (), {
|
||||||
"abba_dir": self.abba,
|
"abba_dir": self.abba,
|
||||||
|
|||||||
Reference in New Issue
Block a user