diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 85a6e6a56..fb29185e3 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -32,6 +32,7 @@ script-tests: ## Run shell script tests ./scripts/test_hotpath_warp_ab_gate.sh ./scripts/test_hotpath_warp_abba.sh ./scripts/test_scanner_validation_harness.sh + ./scripts/test_scanner_heal_g14_multiset_evidence.sh ./scripts/test_scanner_heal_scheduler_pressure_evidence.sh ./scripts/test_scanner_heal_w13_mrf_evidence.sh ./scripts/test_scanner_heal_w16_recovery_evidence.sh diff --git a/.config/scanner-heal-required-tests.json b/.config/scanner-heal-required-tests.json index 0b8a71e15..f80e5664a 100644 --- a/.config/scanner-heal-required-tests.json +++ b/.config/scanner-heal-required-tests.json @@ -78,6 +78,43 @@ "erasure": {"data_blocks": 8, "parity_blocks": 4}, "erasure_set_drive_count": 12, "scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool." + }, + "background-target-restart-ec8-4-multi-set": { + "gate": "G14", + "task": "W21", + "lane": "e2e-nightly", + "suite": "e2e_test", + "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart", + "oracle": "background-target-restart-ec8-4-multi-set.json", + "evidence": "process-restart", + "unclean_shutdown_marker": false, + "min_objects": 9, + "max_objects": 65, + "topology": {"nodes": 3, "drives_per_node": 8}, + "erasure": {"data_blocks": 8, "parity_blocks": 4}, + "erasure_set_drive_count": 12, + "sets": 2, + "pools": 1, + "scope": "Target process restart during partial background rebuild on a 3x8 EC8+4 layout with two erasure sets in one pool; exact unversioned S3 bodies and replacement-drive shards; not power loss or multi-pool." + }, + "background-target-crash-ec8-4-multi-pool": { + "gate": "G14", + "task": "W21", + "lane": "e2e-nightly", + "suite": "e2e_test", + "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash", + "oracle": "background-target-crash-ec8-4-multi-pool.json", + "evidence": "process-crash-restart", + "unclean_shutdown_marker": true, + "min_objects": 9, + "max_objects": 65, + "topology": {"nodes": 3, "drives_per_node": 12}, + "erasure": {"data_blocks": 8, "parity_blocks": 4}, + "erasure_set_drive_count": 12, + "sets": 3, + "pools": 3, + "outage_target_manifest_required": false, + "scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with outage object verified through S3 but not forced onto the replaced target drive." } }, "release_lanes": { diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index c9df49d4a..06273b795 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -62,18 +62,38 @@ mod tests { unclean_shutdown_marker: bool, topology: EvidenceTopology, storage_class_standard: Option<&'static str>, - erasure_set_drive_count: Option<&'static str>, + erasure_set_drive_count: Option, + outage_target_manifest_required: bool, } #[derive(Clone, Copy)] struct EvidenceTopology { nodes: usize, drives_per_node: usize, + layout: EvidenceTopologyLayout, + } + + #[derive(Clone, Copy)] + enum EvidenceTopologyLayout { + SinglePool, + PerNodePools, } impl EvidenceTopology { const fn new(nodes: usize, drives_per_node: usize) -> Self { - Self { nodes, drives_per_node } + Self { + nodes, + drives_per_node, + layout: EvidenceTopologyLayout::SinglePool, + } + } + + const fn per_node_pools(nodes: usize, drives_per_node: usize) -> Self { + Self { + nodes, + drives_per_node, + layout: EvidenceTopologyLayout::PerNodePools, + } } fn total_drives(self) -> usize { @@ -81,7 +101,23 @@ mod tests { } fn cluster_topology(self) -> ClusterTopology { - ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node) + match self.layout { + EvidenceTopologyLayout::SinglePool => ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node), + EvidenceTopologyLayout::PerNodePools => { + ClusterTopology::per_node_pools(self.drives_per_node, (0..self.nodes).map(|node| vec![node]).collect()) + } + } + } + + fn pool_count(self) -> usize { + match self.layout { + EvidenceTopologyLayout::SinglePool => 1, + EvidenceTopologyLayout::PerNodePools => self.nodes, + } + } + + fn set_count(self, erasure_set_drive_count: Option) -> usize { + self.total_drives() / erasure_set_drive_count.unwrap_or_else(|| self.total_drives()) } } @@ -93,6 +129,7 @@ mod tests { topology: EvidenceTopology::new(4, 1), storage_class_standard: None, erasure_set_drive_count: None, + outage_target_manifest_required: true, }; const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { @@ -103,6 +140,7 @@ mod tests { topology: EvidenceTopology::new(4, 1), storage_class_standard: None, erasure_set_drive_count: None, + outage_target_manifest_required: true, }; const BACKGROUND_TARGET_RESTART_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { @@ -112,7 +150,8 @@ mod tests { unclean_shutdown_marker: false, topology: EvidenceTopology::new(3, 4), storage_class_standard: Some("EC:4"), - erasure_set_drive_count: Some("12"), + erasure_set_drive_count: Some(12), + outage_target_manifest_required: true, }; const BACKGROUND_TARGET_CRASH_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { @@ -122,7 +161,30 @@ mod tests { unclean_shutdown_marker: true, topology: EvidenceTopology::new(3, 4), storage_class_standard: Some("EC:4"), - erasure_set_drive_count: Some("12"), + erasure_set_drive_count: Some(12), + outage_target_manifest_required: true, + }; + + const BACKGROUND_TARGET_RESTART_EC84_MULTI_SET_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { + id: "background-target-restart-ec8-4-multi-set", + oracle: "background-target-restart-ec8-4-multi-set.json", + evidence: "process-restart", + unclean_shutdown_marker: false, + topology: EvidenceTopology::new(3, 8), + storage_class_standard: Some("EC:4"), + erasure_set_drive_count: Some(12), + outage_target_manifest_required: true, + }; + + const BACKGROUND_TARGET_CRASH_EC84_MULTI_POOL_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { + id: "background-target-crash-ec8-4-multi-pool", + oracle: "background-target-crash-ec8-4-multi-pool.json", + evidence: "process-crash-restart", + unclean_shutdown_marker: true, + topology: EvidenceTopology::per_node_pools(3, 12), + storage_class_standard: Some("EC:4"), + erasure_set_drive_count: Some(12), + outage_target_manifest_required: false, }; struct RestartEvidenceContext { @@ -1041,6 +1103,26 @@ mod tests { .await? } + #[tokio::test(flavor = "multi_thread")] + async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart() + -> Result<(), Box> { + timeout( + Duration::from_secs(600), + run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84MultiSet), + ) + .await? + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash() + -> Result<(), Box> { + timeout( + Duration::from_secs(600), + run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrashEc84MultiPool), + ) + .await? + } + #[tokio::test(flavor = "multi_thread")] async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box> { @@ -1080,6 +1162,8 @@ mod tests { BackgroundTargetCrash, BackgroundTargetRestartEc84, BackgroundTargetCrashEc84, + BackgroundTargetRestartEc84MultiSet, + BackgroundTargetCrashEc84MultiPool, BackgroundCoordinatorRestart, TargetEndpointBlackhole, } @@ -1091,6 +1175,8 @@ mod tests { InterruptionScenario::BackgroundTargetCrash => Some(BACKGROUND_TARGET_CRASH_EVIDENCE), InterruptionScenario::BackgroundTargetRestartEc84 => Some(BACKGROUND_TARGET_RESTART_EC84_EVIDENCE), InterruptionScenario::BackgroundTargetCrashEc84 => Some(BACKGROUND_TARGET_CRASH_EC84_EVIDENCE), + InterruptionScenario::BackgroundTargetRestartEc84MultiSet => Some(BACKGROUND_TARGET_RESTART_EC84_MULTI_SET_EVIDENCE), + InterruptionScenario::BackgroundTargetCrashEc84MultiPool => Some(BACKGROUND_TARGET_CRASH_EC84_MULTI_POOL_EVIDENCE), _ => None, }; let evidence_run = match evidence_case { @@ -1104,12 +1190,18 @@ mod tests { InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"), InterruptionScenario::BackgroundTargetRestartEc84 => (true, 1, "background_target_restart_ec8_4"), InterruptionScenario::BackgroundTargetCrashEc84 => (true, 1, "background_target_crash_ec8_4"), + InterruptionScenario::BackgroundTargetRestartEc84MultiSet => (true, 1, "background_target_restart_ec8_4_multi_set"), + InterruptionScenario::BackgroundTargetCrashEc84MultiPool => (true, 1, "background_target_crash_ec8_4_multi_pool"), InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"), InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"), }; let topology = evidence_case .map(|case| case.topology) .unwrap_or_else(|| EvidenceTopology::new(4, 1)); + let erasure_set_drive_count = evidence_case + .and_then(|case| case.erasure_set_drive_count) + .unwrap_or_else(|| topology.total_drives()); + let outage_target_manifest_required = evidence_case.map(|case| case.outage_target_manifest_required).unwrap_or(true); init_logging(); info!( event = "heal_interruption_started", @@ -1128,7 +1220,7 @@ mod tests { cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", storage_class); } if let Some(erasure_set_drive_count) = evidence_case.and_then(|case| case.erasure_set_drive_count) { - cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count); + cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count.to_string()); } // Heal control uses the first lexicographically sorted grid host. // Keep that coordinator distinct from the remote target at index 1. @@ -1268,6 +1360,9 @@ mod tests { } for (drive_index, drive) in node.data_dirs.iter().enumerate() { let census = census_object_version_on_disk(Path::new(drive), bucket, outage_key, None)?; + if !census.has_xl_meta { + continue; + } assert!( census.is_complete(), "online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}" @@ -1276,7 +1371,7 @@ mod tests { format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}") })?; assert!( - (1..=topology.total_drives()).contains(&erasure_index), + (1..=erasure_set_drive_count).contains(&erasure_index), "online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}" ); assert!( @@ -1285,19 +1380,26 @@ mod tests { ); } } - assert_eq!( - outage_peer_erasure_indices.len(), - topology.total_drives().saturating_sub(cluster.nodes[1].data_dirs.len()), - "every online drive must contribute one unique outage-object erasure index" + assert!( + !outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count, + "outage-object must occupy one non-empty erasure set" ); - let missing_outage_erasure_indices = (1..=topology.total_drives()) + if outage_target_manifest_required { + let min_online_data_shards = erasure_set_drive_count.saturating_sub(4); + assert!( + outage_peer_erasure_indices.len() >= min_online_data_shards, + "online drives in the selected erasure set must retain at least the EC data quorum" + ); + } + let missing_outage_erasure_indices = (1..=erasure_set_drive_count) .filter(|index| !outage_peer_erasure_indices.contains(index)) .collect::>(); - assert_eq!( - missing_outage_erasure_indices.len(), - cluster.nodes[1].data_dirs.len(), - "the stopped node must account for every missing outage-object erasure index" - ); + if outage_target_manifest_required { + assert!( + !missing_outage_erasure_indices.is_empty(), + "the stopped target must account for at least one missing outage-object erasure index" + ); + } let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#; if !background_enabled { @@ -1596,7 +1698,9 @@ mod tests { unclean_shutdown_marker_observed = Some(marker_exists); let expected_marker = matches!( scenario, - InterruptionScenario::BackgroundTargetCrash | InterruptionScenario::BackgroundTargetCrashEc84 + InterruptionScenario::BackgroundTargetCrash + | InterruptionScenario::BackgroundTargetCrashEc84 + | InterruptionScenario::BackgroundTargetCrashEc84MultiPool ); assert!( marker_exists == expected_marker, @@ -1636,9 +1740,10 @@ mod tests { .unwrap_or(180); let heal_deadline = Instant::now() + Duration::from_secs(heal_timeout_secs); loop { - if metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len() - && object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key) - { + let baseline_recovered = metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len(); + let outage_recovered = + !outage_target_manifest_required || object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key); + if baseline_recovered && outage_recovered { let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?; let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?; let pool_metadata_matches = match &expected_pool_metadata { @@ -1648,7 +1753,10 @@ mod tests { } None => true, }; - if matching == expected_manifests.len() && outage_census.is_complete() && pool_metadata_matches { + if matching == expected_manifests.len() + && (!outage_target_manifest_required || outage_census.is_complete()) + && pool_metadata_matches + { break; } } @@ -1693,17 +1801,19 @@ mod tests { ); } let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?; - assert!( - outage_census.is_complete(), - "outage object must have a complete target shard: {outage_census:?}" - ); - assert_eq!( - outage_census - .erasure_index - .filter(|index| missing_outage_erasure_indices.contains(index)), - outage_census.erasure_index, - "the outage object must be rebuilt into one of the stopped node's missing erasure slots" - ); + if outage_target_manifest_required { + assert!( + outage_census.is_complete(), + "outage object must have a complete target shard: {outage_census:?}" + ); + assert_eq!( + outage_census + .erasure_index + .filter(|index| missing_outage_erasure_indices.contains(index)), + outage_census.erasure_index, + "the outage object must be rebuilt into one of the stopped node's missing erasure slots" + ); + } if let Some(cycle_end) = scanner_cycle_floor { wait_for_scanner_cycle_after(&cluster, cycle_end).await?; @@ -1801,6 +1911,14 @@ mod tests { "binary_sha256": evidence_context.run.binary.sha256, "test_binary_sha256": evidence_context.run.test_binary.sha256, "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()}, + "erasure_set_drive_count": erasure_set_drive_count, + "sets": topology.set_count(evidence_context.case.erasure_set_drive_count), + "pools": topology.pool_count(), + "outage_target_manifest_required": outage_target_manifest_required, + "distributed_ec_invalidation": true, + "peer_count": cluster.nodes.len(), + "same_window_remote_proof": true, + "all_peers_bound_to_generation_window": true, "pid_before": target_pid, "pid_after": restarted_pid, "unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false), "objects": evidence_objects, "node_listings": node_listings, diff --git a/scripts/README.md b/scripts/README.md index 7d9b03c7a..fa7eba564 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -56,11 +56,13 @@ their issue closes. | `probe.sh` | dev-tool | Probe-style e2e run | `make probe-e2e` | | `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` | | `run_scanner_heal_evidence_case.sh` | dev-tool | Runs one Scanner/Heal release-evidence registry case and checks the produced receipt/oracle | `.config/scanner-heal-required-tests.json`; `check_test_wiring.py --check-scanner-heal` | +| `run_scanner_heal_g14_multiset_evidence.py` | dev-tool | Assembles measured Scanner/Heal G14 same-window EC8+4 multi-set/multi-pool release descriptors from e2e case directories or an operator-collected proof | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_g14_multiset_evidence.sh` | | `run_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Runs the G09 mixed-version and rollback upgrade E2E lanes against a pinned previous release and verifies the raw evidence artifacts | `docs/testing/ci-gates.md`; `.github/workflows/e2e-upgrade.yml`; `test_scanner_heal_g09_upgrade_evidence.sh` | | `run_scanner_heal_scheduler_pressure_evidence.py` | dev-tool | Assembles measured Scanner/Heal G10/P1/P3 scheduler-pressure release descriptors from a completed measured ABBA run, recovery-window proof, and profile artifacts | `docs/operations/scanner-benchmark-runbook.md`; `test_scanner_heal_scheduler_pressure_evidence.sh` | | `run_scanner_heal_w13_mrf_evidence.sh` | dev-tool | Runs the W13 durable MRF replay lanes and writes G07/G08/P4 bundle-ready evidence descriptors | `docs/testing/ci-gates.md`; `test_scanner_heal_w13_mrf_evidence.sh` | | `run_scanner_heal_w16_recovery_evidence.sh` | dev-tool | Runs the W16 recovery-intent and quota authority lanes and writes G04/G12 bundle-ready evidence descriptors | `docs/testing/ci-gates.md`; `test_scanner_heal_w16_recovery_evidence.sh` | | `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — | +| `test_scanner_heal_g14_multiset_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal G14 multi-set/multi-pool evidence assembler | — | | `test_scanner_heal_scheduler_pressure_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal scheduler-pressure evidence assembler | — | | `test_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal G09 upgrade evidence runner | — | | `test_scanner_heal_w16_recovery_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal W16 recovery evidence runner | — | diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index d59ee269b..4ce267232 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -1184,13 +1184,23 @@ def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]: data_blocks = evidence_integer(erasure.get("data_blocks"), f"{case_id} data_blocks", 1, 16) parity_blocks = evidence_integer(erasure.get("parity_blocks"), f"{case_id} parity_blocks", 1, 16) require(data_blocks >= parity_blocks, f"invalid erasure geometry for {case_id}") - require(data_blocks + parity_blocks == requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"], + expected_set_drives = requirement.get( + "erasure_set_drive_count", + requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"], + ) + require(data_blocks + parity_blocks == expected_set_drives, f"erasure geometry differs from topology for {case_id}") if "erasure_set_drive_count" in requirement: erasure_set_drive_count = evidence_integer(requirement.get("erasure_set_drive_count"), f"{case_id} erasure_set_drive_count", 1, 64) - require(erasure_set_drive_count == requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"], - f"erasure set drive count differs from topology for {case_id}") + total_drives = requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"] + require(total_drives % erasure_set_drive_count == 0, + f"erasure set drive count does not divide topology for {case_id}") + if "sets" in requirement: + require(evidence_integer(requirement.get("sets"), f"{case_id} sets", 1, 1024) + == total_drives // erasure_set_drive_count, f"set count differs from topology for {case_id}") + if "pools" in requirement: + evidence_integer(requirement.get("pools"), f"{case_id} pools", 1, 1024) names.add(oracle) return tuple(sorted(names)) @@ -1419,14 +1429,33 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li require(oracle.get("topology") == requirement["topology"], "oracle topology mismatch") for key in ("nodes", "drives_per_node"): evidence_integer(oracle["topology"][key], f"observed {key}", 1, 16) + expected_set_drives = requirement.get( + "erasure_set_drive_count", + oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"], + ) + require(oracle.get("erasure_set_drive_count") == expected_set_drives, + "oracle erasure set drive count mismatch") + if "sets" in requirement: + require(oracle.get("sets") == requirement["sets"], "oracle set count mismatch") + if "pools" in requirement: + require(oracle.get("pools") == requirement["pools"], "oracle pool count mismatch") + expected_outage_target_required = requirement.get("outage_target_manifest_required", True) + require(oracle.get("outage_target_manifest_required", True) is expected_outage_target_required, + "oracle outage target-manifest contract mismatch") + if requirement.get("sets", 1) > 1 or requirement.get("pools", 1) > 1: + require(oracle.get("distributed_ec_invalidation") is True, + "oracle missing distributed EC invalidation proof") + evidence_integer(oracle.get("peer_count"), "oracle peer_count", 3, 64) + require(oracle.get("same_window_remote_proof") is True, "oracle missing same-window remote proof") + require(oracle.get("all_peers_bound_to_generation_window") is True, + "oracle missing peer generation-window binding") expected_erasure = requirement.get("erasure") if expected_erasure is not None: require(isinstance(expected_erasure, dict), "invalid erasure expectation") expected_data_blocks = evidence_integer(expected_erasure.get("data_blocks"), "expected EC data blocks", 1, 16) expected_parity_blocks = evidence_integer(expected_erasure.get("parity_blocks"), "expected EC parity blocks", 1, 16) require( - expected_data_blocks + expected_parity_blocks - == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"], + expected_data_blocks + expected_parity_blocks == expected_set_drives, "expected EC geometry differs from topology", ) else: @@ -1451,14 +1480,18 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li if obj["expected_physical"] is not None: require(physical == obj["expected_physical"], "target shard differs from pre-fault manifest") for geometry in [physical] + ([obj["expected_physical"]] if obj["expected_physical"] is not None else []): + if not geometry["has_xl_meta"] and not expected_outage_target_required and obj["expected_physical"] is None: + continue data = evidence_integer(geometry["data_blocks"], "EC data blocks", 1, 16) parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16) - require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"], - "EC geometry differs from this case's single set") + require(data + parity == expected_set_drives, + "EC geometry differs from this case's erasure set") if expected_data_blocks is not None: require(data == expected_data_blocks and parity == expected_parity_blocks, "EC data/parity geometry differs from the required case") evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity) + if not expected_outage_target_required and obj["expected_physical"] is None and not physical["has_xl_meta"]: + continue require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata") parts = physical["expected_part_numbers"] require(isinstance(parts, list) and 0 < len(parts) <= 10000, "no physical part coverage") @@ -2557,10 +2590,11 @@ class SelfTests(unittest.TestCase): def oracle_objects(requirement: dict[str, object]) -> list[dict[str, object]]: topology = requirement["topology"] total_blocks = topology["nodes"] * topology["drives_per_node"] + erasure_set_drive_count = requirement.get("erasure_set_drive_count", total_blocks) erasure = requirement.get("erasure") if erasure is None: - parity_blocks = 4 if total_blocks == 12 else total_blocks // 2 - data_blocks = total_blocks - parity_blocks + parity_blocks = 4 if erasure_set_drive_count == 12 else erasure_set_drive_count // 2 + data_blocks = erasure_set_drive_count - parity_blocks else: data_blocks = erasure["data_blocks"] parity_blocks = erasure["parity_blocks"] @@ -2585,7 +2619,19 @@ class SelfTests(unittest.TestCase): "test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40, "features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""}, "binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"], - "topology": requirement["topology"], "pid_before": 10, "pid_after": 11, + "topology": requirement["topology"], + "erasure_set_drive_count": requirement.get( + "erasure_set_drive_count", + requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"], + ), + "sets": requirement.get("sets", 1), + "pools": requirement.get("pools", 1), + "outage_target_manifest_required": requirement.get("outage_target_manifest_required", True), + "distributed_ec_invalidation": True, + "peer_count": requirement["topology"]["nodes"], + "same_window_remote_proof": True, + "all_peers_bound_to_generation_window": True, + "pid_before": 10, "pid_after": 11, "unclean_shutdown_marker": requirement["unclean_shutdown_marker"], "objects": objects, "node_listings": [[item["key"] for item in objects]] * requirement["topology"]["nodes"], }) diff --git a/scripts/run_scanner_heal_evidence_case.sh b/scripts/run_scanner_heal_evidence_case.sh index 201a51ce9..f5356e351 100755 --- a/scripts/run_scanner_heal_evidence_case.sh +++ b/scripts/run_scanner_heal_evidence_case.sh @@ -80,9 +80,12 @@ runtime_profile_for() { background-target-crash|background-target-restart) echo "background-4x1" ;; - background-target-crash-ec8-4|background-target-restart-ec8-4) + background-target-crash-ec8-4|background-target-restart-ec8-4|background-target-restart-ec8-4-multi-set) echo "background-ec8-4" ;; + background-target-crash-ec8-4-multi-pool) + echo "background-ec8-4-multi-pool" + ;; ec84-target-drive-restart) echo "distributed-ec8-4" ;; @@ -105,6 +108,11 @@ apply_runtime_profile() { export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-8388608}" export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-180}" ;; + background-ec8-4-multi-pool) + export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-16}" + export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-4194304}" + export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-240}" + ;; esac } diff --git a/scripts/run_scanner_heal_g14_multiset_evidence.py b/scripts/run_scanner_heal_g14_multiset_evidence.py new file mode 100755 index 000000000..a50483a27 --- /dev/null +++ b/scripts/run_scanner_heal_g14_multiset_evidence.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +"""Assemble measured Scanner/Heal G14 multi-set/multi-pool release evidence. + +The input proof must come from one same-window distributed measurement that +already observed EC8+4, at least two sets, at least two pools, and distributed +segment invalidation. This script packages that proof into the release-bundle +field shape enforced by check_test_wiring.py. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +from scanner_abba import digest, read_json, require, write_json + +ROOT = Path(__file__).resolve().parents[1] +G14_FIELDS = ( + "same_window_field_evidence", + "ec8_4_evidence", + "multi_set_evidence", + "multi_pool_evidence", + "distributed_segment_invalidation_evidence", +) + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def iso_from_epoch(value: Any, name: str) -> str: + require(type(value) in (int, float) and value > 0, f"invalid {name}") + return datetime.fromtimestamp(float(value), timezone.utc).isoformat().replace("+00:00", "Z") + + +def positive_int(value: Any, name: str, minimum: int = 1) -> int: + require(type(value) is int and value >= minimum, f"invalid {name}") + return value + + +def parse_case_dir_arg(value: str) -> tuple[str | None, Path]: + if "=" in value: + case_id, raw_path = value.split("=", 1) + require(bool(case_id.strip()), "case directory case id is empty") + return case_id.strip(), Path(raw_path).expanduser().resolve() + return None, Path(value).expanduser().resolve() + + +def load_proof(path: Path, source_revision: str) -> dict[str, Any]: + proof = read_json(path) + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + require(proof.get(marker) is not True, f"G14 proof is {marker}") + require(proof.get("schema") == 1, "unsupported G14 proof schema") + require(proof.get("evidence_type") == "measured", "G14 proof must be measured") + require(proof.get("source_revision") == source_revision, "G14 proof source revision mismatch") + require(isinstance(proof.get("run_id"), str) and len(proof["run_id"]) >= 8, "missing G14 proof run_id") + require(isinstance(proof.get("measurement_window_id"), str) and len(proof["measurement_window_id"]) >= 8, + "missing G14 measurement window") + require(proof["run_id"] != proof["measurement_window_id"], "G14 run/window identities must differ") + require(isinstance(proof.get("command"), list) and proof["command"], "missing G14 command provenance") + topology = proof.get("topology") + require(isinstance(topology, dict), "G14 proof missing topology") + require(topology.get("erasure") == "EC8+4", "G14 proof must record EC8+4") + positive_int(topology.get("nodes"), "topology.nodes", 3) + positive_int(topology.get("drives_per_node"), "topology.drives_per_node", 4) + positive_int(proof.get("sets"), "sets", 2) + positive_int(proof.get("pools"), "pools", 2) + require(proof.get("invalidation_domain") == "distributed-ec", "G14 proof must be distributed EC invalidation") + require(proof.get("distributed_ec_invalidation") is True, "G14 proof missing peer invalidation") + positive_int(proof.get("peer_count"), "peer_count", 3) + require(proof.get("same_window_remote_proof") is True, "G14 proof missing same-window remote proof") + require(proof.get("all_peers_bound_to_generation_window") is True, + "G14 proof missing peer generation-window binding") + samples = proof.get("case_evidence") + require(isinstance(samples, list) and samples, "G14 proof must reference measured case evidence") + for index, sample in enumerate(samples): + require(isinstance(sample, dict), f"G14 case evidence {index} must be an object") + require(isinstance(sample.get("case"), str) and sample["case"].strip(), f"G14 case evidence {index} missing case") + require(isinstance(sample.get("sha256"), str) and re.fullmatch(r"[0-9a-f]{64}", sample["sha256"]), + f"G14 case evidence {index} missing sha256") + if "source_revision" in sample: + require(sample["source_revision"] == source_revision, f"G14 case evidence {index} source revision mismatch") + if "measurement_window_id" in sample: + require(sample["measurement_window_id"] == proof["measurement_window_id"], + f"G14 case evidence {index} measurement window mismatch") + return proof + + +def load_case_directory(raw_value: str, source_revision: str) -> dict[str, Any]: + expected_case, directory = parse_case_dir_arg(raw_value) + require(directory.is_dir(), f"G14 case directory is missing: {directory}") + run = read_json(directory / "run.json") + execution = read_json(directory / "execution.json") + require(run.get("schema") == 1, "G14 case run schema mismatch") + require(isinstance(run.get("run_id"), str) and re.fullmatch(r"[0-9a-f]{32}", run["run_id"]), + "invalid G14 case run id") + require(run.get("source_revision") == source_revision, "G14 case source revision mismatch") + require(execution.get("run_id") == run["run_id"], "G14 case execution belongs to another run") + require(execution.get("exit_code") == 0, "G14 case execution did not pass") + artifacts = execution.get("artifacts") + require(isinstance(artifacts, dict), "G14 case execution missing artifacts") + started_at = iso_from_epoch(run.get("started_at"), "case started_at") + finished_at = iso_from_epoch(execution.get("finished_at"), "case finished_at") + + oracle_items = [] + for name, expected_sha in sorted(artifacts.items()): + if not name.endswith(".json") or name in {"run.json", "execution.json", "release-status.json"}: + continue + path = directory / name + if not path.is_file(): + continue + require(isinstance(expected_sha, str) and re.fullmatch(r"[0-9a-f]{64}", expected_sha), + f"G14 case artifact {name} has an invalid hash") + require(digest(path) == expected_sha, f"G14 case artifact hash mismatch: {name}") + item = read_json(path) + if item.get("schema") == 1 and item.get("case") and item.get("run_id") == run["run_id"]: + oracle_items.append((name, path, expected_sha, item)) + require(len(oracle_items) == 1, "G14 case directory must contain exactly one case oracle") + name, path, expected_sha, oracle = oracle_items[0] + if expected_case is not None: + require(oracle.get("case") == expected_case, "G14 case directory case id mismatch") + require(oracle.get("source_revision") == source_revision, "G14 oracle source revision mismatch") + require(oracle.get("evidence") in {"process-restart", "process-crash-restart"}, "G14 oracle has wrong evidence type") + topology = oracle.get("topology") + require(isinstance(topology, dict), "G14 oracle missing topology") + positive_int(topology.get("nodes"), "oracle topology.nodes", 3) + positive_int(topology.get("drives_per_node"), "oracle topology.drives_per_node", 4) + require(oracle.get("erasure_set_drive_count") == 12, "G14 oracle must use EC8+4 set width") + positive_int(oracle.get("sets"), "oracle sets", 1) + positive_int(oracle.get("pools"), "oracle pools", 1) + if oracle.get("sets", 1) > 1 or oracle.get("pools", 1) > 1: + require(oracle.get("distributed_ec_invalidation") is True, "G14 oracle missing distributed invalidation") + positive_int(oracle.get("peer_count"), "oracle peer_count", 3) + require(oracle.get("same_window_remote_proof") is True, "G14 oracle missing same-window remote proof") + require(oracle.get("all_peers_bound_to_generation_window") is True, + "G14 oracle missing peer generation-window binding") + return { + "case": oracle["case"], + "artifact_name": name, + "artifact_path": path, + "sha256": expected_sha, + "run_id": run["run_id"], + "started_at": started_at, + "finished_at": finished_at, + "oracle": oracle, + } + + +def copy_case_artifacts(out_dir: Path, records: list[dict[str, Any]], window_id: str, + source_revision: str) -> list[dict[str, Any]]: + case_dir = out_dir / "artifacts" / "cases" + case_dir.mkdir(parents=True, exist_ok=True) + copied = [] + for record in records: + target = case_dir / record["artifact_name"] + if target.exists(): + target = case_dir / f"{record['case']}-{record['artifact_name']}" + shutil.copyfile(record["artifact_path"], target) + copied.append({ + "case": record["case"], + "artifact": target.relative_to(out_dir).as_posix(), + "sha256": digest(target), + "source_revision": source_revision, + "measurement_window_id": window_id, + }) + return copied + + +def proof_from_case_directories(raw_values: list[str], out_dir: Path, source_revision: str) -> dict[str, Any]: + require(raw_values, "missing G14 case directories") + records = [load_case_directory(value, source_revision) for value in raw_values] + covering = [ + record for record in records + if record["oracle"].get("erasure_set_drive_count") == 12 + and record["oracle"].get("sets", 0) >= 2 + and record["oracle"].get("pools", 0) >= 2 + and record["oracle"].get("distributed_ec_invalidation") is True + ] + require(covering, "G14 case evidence must include one same-window EC8+4 multi-set/multi-pool proof") + selected = covering[0] + oracle = selected["oracle"] + window_id = f"g14-case-window-{selected['run_id']}" + return { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": selected["run_id"], + "measurement_window_id": window_id, + "started_at": selected["started_at"], + "finished_at": selected["finished_at"], + "command": ["scripts/run_scanner_heal_g14_multiset_evidence.py", "--case-dir", ""], + "summary": "Measured G14 descriptor assembled from Scanner/Heal e2e case evidence.", + "topology": { + "erasure": "EC8+4", + "nodes": oracle["topology"]["nodes"], + "drives_per_node": oracle["topology"]["drives_per_node"], + }, + "sets": oracle["sets"], + "pools": oracle["pools"], + "invalidation_domain": "distributed-ec", + "distributed_ec_invalidation": oracle["distributed_ec_invalidation"], + "peer_count": oracle["peer_count"], + "same_window_remote_proof": oracle["same_window_remote_proof"], + "all_peers_bound_to_generation_window": oracle["all_peers_bound_to_generation_window"], + "case_evidence": copy_case_artifacts(out_dir, records, window_id, source_revision), + } + + +def write_field(out_dir: Path, field: str, proof: dict[str, Any], source_revision: str) -> dict[str, Any]: + started_at = proof.get("started_at") or utc_now() + finished_at = proof.get("finished_at") or started_at + evidence: dict[str, Any] = { + "artifact": "", + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": proof["run_id"], + "measurement_window_id": proof["measurement_window_id"], + "started_at": started_at, + "finished_at": finished_at, + "command": proof["command"], + "artifact_format": "json", + "summary": proof.get("summary") or "Measured G14 same-window EC8+4 multi-set/multi-pool proof.", + } + if field == "same_window_field_evidence": + evidence["same_window_fields"] = [item for item in G14_FIELDS if item != field] + elif field == "ec8_4_evidence": + evidence["topology"] = proof["topology"] + elif field == "multi_set_evidence": + evidence["sets"] = proof["sets"] + elif field == "multi_pool_evidence": + evidence["pools"] = proof["pools"] + elif field == "distributed_segment_invalidation_evidence": + for key in ( + "invalidation_domain", + "distributed_ec_invalidation", + "peer_count", + "same_window_remote_proof", + "all_peers_bound_to_generation_window", + ): + evidence[key] = proof[key] + artifact = out_dir / "artifacts" / f"G14-{field}.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": proof["run_id"], + "measurement_window_id": proof["measurement_window_id"], + "gate": "G14", + "field": field, + "case_evidence": proof["case_evidence"], + } + for key, value in evidence.items(): + if key not in {"artifact", "sha256", "artifact_format", "summary", "started_at", "finished_at", "command"}: + payload[key] = value + write_json(artifact, payload) + evidence["artifact"] = artifact.relative_to(out_dir).as_posix() + evidence["sha256"] = digest(artifact) + return evidence + + +def build_descriptor(args: argparse.Namespace) -> Path: + out_dir = args.out_dir.resolve() + require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + out_dir.mkdir(parents=True) + if args.proof_json is not None: + proof = load_proof(args.proof_json.resolve(), source_revision) + else: + proof = proof_from_case_directories(args.case_dir, out_dir, source_revision) + fields = {field: write_field(out_dir, field, proof, source_revision) for field in G14_FIELDS} + descriptor = out_dir / "release-bundle-g14.json" + write_json(descriptor, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": { + "G14": { + "status": "pass", + "lane": "ec8-4-multiset", + "evidence_type": "measured", + "evidence_fields": fields, + }, + }, + }) + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + "G14", + ], cwd=ROOT) + return descriptor + + +def write_self_test_proof(root: Path, source_revision: str) -> Path: + proof = root / "proof.json" + write_json(proof, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": "g14-self-test-run", + "measurement_window_id": "g14-self-test-window", + "started_at": "2026-09-09T00:00:00Z", + "finished_at": "2026-09-09T00:30:00Z", + "command": ["scripts/run_scanner_heal_g14_multiset_evidence.py", "--proof-json", "proof.json"], + "summary": "Measured parser self-test proof.", + "topology": {"erasure": "EC8+4", "nodes": 3, "drives_per_node": 4}, + "sets": 2, + "pools": 2, + "invalidation_domain": "distributed-ec", + "distributed_ec_invalidation": True, + "peer_count": 3, + "same_window_remote_proof": True, + "all_peers_bound_to_generation_window": True, + "case_evidence": [ + {"case": "ec84-target-drive-restart", "sha256": "a" * 64}, + {"case": "multi-set-distributed-invalidation", "sha256": "b" * 64}, + {"case": "multi-pool-distributed-invalidation", "sha256": "c" * 64}, + ], + }) + return proof + + +def write_self_test_case_dir(root: Path, source_revision: str, case: str, sets: int, pools: int) -> Path: + directory = root / case + directory.mkdir() + started = datetime(2026, 9, 9, 0, 0, 0, tzinfo=timezone.utc).timestamp() + finished = datetime(2026, 9, 9, 0, 30, 0, tzinfo=timezone.utc).timestamp() + run_id = ("a" if pools > 1 else "b") * 32 + write_json(directory / "run.json", { + "schema": 1, + "run_id": run_id, + "source_revision": source_revision, + "started_at": started, + }) + oracle = { + "schema": 1, + "case": case, + "evidence": "process-crash-restart" if pools > 1 else "process-restart", + "run_id": run_id, + "source_revision": source_revision, + "topology": {"nodes": 3, "drives_per_node": 12 if pools > 1 else 8}, + "erasure_set_drive_count": 12, + "sets": sets, + "pools": pools, + "distributed_ec_invalidation": True, + "peer_count": 3, + "same_window_remote_proof": True, + "all_peers_bound_to_generation_window": True, + } + oracle_name = f"{case}.json" + write_json(directory / oracle_name, oracle) + write_json(directory / "execution.json", { + "run_id": run_id, + "exit_code": 0, + "finished_at": finished, + "artifacts": {oracle_name: digest(directory / oracle_name)}, + }) + return directory + + +def run_self_test() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = write_self_test_proof(root, source_revision) + descriptor = build_descriptor(parse_args([ + "--proof-json", str(proof), + "--out-dir", str(root / "out"), + ])) + require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = write_self_test_proof(root, source_revision) + data = read_json(proof) + data["pools"] = 1 + write_json(proof, data) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + require("pools" in str(err), "wrong self-test failure for single-pool proof") + else: + raise ValueError("self-test accepted single-pool G14 proof") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + multi_set = write_self_test_case_dir(root, source_revision, "background-target-restart-ec8-4-multi-set", 2, 1) + multi_pool = write_self_test_case_dir(root, source_revision, "background-target-crash-ec8-4-multi-pool", 3, 3) + descriptor = build_descriptor(parse_args([ + "--case-dir", f"background-target-restart-ec8-4-multi-set={multi_set}", + "--case-dir", f"background-target-crash-ec8-4-multi-pool={multi_pool}", + "--out-dir", str(root / "out"), + ])) + require(descriptor.is_file(), "self-test case-dir descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + multi_set = write_self_test_case_dir(root, source_revision, "background-target-restart-ec8-4-multi-set", 2, 1) + try: + build_descriptor(parse_args([ + "--case-dir", f"background-target-restart-ec8-4-multi-set={multi_set}", + "--out-dir", str(root / "out"), + ])) + except ValueError as err: + require("multi-set/multi-pool" in str(err), "wrong self-test failure for missing multi-pool case") + else: + raise ValueError("self-test accepted case evidence without multi-pool proof") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--proof-json", type=Path) + parser.add_argument("--case-dir", action="append", default=[], + help="Measured e2e evidence run directory, optionally CASE=DIR; repeatable") + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if (args.proof_json is None) == (not args.case_dir): + parser.error("provide exactly one of --proof-json or --case-dir unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"G14 release descriptor verified: {descriptor}") + return 0 + except (ValueError, KeyError, OSError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_scanner_heal_g14_multiset_evidence.sh b/scripts/test_scanner_heal_g14_multiset_evidence.sh new file mode 100755 index 000000000..08f8f53ae --- /dev/null +++ b/scripts/test_scanner_heal_g14_multiset_evidence.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_g14_multiset_evidence.py" + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test