From 3149c87cf2fb44e417749b4d19084cbf10d72f32 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 8 Sep 2026 12:02:58 +0800 Subject: [PATCH] test(heal): cover EC8+4 restart shard rebuild (#7461) Retry heal-control RPCs once after transport auth rejects a stale replay-scope epoch, and add a distributed EC8+4 restart heal evidence case that rebuilds a replaced drive with exact shard/body assertions. Co-authored-by: zhi22915 --- .config/scanner-heal-required-tests.json | 14 ++ crates/e2e_test/src/distributed/harness.rs | 6 +- crates/e2e_test/src/distributed/heal_test.rs | 183 ++++++++++++++++++ crates/e2e_test/src/distributed/mod.rs | 1 + crates/ecstore/src/cluster/rpc/client.rs | 38 ++++ .../src/cluster/rpc/peer_rest_client.rs | 134 +++++++++---- scripts/check_test_wiring.py | 36 ++-- 7 files changed, 359 insertions(+), 53 deletions(-) create mode 100644 crates/e2e_test/src/distributed/heal_test.rs diff --git a/.config/scanner-heal-required-tests.json b/.config/scanner-heal-required-tests.json index 9d6966cc3..1ea1f52e2 100644 --- a/.config/scanner-heal-required-tests.json +++ b/.config/scanner-heal-required-tests.json @@ -28,6 +28,20 @@ "max_objects": 65, "topology": {"nodes": 4, "drives_per_node": 1}, "scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4." + }, + "ec84-target-drive-restart": { + "gate": "G14", + "task": "W20/W21", + "lane": "e2e-distributed", + "suite": "e2e_test", + "name": "distributed::heal_test::three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart", + "oracle": "ec84-target-drive-restart.json", + "evidence": "process-restart", + "unclean_shutdown_marker": false, + "min_objects": 5, + "max_objects": 5, + "topology": {"nodes": 3, "drives_per_node": 4}, + "scope": "3-node x 4-drive single-set EC8+4, graceful target restart, preformatted replacement drive, exact unversioned S3 bodies and physical target shards; not mixed-version, multi-pool or long-window ABBA." } }, "release_lanes": { diff --git a/crates/e2e_test/src/distributed/harness.rs b/crates/e2e_test/src/distributed/harness.rs index f610beaa4..aed1021dc 100644 --- a/crates/e2e_test/src/distributed/harness.rs +++ b/crates/e2e_test/src/distributed/harness.rs @@ -59,6 +59,9 @@ const POOL_META_V3_ENV: [(&str, &str); 2] = [ #[derive(Clone, Copy, Debug)] pub(crate) enum DistLayout { + /// 3 nodes × 4 drives, one erasure pool. With `EC:4` this is the + /// release-evidence EC8+4 geometry. + ThreeByFourEc84, /// 4 nodes × 4 drives, one erasure pool spanning every endpoint. FourByFour, /// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout). @@ -94,6 +97,7 @@ impl DistCluster { pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult { let topology = match layout { + DistLayout::ThreeByFourEc84 => ClusterTopology::single_pool_multidrive(3, DRIVES_PER_NODE), DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE), DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT), DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]), @@ -101,7 +105,7 @@ impl DistCluster { let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?; let pool_storage_roots = match layout { DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?), - DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None, + DistLayout::ThreeByFourEc84 | DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None, }; let mut owned_pool_dirs = Vec::new(); if let Some(roots) = pool_storage_roots.as_deref() { diff --git a/crates/e2e_test/src/distributed/heal_test.rs b/crates/e2e_test/src/distributed/heal_test.rs new file mode 100644 index 000000000..01d3f099a --- /dev/null +++ b/crates/e2e_test/src/distributed/heal_test.rs @@ -0,0 +1,183 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, assert_inventory, payload_for, put_object, unique_bucket, wait_until}; +use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post}; +use crate::common::init_logging; +use aws_sdk_s3::Client; +use aws_sdk_s3::primitives::ByteStream; +use std::collections::{BTreeMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +const EC84_NODE_COUNT: usize = 3; +const EC84_DRIVES_PER_NODE: usize = 4; +const EC84_DATA_BLOCKS: usize = 8; +const EC84_PARITY_BLOCKS: usize = 4; + +#[derive(Clone)] +struct ExpectedShard { + key: String, + body: Vec, + baseline: VersionShardCensus, +} + +fn assert_ec84_geometry(census: &VersionShardCensus, key: &str) -> TestResult { + if census.data_blocks != Some(EC84_DATA_BLOCKS) || census.parity_blocks != Some(EC84_PARITY_BLOCKS) { + return Err(format!("object {key} did not use EC8+4 geometry: {census:?}").into()); + } + let erasure_index = census + .erasure_index + .ok_or_else(|| format!("object {key} did not record an erasure index: {census:?}"))?; + if !(1..=EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS).contains(&erasure_index) { + return Err(format!("object {key} has out-of-range erasure index {erasure_index}: {census:?}").into()); + } + if !census.is_complete() || census.expected_part_numbers.is_empty() { + return Err(format!("object {key} does not have complete physical shard evidence: {census:?}").into()); + } + Ok(()) +} + +fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> TestResult { + for key in keys { + let census = census_object_version_on_disk(drive, bucket, key, None)?; + if census.has_xl_meta { + return Err(format!("replacement drive unexpectedly retained {bucket}/{key}: {census:?}").into()); + } + } + Ok(()) +} + +async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult> { + let mut expected = Vec::new(); + for index in 0..4 { + let key = format!("ec84/prefix-{}/object-{index:04}.bin", index % 2); + let body = payload_for(&key, 10 * 1024 * 1024); + put_object(client, bucket, &key, body.clone()).await?; + expected.push(ExpectedShard { + key, + body, + baseline: VersionShardCensus { + version_id: None, + has_xl_meta: false, + data_dir: None, + erasure_index: None, + data_blocks: None, + parity_blocks: None, + expected_part_numbers: Default::default(), + present_part_fingerprints: Default::default(), + inline_data_fingerprint: None, + }, + }); + } + Ok(expected) +} + +#[tokio::test] +async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart() -> TestResult { + init_logging(); + let mut dist = DistCluster::start_with_env( + DistLayout::ThreeByFourEc84, + &[ + ("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"), + ("RUSTFS_HEAL_ENABLED", "true"), + ("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"), + ("RUSTFS_HEAL_MRF_ENABLE", "false"), + ("RUSTFS_SCANNER_ENABLED", "false"), + ], + ) + .await?; + assert_eq!(dist.cluster.nodes.len(), EC84_NODE_COUNT); + assert_eq!(dist.cluster.topology.drives_per_node, EC84_DRIVES_PER_NODE); + + let bucket = unique_bucket("healec84"); + dist.create_bucket(&bucket).await?; + let writer = dist.client(0)?; + let mut expected = put_large_inventory(&writer, &bucket).await?; + let replaced_node = 1; + let replaced_drive_index = 2; + let replaced_drive = PathBuf::from(&dist.cluster.nodes[replaced_node].data_dirs[replaced_drive_index]); + + for item in &mut expected { + item.baseline = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?; + assert_ec84_geometry(&item.baseline, &item.key)?; + } + + let format_path = replaced_drive.join(".rustfs.sys").join("format.json"); + let format_json = std::fs::read(&format_path)?; + dist.cluster.stop_node_gracefully(replaced_node).await?; + let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display())); + std::fs::rename(&replaced_drive, &retired_drive)?; + std::fs::create_dir_all(format_path.parent().ok_or("replacement format path has no parent")?)?; + std::fs::write(&format_path, format_json)?; + assert_replaced_drive_empty( + &replaced_drive, + &bucket, + &expected.iter().map(|item| item.key.clone()).collect::>(), + )?; + + let outage_key = "ec84/written-while-node-restarting.bin"; + let outage_body = payload_for(outage_key, 10 * 1024 * 1024); + writer + .put_object() + .bucket(&bucket) + .key(outage_key) + .body(ByteStream::from(outage_body.clone())) + .send() + .await?; + + dist.cluster.start_node(replaced_node).await?; + let heal_body = + r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#; + let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url); + signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?; + + wait_until( + Duration::from_secs(120), + || async { + for item in &expected { + let current = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?; + if !current.matches_manifest(&item.baseline) { + return Ok(false); + } + } + let outage = census_object_version_on_disk(&replaced_drive, &bucket, outage_key, None)?; + Ok(outage.is_complete() + && outage.data_blocks == Some(EC84_DATA_BLOCKS) + && outage.parity_blocks == Some(EC84_PARITY_BLOCKS)) + }, + "EC8+4 replacement drive rebuilt baseline and outage shards", + ) + .await?; + + let inventory = expected + .iter() + .map(|item| (item.key.clone(), item.body.clone())) + .chain(std::iter::once((outage_key.to_string(), outage_body.clone()))) + .collect::>(); + let expected_keys = inventory.keys().cloned().collect::>(); + for node_index in 0..dist.cluster.nodes.len() { + let client = dist.client(node_index)?; + assert_inventory(&client, &bucket, &inventory).await?; + let listing = client.list_objects_v2().bucket(&bucket).send().await?; + let observed = listing + .contents() + .iter() + .filter_map(|object| object.key().map(str::to_owned)) + .collect::>(); + assert_eq!(observed, expected_keys, "node {node_index} listing diverged after EC8+4 heal"); + } + + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/mod.rs b/crates/e2e_test/src/distributed/mod.rs index 108721f4c..efa5ff433 100644 --- a/crates/e2e_test/src/distributed/mod.rs +++ b/crates/e2e_test/src/distributed/mod.rs @@ -25,6 +25,7 @@ mod data_integrity_movement_test; mod expand_decommission_rebalance_test; mod extra_test; mod harness; +mod heal_test; mod object_lock_test; mod observability_test; mod replication_quota_test; diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index c4bf786f9..b6a3bda17 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -285,6 +285,20 @@ fn peer_replay_state(audience: &str) -> PeerReplayState { .unwrap_or_default() } +pub(crate) fn clear_peer_replay_state_for_addr(addr: &str) -> std::io::Result<()> { + let uri = addr + .parse::() + .map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?; + let audience = uri + .authority() + .map(|authority| normalize_tonic_rpc_audience(authority.as_str())) + .ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))??; + if let Ok(mut states) = PEER_REPLAY_STATES.lock() { + states.remove(&audience); + } + Ok(()) +} + fn apply_peer_replay_response( audience: String, sent_state: PeerReplayState, @@ -619,6 +633,13 @@ mod tests { .remove(audience); } + fn set_peer_capability(audience: &str, state: PeerReplayState) { + PEER_REPLAY_STATES + .lock() + .expect("peer capability cache lock must not be poisoned") + .insert(audience.to_string(), state); + } + fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> { let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest { args: "canonical mutation request".to_string(), @@ -1090,6 +1111,23 @@ mod tests { clear_peer_capability(audience); } + #[test] + fn clear_peer_replay_state_for_addr_removes_normalized_audience() { + let audience = "clear-peer-replay-state-test:9000"; + let boot_epoch = Uuid::new_v4(); + set_peer_capability( + audience, + PeerReplayState { + boot_epoch: Some(boot_epoch), + cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }), + }, + ); + + clear_peer_replay_state_for_addr("http://clear-peer-replay-state-test:9000").expect("peer URI should clear replay state"); + + assert_eq!(peer_replay_state(audience), PeerReplayState::default()); + } + #[test] fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() { ensure_test_rpc_secret(); diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 8b992985f..b59323a92 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -13,8 +13,9 @@ // limitations under the License. use crate::cluster::rpc::client::{ - AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client, - is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client, + AuthenticatedChannel, TonicInterceptor, clear_peer_replay_state_for_addr, embedded_tonic_status, + gen_tonic_signature_interceptor, heal_control_time_out_client, is_network_like_status, message_has_network_needle, + node_service_time_out_client, tier_mutation_control_time_out_client, }; use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof}; use crate::error::{Error, Result}; @@ -544,6 +545,16 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8]) .map_err(|_| Error::other("peer returned an invalid heal control response proof")) } +fn heal_control_auth_may_need_replay_scope_refresh(err: &Error) -> bool { + matches!( + err, + Error::Io(io_err) + if embedded_tonic_status(io_err).is_some_and(|status| { + status.code() == tonic::Code::Unauthenticated && status.message() == "No valid auth token" + }) + ) +} + fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result { let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?; if topology_member != expected_member { @@ -1720,45 +1731,72 @@ impl PeerRestClient { return Err(Error::other("heal control command exceeds size limit")); } let capability_probe = rustfs_protos::is_heal_control_capability_probe(&command); - self.finalize_result( - async { - let mut client = self - .get_heal_control_client() - .await? - .max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE) - .max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE); - let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, &topology_fingerprint, &command) - .map_err(|_| Error::other("heal control request length cannot be represented"))?; - let mut request = Request::new(HealControlRequest { - version, - topology_fingerprint: topology_fingerprint.clone(), - command: command.clone().into(), - }); - request.set_timeout(rustfs_protos::heal_control_execution_timeout()); - set_tonic_canonical_body_digest(&mut request, &canonical_body)?; - let response = client.heal_control(request).await?.into_inner(); - if !response.success { - return Err(Error::other( - response - .error_info - .unwrap_or_else(|| "peer heal control failed without an error".to_string()), - )); - } - if !capability_probe { - let canonical_response = rustfs_protos::canonical_heal_control_response_body( - version, - &topology_fingerprint, - &command, - &response.result, - ) + let result = self + .heal_control_once(version, &topology_fingerprint, &command, capability_probe) + .await; + if result + .as_ref() + .err() + .is_some_and(heal_control_auth_may_need_replay_scope_refresh) + { + self.prepare_heal_control_auth_retry().await; + return self + .finalize_result( + self.heal_control_once(version, &topology_fingerprint, &command, capability_probe) + .await, + ) + .await; + } + self.finalize_result(result).await + } + + async fn prepare_heal_control_auth_retry(&self) { + if let Err(err) = clear_peer_replay_state_for_addr(&self.grid_host) { + debug!( + peer = %self.grid_host, + error = %err, + "could not clear heal control replay state before retry" + ); + } + self.evict_connection().await; + } + + async fn heal_control_once( + &self, + version: u32, + topology_fingerprint: &str, + command: &[u8], + capability_probe: bool, + ) -> Result> { + let mut client = self + .get_heal_control_client() + .await? + .max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE) + .max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE); + let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, topology_fingerprint, command) + .map_err(|_| Error::other("heal control request length cannot be represented"))?; + let mut request = Request::new(HealControlRequest { + version, + topology_fingerprint: topology_fingerprint.to_string(), + command: command.to_vec().into(), + }); + request.set_timeout(rustfs_protos::heal_control_execution_timeout()); + set_tonic_canonical_body_digest(&mut request, &canonical_body)?; + let response = client.heal_control(request).await?.into_inner(); + if !response.success { + return Err(Error::other( + response + .error_info + .unwrap_or_else(|| "peer heal control failed without an error".to_string()), + )); + } + if !capability_probe { + let canonical_response = + rustfs_protos::canonical_heal_control_response_body(version, topology_fingerprint, command, &response.result) .map_err(|_| Error::other("heal control response length cannot be represented"))?; - validate_heal_control_response_proof(&canonical_response, &response.response_proof)?; - } - Ok(response.result.to_vec()) - } - .await, - ) - .await + validate_heal_control_response_proof(&canonical_response, &response.response_proof)?; + } + Ok(response.result.to_vec()) } /// Confirms that a peer supports the current heal-control coordination @@ -3728,6 +3766,22 @@ mod tests { } } + #[test] + fn heal_control_auth_retry_is_limited_to_transport_auth_rejection() { + assert!(heal_control_auth_may_need_replay_scope_refresh(&Error::from( + tonic::Status::unauthenticated("No valid auth token") + ))); + assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from( + tonic::Status::permission_denied("bad signature") + ))); + assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from( + tonic::Status::unauthenticated("application rejected heal control") + ))); + assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::other( + "Io error: code: 'Unauthenticated', message: \"No valid auth token\"" + ))); + } + #[test] fn peer_rest_client_network_classifier_keeps_slow_peers_online() { // The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s) diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 0e5008f9a..a10aeaa06 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -1551,16 +1551,26 @@ class SelfTests(unittest.TestCase): ) + "" ) - physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation", - "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1], - "present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}}, - "inline_data_fingerprint": None} - obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16, - "expected_sha256": "d" * 64, "actual_sha256": "d" * 64, - "expected_physical": physical, "physical": physical} - objects = [dict(obj, key=f"object-{index}") for index in range(9)] - objects[-1] = dict(objects[-1], expected_physical=None) + def oracle_objects(requirement: dict[str, object]) -> list[dict[str, object]]: + topology = requirement["topology"] + total_blocks = topology["nodes"] * topology["drives_per_node"] + parity_blocks = 4 if total_blocks == 12 else total_blocks // 2 + data_blocks = total_blocks - parity_blocks + physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation", + "erasure_index": 1, "data_blocks": data_blocks, "parity_blocks": parity_blocks, + "expected_part_numbers": [1], + "present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}}, + "inline_data_fingerprint": None} + obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16, + "expected_sha256": "d" * 64, "actual_sha256": "d" * 64, + "expected_physical": physical, "physical": physical} + count = requirement.get("min_objects", 9) + objects = [dict(obj, key=f"object-{index}") for index in range(count)] + objects[-1] = dict(objects[-1], expected_physical=None) + return objects + for case_id, requirement in requirements.items(): + objects = oracle_objects(requirement) write_json(run_dir / requirement["oracle"], { "schema": 1, "evidence": requirement["evidence"], "case": case_id, "run_id": "a" * 32, "source_revision": "b" * 40, @@ -1569,7 +1579,7 @@ class SelfTests(unittest.TestCase): "binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"], "topology": requirement["topology"], "pid_before": 10, "pid_after": 11, "unclean_shutdown_marker": requirement["unclean_shutdown_marker"], - "objects": objects, "node_listings": [[item["key"] for item in objects]] * 4, + "objects": objects, "node_listings": [[item["key"] for item in objects]] * requirement["topology"]["nodes"], }) finish_scanner_heal_receipt(run_dir, 0, root) return root, run_dir @@ -1810,7 +1820,9 @@ class SelfTests(unittest.TestCase): registry.pop("release_lanes") registry.pop("release_requirements") write_json(root / ".config/scanner-heal-required-tests.json", registry) - (run_dir / "background-target-crash.json").unlink() + for case_id, requirement in registry["cases"].items(): + if case_id != "background-target-restart": + (run_dir / requirement["oracle"]).unlink() (run_dir / "execution.json").unlink() finish_scanner_heal_receipt(run_dir, 0, root) @@ -1818,7 +1830,7 @@ class SelfTests(unittest.TestCase): self.assertEqual(status["decision"], "blocked") self.assertFalse(status["release_approved"]) self.assertEqual(status["verified_cases"], ["background-target-restart"]) - self.assertEqual(status["rejected_cases"], ["background-target-crash"]) + self.assertEqual(status["rejected_cases"], sorted(set(registry["cases"]) - {"background-target-restart"})) def test_scanner_heal_finish_collects_oracles_from_registry(self) -> None: with tempfile.TemporaryDirectory() as tmp: