mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-09 05:36:24 +00:00
test(heal): cover EC8+4 restart shard rebuild (#7461)
Retry heal-control RPCs once after transport auth rejects a stale replay-scope epoch, and add a distributed EC8+4 restart heal evidence case that rebuilds a replaced drive with exact shard/body assertions. Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
@@ -28,6 +28,20 @@
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
},
|
||||
"ec84-target-drive-restart": {
|
||||
"gate": "G14",
|
||||
"task": "W20/W21",
|
||||
"lane": "e2e-distributed",
|
||||
"suite": "e2e_test",
|
||||
"name": "distributed::heal_test::three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart",
|
||||
"oracle": "ec84-target-drive-restart.json",
|
||||
"evidence": "process-restart",
|
||||
"unclean_shutdown_marker": false,
|
||||
"min_objects": 5,
|
||||
"max_objects": 5,
|
||||
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||
"scope": "3-node x 4-drive single-set EC8+4, graceful target restart, preformatted replacement drive, exact unversioned S3 bodies and physical target shards; not mixed-version, multi-pool or long-window ABBA."
|
||||
}
|
||||
},
|
||||
"release_lanes": {
|
||||
|
||||
@@ -59,6 +59,9 @@ const POOL_META_V3_ENV: [(&str, &str); 2] = [
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) enum DistLayout {
|
||||
/// 3 nodes × 4 drives, one erasure pool. With `EC:4` this is the
|
||||
/// release-evidence EC8+4 geometry.
|
||||
ThreeByFourEc84,
|
||||
/// 4 nodes × 4 drives, one erasure pool spanning every endpoint.
|
||||
FourByFour,
|
||||
/// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout).
|
||||
@@ -94,6 +97,7 @@ impl DistCluster {
|
||||
|
||||
pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
|
||||
let topology = match layout {
|
||||
DistLayout::ThreeByFourEc84 => ClusterTopology::single_pool_multidrive(3, DRIVES_PER_NODE),
|
||||
DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE),
|
||||
DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT),
|
||||
DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]),
|
||||
@@ -101,7 +105,7 @@ impl DistCluster {
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?;
|
||||
let pool_storage_roots = match layout {
|
||||
DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?),
|
||||
DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
|
||||
DistLayout::ThreeByFourEc84 | DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
|
||||
};
|
||||
let mut owned_pool_dirs = Vec::new();
|
||||
if let Some(roots) = pool_storage_roots.as_deref() {
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_inventory, payload_for, put_object, unique_bucket, wait_until};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
const EC84_NODE_COUNT: usize = 3;
|
||||
const EC84_DRIVES_PER_NODE: usize = 4;
|
||||
const EC84_DATA_BLOCKS: usize = 8;
|
||||
const EC84_PARITY_BLOCKS: usize = 4;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ExpectedShard {
|
||||
key: String,
|
||||
body: Vec<u8>,
|
||||
baseline: VersionShardCensus,
|
||||
}
|
||||
|
||||
fn assert_ec84_geometry(census: &VersionShardCensus, key: &str) -> TestResult {
|
||||
if census.data_blocks != Some(EC84_DATA_BLOCKS) || census.parity_blocks != Some(EC84_PARITY_BLOCKS) {
|
||||
return Err(format!("object {key} did not use EC8+4 geometry: {census:?}").into());
|
||||
}
|
||||
let erasure_index = census
|
||||
.erasure_index
|
||||
.ok_or_else(|| format!("object {key} did not record an erasure index: {census:?}"))?;
|
||||
if !(1..=EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS).contains(&erasure_index) {
|
||||
return Err(format!("object {key} has out-of-range erasure index {erasure_index}: {census:?}").into());
|
||||
}
|
||||
if !census.is_complete() || census.expected_part_numbers.is_empty() {
|
||||
return Err(format!("object {key} does not have complete physical shard evidence: {census:?}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> TestResult {
|
||||
for key in keys {
|
||||
let census = census_object_version_on_disk(drive, bucket, key, None)?;
|
||||
if census.has_xl_meta {
|
||||
return Err(format!("replacement drive unexpectedly retained {bucket}/{key}: {census:?}").into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
|
||||
let mut expected = Vec::new();
|
||||
for index in 0..4 {
|
||||
let key = format!("ec84/prefix-{}/object-{index:04}.bin", index % 2);
|
||||
let body = payload_for(&key, 10 * 1024 * 1024);
|
||||
put_object(client, bucket, &key, body.clone()).await?;
|
||||
expected.push(ExpectedShard {
|
||||
key,
|
||||
body,
|
||||
baseline: VersionShardCensus {
|
||||
version_id: None,
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: Default::default(),
|
||||
present_part_fingerprints: Default::default(),
|
||||
inline_data_fingerprint: None,
|
||||
},
|
||||
});
|
||||
}
|
||||
Ok(expected)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start_with_env(
|
||||
DistLayout::ThreeByFourEc84,
|
||||
&[
|
||||
("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
|
||||
("RUSTFS_HEAL_ENABLED", "true"),
|
||||
("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"),
|
||||
("RUSTFS_HEAL_MRF_ENABLE", "false"),
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(dist.cluster.nodes.len(), EC84_NODE_COUNT);
|
||||
assert_eq!(dist.cluster.topology.drives_per_node, EC84_DRIVES_PER_NODE);
|
||||
|
||||
let bucket = unique_bucket("healec84");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let mut expected = put_large_inventory(&writer, &bucket).await?;
|
||||
let replaced_node = 1;
|
||||
let replaced_drive_index = 2;
|
||||
let replaced_drive = PathBuf::from(&dist.cluster.nodes[replaced_node].data_dirs[replaced_drive_index]);
|
||||
|
||||
for item in &mut expected {
|
||||
item.baseline = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
|
||||
assert_ec84_geometry(&item.baseline, &item.key)?;
|
||||
}
|
||||
|
||||
let format_path = replaced_drive.join(".rustfs.sys").join("format.json");
|
||||
let format_json = std::fs::read(&format_path)?;
|
||||
dist.cluster.stop_node_gracefully(replaced_node).await?;
|
||||
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
|
||||
std::fs::rename(&replaced_drive, &retired_drive)?;
|
||||
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path has no parent")?)?;
|
||||
std::fs::write(&format_path, format_json)?;
|
||||
assert_replaced_drive_empty(
|
||||
&replaced_drive,
|
||||
&bucket,
|
||||
&expected.iter().map(|item| item.key.clone()).collect::<Vec<_>>(),
|
||||
)?;
|
||||
|
||||
let outage_key = "ec84/written-while-node-restarting.bin";
|
||||
let outage_body = payload_for(outage_key, 10 * 1024 * 1024);
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(outage_key)
|
||||
.body(ByteStream::from(outage_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
dist.cluster.start_node(replaced_node).await?;
|
||||
let heal_body =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
|
||||
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
|
||||
wait_until(
|
||||
Duration::from_secs(120),
|
||||
|| async {
|
||||
for item in &expected {
|
||||
let current = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
|
||||
if !current.matches_manifest(&item.baseline) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
let outage = census_object_version_on_disk(&replaced_drive, &bucket, outage_key, None)?;
|
||||
Ok(outage.is_complete()
|
||||
&& outage.data_blocks == Some(EC84_DATA_BLOCKS)
|
||||
&& outage.parity_blocks == Some(EC84_PARITY_BLOCKS))
|
||||
},
|
||||
"EC8+4 replacement drive rebuilt baseline and outage shards",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let inventory = expected
|
||||
.iter()
|
||||
.map(|item| (item.key.clone(), item.body.clone()))
|
||||
.chain(std::iter::once((outage_key.to_string(), outage_body.clone())))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let expected_keys = inventory.keys().cloned().collect::<HashSet<_>>();
|
||||
for node_index in 0..dist.cluster.nodes.len() {
|
||||
let client = dist.client(node_index)?;
|
||||
assert_inventory(&client, &bucket, &inventory).await?;
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let observed = listing
|
||||
.contents()
|
||||
.iter()
|
||||
.filter_map(|object| object.key().map(str::to_owned))
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(observed, expected_keys, "node {node_index} listing diverged after EC8+4 heal");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -25,6 +25,7 @@ mod data_integrity_movement_test;
|
||||
mod expand_decommission_rebalance_test;
|
||||
mod extra_test;
|
||||
mod harness;
|
||||
mod heal_test;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod replication_quota_test;
|
||||
|
||||
@@ -285,6 +285,20 @@ fn peer_replay_state(audience: &str) -> PeerReplayState {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) fn clear_peer_replay_state_for_addr(addr: &str) -> std::io::Result<()> {
|
||||
let uri = addr
|
||||
.parse::<Uri>()
|
||||
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
|
||||
let audience = uri
|
||||
.authority()
|
||||
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
|
||||
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))??;
|
||||
if let Ok(mut states) = PEER_REPLAY_STATES.lock() {
|
||||
states.remove(&audience);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_peer_replay_response(
|
||||
audience: String,
|
||||
sent_state: PeerReplayState,
|
||||
@@ -619,6 +633,13 @@ mod tests {
|
||||
.remove(audience);
|
||||
}
|
||||
|
||||
fn set_peer_capability(audience: &str, state: PeerReplayState) {
|
||||
PEER_REPLAY_STATES
|
||||
.lock()
|
||||
.expect("peer capability cache lock must not be poisoned")
|
||||
.insert(audience.to_string(), state);
|
||||
}
|
||||
|
||||
fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> {
|
||||
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest {
|
||||
args: "canonical mutation request".to_string(),
|
||||
@@ -1090,6 +1111,23 @@ mod tests {
|
||||
clear_peer_capability(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_peer_replay_state_for_addr_removes_normalized_audience() {
|
||||
let audience = "clear-peer-replay-state-test:9000";
|
||||
let boot_epoch = Uuid::new_v4();
|
||||
set_peer_capability(
|
||||
audience,
|
||||
PeerReplayState {
|
||||
boot_epoch: Some(boot_epoch),
|
||||
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
|
||||
},
|
||||
);
|
||||
|
||||
clear_peer_replay_state_for_addr("http://clear-peer-replay-state-test:9000").expect("peer URI should clear replay state");
|
||||
|
||||
assert_eq!(peer_replay_state(audience), PeerReplayState::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -13,8 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
AuthenticatedChannel, TonicInterceptor, clear_peer_replay_state_for_addr, embedded_tonic_status,
|
||||
gen_tonic_signature_interceptor, heal_control_time_out_client, is_network_like_status, message_has_network_needle,
|
||||
node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -544,6 +545,16 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
|
||||
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
|
||||
}
|
||||
|
||||
fn heal_control_auth_may_need_replay_scope_refresh(err: &Error) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
Error::Io(io_err)
|
||||
if embedded_tonic_status(io_err).is_some_and(|status| {
|
||||
status.code() == tonic::Code::Unauthenticated && status.message() == "No valid auth token"
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
|
||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
|
||||
if topology_member != expected_member {
|
||||
@@ -1720,45 +1731,72 @@ impl PeerRestClient {
|
||||
return Err(Error::other("heal control command exceeds size limit"));
|
||||
}
|
||||
let capability_probe = rustfs_protos::is_heal_control_capability_probe(&command);
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self
|
||||
.get_heal_control_client()
|
||||
.await?
|
||||
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
|
||||
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
|
||||
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, &topology_fingerprint, &command)
|
||||
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
|
||||
let mut request = Request::new(HealControlRequest {
|
||||
version,
|
||||
topology_fingerprint: topology_fingerprint.clone(),
|
||||
command: command.clone().into(),
|
||||
});
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||
let response = client.heal_control(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
|
||||
));
|
||||
}
|
||||
if !capability_probe {
|
||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
version,
|
||||
&topology_fingerprint,
|
||||
&command,
|
||||
&response.result,
|
||||
)
|
||||
let result = self
|
||||
.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
|
||||
.await;
|
||||
if result
|
||||
.as_ref()
|
||||
.err()
|
||||
.is_some_and(heal_control_auth_may_need_replay_scope_refresh)
|
||||
{
|
||||
self.prepare_heal_control_auth_retry().await;
|
||||
return self
|
||||
.finalize_result(
|
||||
self.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
|
||||
.await,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.finalize_result(result).await
|
||||
}
|
||||
|
||||
async fn prepare_heal_control_auth_retry(&self) {
|
||||
if let Err(err) = clear_peer_replay_state_for_addr(&self.grid_host) {
|
||||
debug!(
|
||||
peer = %self.grid_host,
|
||||
error = %err,
|
||||
"could not clear heal control replay state before retry"
|
||||
);
|
||||
}
|
||||
self.evict_connection().await;
|
||||
}
|
||||
|
||||
async fn heal_control_once(
|
||||
&self,
|
||||
version: u32,
|
||||
topology_fingerprint: &str,
|
||||
command: &[u8],
|
||||
capability_probe: bool,
|
||||
) -> Result<Vec<u8>> {
|
||||
let mut client = self
|
||||
.get_heal_control_client()
|
||||
.await?
|
||||
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
|
||||
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
|
||||
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, topology_fingerprint, command)
|
||||
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
|
||||
let mut request = Request::new(HealControlRequest {
|
||||
version,
|
||||
topology_fingerprint: topology_fingerprint.to_string(),
|
||||
command: command.to_vec().into(),
|
||||
});
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||
let response = client.heal_control(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
|
||||
));
|
||||
}
|
||||
if !capability_probe {
|
||||
let canonical_response =
|
||||
rustfs_protos::canonical_heal_control_response_body(version, topology_fingerprint, command, &response.result)
|
||||
.map_err(|_| Error::other("heal control response length cannot be represented"))?;
|
||||
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
|
||||
}
|
||||
Ok(response.result.to_vec())
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
|
||||
}
|
||||
Ok(response.result.to_vec())
|
||||
}
|
||||
|
||||
/// Confirms that a peer supports the current heal-control coordination
|
||||
@@ -3728,6 +3766,22 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_control_auth_retry_is_limited_to_transport_auth_rejection() {
|
||||
assert!(heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||
tonic::Status::unauthenticated("No valid auth token")
|
||||
)));
|
||||
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||
tonic::Status::permission_denied("bad signature")
|
||||
)));
|
||||
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||
tonic::Status::unauthenticated("application rejected heal control")
|
||||
)));
|
||||
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::other(
|
||||
"Io error: code: 'Unauthenticated', message: \"No valid auth token\""
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_keeps_slow_peers_online() {
|
||||
// The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s)
|
||||
|
||||
@@ -1551,16 +1551,26 @@ class SelfTests(unittest.TestCase):
|
||||
)
|
||||
+ "</testsuite></testsuites>"
|
||||
)
|
||||
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
|
||||
"erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1],
|
||||
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
|
||||
"inline_data_fingerprint": None}
|
||||
obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
|
||||
"expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
|
||||
"expected_physical": physical, "physical": physical}
|
||||
objects = [dict(obj, key=f"object-{index}") for index in range(9)]
|
||||
objects[-1] = dict(objects[-1], expected_physical=None)
|
||||
def oracle_objects(requirement: dict[str, object]) -> list[dict[str, object]]:
|
||||
topology = requirement["topology"]
|
||||
total_blocks = topology["nodes"] * topology["drives_per_node"]
|
||||
parity_blocks = 4 if total_blocks == 12 else total_blocks // 2
|
||||
data_blocks = total_blocks - parity_blocks
|
||||
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
|
||||
"erasure_index": 1, "data_blocks": data_blocks, "parity_blocks": parity_blocks,
|
||||
"expected_part_numbers": [1],
|
||||
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
|
||||
"inline_data_fingerprint": None}
|
||||
obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
|
||||
"expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
|
||||
"expected_physical": physical, "physical": physical}
|
||||
count = requirement.get("min_objects", 9)
|
||||
objects = [dict(obj, key=f"object-{index}") for index in range(count)]
|
||||
objects[-1] = dict(objects[-1], expected_physical=None)
|
||||
return objects
|
||||
|
||||
for case_id, requirement in requirements.items():
|
||||
objects = oracle_objects(requirement)
|
||||
write_json(run_dir / requirement["oracle"], {
|
||||
"schema": 1, "evidence": requirement["evidence"], "case": case_id,
|
||||
"run_id": "a" * 32, "source_revision": "b" * 40,
|
||||
@@ -1569,7 +1579,7 @@ class SelfTests(unittest.TestCase):
|
||||
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
|
||||
"topology": requirement["topology"], "pid_before": 10, "pid_after": 11,
|
||||
"unclean_shutdown_marker": requirement["unclean_shutdown_marker"],
|
||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
|
||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * requirement["topology"]["nodes"],
|
||||
})
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
return root, run_dir
|
||||
@@ -1810,7 +1820,9 @@ class SelfTests(unittest.TestCase):
|
||||
registry.pop("release_lanes")
|
||||
registry.pop("release_requirements")
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
(run_dir / "background-target-crash.json").unlink()
|
||||
for case_id, requirement in registry["cases"].items():
|
||||
if case_id != "background-target-restart":
|
||||
(run_dir / requirement["oracle"]).unlink()
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
|
||||
@@ -1818,7 +1830,7 @@ class SelfTests(unittest.TestCase):
|
||||
self.assertEqual(status["decision"], "blocked")
|
||||
self.assertFalse(status["release_approved"])
|
||||
self.assertEqual(status["verified_cases"], ["background-target-restart"])
|
||||
self.assertEqual(status["rejected_cases"], ["background-target-crash"])
|
||||
self.assertEqual(status["rejected_cases"], sorted(set(registry["cases"]) - {"background-target-restart"}))
|
||||
|
||||
def test_scanner_heal_finish_collects_oracles_from_registry(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
|
||||
Reference in New Issue
Block a user