test(heal): cover EC8+4 restart shard rebuild (#7461)

Retry heal-control RPCs once after transport auth rejects a stale replay-scope epoch, and add a distributed EC8+4 restart heal evidence case that rebuilds a replaced drive with exact shard/body assertions.

Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-09-08 12:02:58 +08:00
committed by GitHub
parent b7fa6a4615
commit 3149c87cf2
7 changed files with 359 additions and 53 deletions
+14
View File
@@ -28,6 +28,20 @@
"max_objects": 65,
"topology": {"nodes": 4, "drives_per_node": 1},
"scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
},
"ec84-target-drive-restart": {
"gate": "G14",
"task": "W20/W21",
"lane": "e2e-distributed",
"suite": "e2e_test",
"name": "distributed::heal_test::three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart",
"oracle": "ec84-target-drive-restart.json",
"evidence": "process-restart",
"unclean_shutdown_marker": false,
"min_objects": 5,
"max_objects": 5,
"topology": {"nodes": 3, "drives_per_node": 4},
"scope": "3-node x 4-drive single-set EC8+4, graceful target restart, preformatted replacement drive, exact unversioned S3 bodies and physical target shards; not mixed-version, multi-pool or long-window ABBA."
}
},
"release_lanes": {
+5 -1
View File
@@ -59,6 +59,9 @@ const POOL_META_V3_ENV: [(&str, &str); 2] = [
#[derive(Clone, Copy, Debug)]
pub(crate) enum DistLayout {
/// 3 nodes × 4 drives, one erasure pool. With `EC:4` this is the
/// release-evidence EC8+4 geometry.
ThreeByFourEc84,
/// 4 nodes × 4 drives, one erasure pool spanning every endpoint.
FourByFour,
/// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout).
@@ -94,6 +97,7 @@ impl DistCluster {
pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
let topology = match layout {
DistLayout::ThreeByFourEc84 => ClusterTopology::single_pool_multidrive(3, DRIVES_PER_NODE),
DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE),
DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT),
DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]),
@@ -101,7 +105,7 @@ impl DistCluster {
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?;
let pool_storage_roots = match layout {
DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?),
DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
DistLayout::ThreeByFourEc84 | DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
};
let mut owned_pool_dirs = Vec::new();
if let Some(roots) = pool_storage_roots.as_deref() {
@@ -0,0 +1,183 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, assert_inventory, payload_for, put_object, unique_bucket, wait_until};
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
use crate::common::init_logging;
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use std::collections::{BTreeMap, HashSet};
use std::path::{Path, PathBuf};
use std::time::Duration;
const EC84_NODE_COUNT: usize = 3;
const EC84_DRIVES_PER_NODE: usize = 4;
const EC84_DATA_BLOCKS: usize = 8;
const EC84_PARITY_BLOCKS: usize = 4;
#[derive(Clone)]
struct ExpectedShard {
key: String,
body: Vec<u8>,
baseline: VersionShardCensus,
}
fn assert_ec84_geometry(census: &VersionShardCensus, key: &str) -> TestResult {
if census.data_blocks != Some(EC84_DATA_BLOCKS) || census.parity_blocks != Some(EC84_PARITY_BLOCKS) {
return Err(format!("object {key} did not use EC8+4 geometry: {census:?}").into());
}
let erasure_index = census
.erasure_index
.ok_or_else(|| format!("object {key} did not record an erasure index: {census:?}"))?;
if !(1..=EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS).contains(&erasure_index) {
return Err(format!("object {key} has out-of-range erasure index {erasure_index}: {census:?}").into());
}
if !census.is_complete() || census.expected_part_numbers.is_empty() {
return Err(format!("object {key} does not have complete physical shard evidence: {census:?}").into());
}
Ok(())
}
fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> TestResult {
for key in keys {
let census = census_object_version_on_disk(drive, bucket, key, None)?;
if census.has_xl_meta {
return Err(format!("replacement drive unexpectedly retained {bucket}/{key}: {census:?}").into());
}
}
Ok(())
}
async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
let mut expected = Vec::new();
for index in 0..4 {
let key = format!("ec84/prefix-{}/object-{index:04}.bin", index % 2);
let body = payload_for(&key, 10 * 1024 * 1024);
put_object(client, bucket, &key, body.clone()).await?;
expected.push(ExpectedShard {
key,
body,
baseline: VersionShardCensus {
version_id: None,
has_xl_meta: false,
data_dir: None,
erasure_index: None,
data_blocks: None,
parity_blocks: None,
expected_part_numbers: Default::default(),
present_part_fingerprints: Default::default(),
inline_data_fingerprint: None,
},
});
}
Ok(expected)
}
#[tokio::test]
async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart() -> TestResult {
init_logging();
let mut dist = DistCluster::start_with_env(
DistLayout::ThreeByFourEc84,
&[
("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
("RUSTFS_HEAL_ENABLED", "true"),
("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"),
("RUSTFS_HEAL_MRF_ENABLE", "false"),
("RUSTFS_SCANNER_ENABLED", "false"),
],
)
.await?;
assert_eq!(dist.cluster.nodes.len(), EC84_NODE_COUNT);
assert_eq!(dist.cluster.topology.drives_per_node, EC84_DRIVES_PER_NODE);
let bucket = unique_bucket("healec84");
dist.create_bucket(&bucket).await?;
let writer = dist.client(0)?;
let mut expected = put_large_inventory(&writer, &bucket).await?;
let replaced_node = 1;
let replaced_drive_index = 2;
let replaced_drive = PathBuf::from(&dist.cluster.nodes[replaced_node].data_dirs[replaced_drive_index]);
for item in &mut expected {
item.baseline = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
assert_ec84_geometry(&item.baseline, &item.key)?;
}
let format_path = replaced_drive.join(".rustfs.sys").join("format.json");
let format_json = std::fs::read(&format_path)?;
dist.cluster.stop_node_gracefully(replaced_node).await?;
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
std::fs::rename(&replaced_drive, &retired_drive)?;
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path has no parent")?)?;
std::fs::write(&format_path, format_json)?;
assert_replaced_drive_empty(
&replaced_drive,
&bucket,
&expected.iter().map(|item| item.key.clone()).collect::<Vec<_>>(),
)?;
let outage_key = "ec84/written-while-node-restarting.bin";
let outage_body = payload_for(outage_key, 10 * 1024 * 1024);
writer
.put_object()
.bucket(&bucket)
.key(outage_key)
.body(ByteStream::from(outage_body.clone()))
.send()
.await?;
dist.cluster.start_node(replaced_node).await?;
let heal_body =
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
wait_until(
Duration::from_secs(120),
|| async {
for item in &expected {
let current = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
if !current.matches_manifest(&item.baseline) {
return Ok(false);
}
}
let outage = census_object_version_on_disk(&replaced_drive, &bucket, outage_key, None)?;
Ok(outage.is_complete()
&& outage.data_blocks == Some(EC84_DATA_BLOCKS)
&& outage.parity_blocks == Some(EC84_PARITY_BLOCKS))
},
"EC8+4 replacement drive rebuilt baseline and outage shards",
)
.await?;
let inventory = expected
.iter()
.map(|item| (item.key.clone(), item.body.clone()))
.chain(std::iter::once((outage_key.to_string(), outage_body.clone())))
.collect::<BTreeMap<_, _>>();
let expected_keys = inventory.keys().cloned().collect::<HashSet<_>>();
for node_index in 0..dist.cluster.nodes.len() {
let client = dist.client(node_index)?;
assert_inventory(&client, &bucket, &inventory).await?;
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
let observed = listing
.contents()
.iter()
.filter_map(|object| object.key().map(str::to_owned))
.collect::<HashSet<_>>();
assert_eq!(observed, expected_keys, "node {node_index} listing diverged after EC8+4 heal");
}
Ok(())
}
+1
View File
@@ -25,6 +25,7 @@ mod data_integrity_movement_test;
mod expand_decommission_rebalance_test;
mod extra_test;
mod harness;
mod heal_test;
mod object_lock_test;
mod observability_test;
mod replication_quota_test;
+38
View File
@@ -285,6 +285,20 @@ fn peer_replay_state(audience: &str) -> PeerReplayState {
.unwrap_or_default()
}
pub(crate) fn clear_peer_replay_state_for_addr(addr: &str) -> std::io::Result<()> {
let uri = addr
.parse::<Uri>()
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
let audience = uri
.authority()
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))??;
if let Ok(mut states) = PEER_REPLAY_STATES.lock() {
states.remove(&audience);
}
Ok(())
}
fn apply_peer_replay_response(
audience: String,
sent_state: PeerReplayState,
@@ -619,6 +633,13 @@ mod tests {
.remove(audience);
}
fn set_peer_capability(audience: &str, state: PeerReplayState) {
PEER_REPLAY_STATES
.lock()
.expect("peer capability cache lock must not be poisoned")
.insert(audience.to_string(), state);
}
fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> {
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest {
args: "canonical mutation request".to_string(),
@@ -1090,6 +1111,23 @@ mod tests {
clear_peer_capability(audience);
}
#[test]
fn clear_peer_replay_state_for_addr_removes_normalized_audience() {
let audience = "clear-peer-replay-state-test:9000";
let boot_epoch = Uuid::new_v4();
set_peer_capability(
audience,
PeerReplayState {
boot_epoch: Some(boot_epoch),
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
},
);
clear_peer_replay_state_for_addr("http://clear-peer-replay-state-test:9000").expect("peer URI should clear replay state");
assert_eq!(peer_replay_state(audience), PeerReplayState::default());
}
#[test]
fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() {
ensure_test_rpc_secret();
@@ -13,8 +13,9 @@
// limitations under the License.
use crate::cluster::rpc::client::{
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
AuthenticatedChannel, TonicInterceptor, clear_peer_replay_state_for_addr, embedded_tonic_status,
gen_tonic_signature_interceptor, heal_control_time_out_client, is_network_like_status, message_has_network_needle,
node_service_time_out_client, tier_mutation_control_time_out_client,
};
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
use crate::error::{Error, Result};
@@ -544,6 +545,16 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
}
fn heal_control_auth_may_need_replay_scope_refresh(err: &Error) -> bool {
matches!(
err,
Error::Io(io_err)
if embedded_tonic_status(io_err).is_some_and(|status| {
status.code() == tonic::Code::Unauthenticated && status.message() == "No valid auth token"
})
)
}
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
if topology_member != expected_member {
@@ -1720,45 +1731,72 @@ impl PeerRestClient {
return Err(Error::other("heal control command exceeds size limit"));
}
let capability_probe = rustfs_protos::is_heal_control_capability_probe(&command);
self.finalize_result(
async {
let mut client = self
.get_heal_control_client()
.await?
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, &topology_fingerprint, &command)
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
let mut request = Request::new(HealControlRequest {
version,
topology_fingerprint: topology_fingerprint.clone(),
command: command.clone().into(),
});
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
let response = client.heal_control(request).await?.into_inner();
if !response.success {
return Err(Error::other(
response
.error_info
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
));
}
if !capability_probe {
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
version,
&topology_fingerprint,
&command,
&response.result,
)
let result = self
.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
.await;
if result
.as_ref()
.err()
.is_some_and(heal_control_auth_may_need_replay_scope_refresh)
{
self.prepare_heal_control_auth_retry().await;
return self
.finalize_result(
self.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
.await,
)
.await;
}
self.finalize_result(result).await
}
async fn prepare_heal_control_auth_retry(&self) {
if let Err(err) = clear_peer_replay_state_for_addr(&self.grid_host) {
debug!(
peer = %self.grid_host,
error = %err,
"could not clear heal control replay state before retry"
);
}
self.evict_connection().await;
}
async fn heal_control_once(
&self,
version: u32,
topology_fingerprint: &str,
command: &[u8],
capability_probe: bool,
) -> Result<Vec<u8>> {
let mut client = self
.get_heal_control_client()
.await?
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, topology_fingerprint, command)
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
let mut request = Request::new(HealControlRequest {
version,
topology_fingerprint: topology_fingerprint.to_string(),
command: command.to_vec().into(),
});
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
let response = client.heal_control(request).await?.into_inner();
if !response.success {
return Err(Error::other(
response
.error_info
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
));
}
if !capability_probe {
let canonical_response =
rustfs_protos::canonical_heal_control_response_body(version, topology_fingerprint, command, &response.result)
.map_err(|_| Error::other("heal control response length cannot be represented"))?;
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
}
Ok(response.result.to_vec())
}
.await,
)
.await
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
}
Ok(response.result.to_vec())
}
/// Confirms that a peer supports the current heal-control coordination
@@ -3728,6 +3766,22 @@ mod tests {
}
}
#[test]
fn heal_control_auth_retry_is_limited_to_transport_auth_rejection() {
assert!(heal_control_auth_may_need_replay_scope_refresh(&Error::from(
tonic::Status::unauthenticated("No valid auth token")
)));
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
tonic::Status::permission_denied("bad signature")
)));
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
tonic::Status::unauthenticated("application rejected heal control")
)));
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::other(
"Io error: code: 'Unauthenticated', message: \"No valid auth token\""
)));
}
#[test]
fn peer_rest_client_network_classifier_keeps_slow_peers_online() {
// The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s)
+24 -12
View File
@@ -1551,16 +1551,26 @@ class SelfTests(unittest.TestCase):
)
+ "</testsuite></testsuites>"
)
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
"erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1],
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
"inline_data_fingerprint": None}
obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
"expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
"expected_physical": physical, "physical": physical}
objects = [dict(obj, key=f"object-{index}") for index in range(9)]
objects[-1] = dict(objects[-1], expected_physical=None)
def oracle_objects(requirement: dict[str, object]) -> list[dict[str, object]]:
topology = requirement["topology"]
total_blocks = topology["nodes"] * topology["drives_per_node"]
parity_blocks = 4 if total_blocks == 12 else total_blocks // 2
data_blocks = total_blocks - parity_blocks
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
"erasure_index": 1, "data_blocks": data_blocks, "parity_blocks": parity_blocks,
"expected_part_numbers": [1],
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
"inline_data_fingerprint": None}
obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
"expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
"expected_physical": physical, "physical": physical}
count = requirement.get("min_objects", 9)
objects = [dict(obj, key=f"object-{index}") for index in range(count)]
objects[-1] = dict(objects[-1], expected_physical=None)
return objects
for case_id, requirement in requirements.items():
objects = oracle_objects(requirement)
write_json(run_dir / requirement["oracle"], {
"schema": 1, "evidence": requirement["evidence"], "case": case_id,
"run_id": "a" * 32, "source_revision": "b" * 40,
@@ -1569,7 +1579,7 @@ class SelfTests(unittest.TestCase):
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
"topology": requirement["topology"], "pid_before": 10, "pid_after": 11,
"unclean_shutdown_marker": requirement["unclean_shutdown_marker"],
"objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
"objects": objects, "node_listings": [[item["key"] for item in objects]] * requirement["topology"]["nodes"],
})
finish_scanner_heal_receipt(run_dir, 0, root)
return root, run_dir
@@ -1810,7 +1820,9 @@ class SelfTests(unittest.TestCase):
registry.pop("release_lanes")
registry.pop("release_requirements")
write_json(root / ".config/scanner-heal-required-tests.json", registry)
(run_dir / "background-target-crash.json").unlink()
for case_id, requirement in registry["cases"].items():
if case_id != "background-target-restart":
(run_dir / requirement["oracle"]).unlink()
(run_dir / "execution.json").unlink()
finish_scanner_heal_receipt(run_dir, 0, root)
@@ -1818,7 +1830,7 @@ class SelfTests(unittest.TestCase):
self.assertEqual(status["decision"], "blocked")
self.assertFalse(status["release_approved"])
self.assertEqual(status["verified_cases"], ["background-target-restart"])
self.assertEqual(status["rejected_cases"], ["background-target-crash"])
self.assertEqual(status["rejected_cases"], sorted(set(registry["cases"]) - {"background-target-restart"}))
def test_scanner_heal_finish_collects_oracles_from_registry(self) -> None:
with tempfile.TemporaryDirectory() as tmp: