mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 08:36:54 +00:00
fix(heal): harden replacement rebuild tracking (#5892)
* fix(heal): gate auto replacement formatting Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): require replacement target outcomes Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): bind resumes to replacement targets Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): fence healing marker ownership Co-Authored-By: heihutu <heihutu@gmail.com> * test(heal): cover replacement target completion Co-Authored-By: heihutu <heihutu@gmail.com> * docs(heal): clarify replacement recovery status Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): canonicalize replacement target checks Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): satisfy marker test module lint Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): scope automatic replacement format Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): require a mounted replacement target Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): avoid cloned ref slice in test Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): revalidate replacement before scanning Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): reset stale resume checkpoints Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): release scanner disk map before probing Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): persist replacement intent before format Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): fail closed on mountinfo read errors Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): fence replacement target identity Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): order replacement completion cleanup Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): atomically seal replacement completion Co-Authored-By: heihutu <heihutu@gmail.com> * test(heal): census replacement target shards Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): fence replacement recovery ownership Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): preserve replacement recovery anchors Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): satisfy replacement recovery lint gates Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): bind replacement identity to mount lease Co-Authored-By: heihutu <heihutu@gmail.com> * test(heal): cover durable replacement recovery states Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): validate persisted resume task identifiers Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): avoid blocking replacement marker CAS Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): report failed marker rollback Co-Authored-By: heihutu <heihutu@gmail.com> * test(heal): pin replacement resume schema compatibility Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): preserve durable recovery anchors Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): preserve public disk path semantics Co-Authored-By: heihutu <heihutu@gmail.com> * test(heal): use canonical replacement task ids Co-Authored-By: heihutu <heihutu@gmail.com> * test(heal): cover automatic replacement in 3x4 cluster Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): verify replacement target commits Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): persist replacement completion proof Co-Authored-By: heihutu <heihutu@gmail.com> * feat(heal): expose durable replacement status Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): bound durable replacement discovery Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): remove replacement readiness bypass Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): retry terminal replacement cleanup Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): isolate replacement intents from legacy resume Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): migrate legacy replacement intents at startup Co-Authored-By: heihutu <heihutu@gmail.com> * style(heal): apply strict clippy fix Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): prioritize active replacement recovery state Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): bind readiness to the admitted mount lease Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): atomically publish replacement intents Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): isolate replacement recovery directory Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): tolerate an empty recovery directory Co-Authored-By: heihutu <heihutu@gmail.com> * style(heal): remove redundant disk bytes conversion Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): reconcile proof-first replacement recovery Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): fence torn intent recovery Co-Authored-By: heihutu <heihutu@gmail.com> * test(heal): cover replacement migration conflicts Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): fence replacement lease mount identity Co-Authored-By: heihutu <heihutu@gmail.com> * test(heal): cover missing replacement path admission Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): reject conflicting legacy completion proof Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): fall back to proc mount identity Co-Authored-By: heihutu <heihutu@gmail.com> * feat(admin): expose replacement recovery status Surface the local durable replacement recovery snapshot in the background heal status response so operators can tell whether replacement cleanup is definitive or still pending. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): keep replacement status compatible Keep the existing background heal status response wire-compatible while retaining the Linux mount lease cleanup needed for the replacement recovery branch. Co-Authored-By: heihutu <heihutu@gmail.com> * style(ecstore): match linux mount lease formatting Keep Linux rustfmt output stable for the replacement mount lease comparison. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): qualify mount lease test constant Use the disk module path for the format config constant in the Linux mount lease regression test. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): keep procfd mount roots directory-safe Use a procfd path with an explicit directory component so Unix directory guards can open the replacement mount lease root with O_NOFOLLOW while preserving handle-relative I/O semantics. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): delete empty leased buckets via dirfd Use the held mount lease fd as the parent for non-force empty bucket deletion on Linux so procfd-rooted paths do not get rejected as BucketNotEmpty. Also make the download-part OpenOptions truncate behavior explicit and keep fsync test recording stable across procfd canonicalization. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): scan leased bucket paths for emptiness Use the local disk I/O root for bucket emptiness probes before non-force bucket deletion and table-bucket metadata checks. This keeps validation on the same mount instance as the subsequent local disk delete path. Co-Authored-By: heihutu <heihutu@gmail.com> * test(ecstore): align lease path test probes Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): block unsafe replacement recovery restarts Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): defer blocked replacement candidates Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): retry transient replacement discovery Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): keep transient recovery errors retryable Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): block corrupt legacy replacement state Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): classify flat replacement intent corruption Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): keep transient resume loads retryable Classify malformed legacy replacement state as blocking corruption while preserving disk and transient load failures for retry. This avoids permanently blocking replacement recovery on temporary storage errors. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): avoid latching transient legacy publishes Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): retry blocked legacy migrations Co-Authored-By: heihutu <heihutu@gmail.com> * fix(heal): defer blocked startup recoveries Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): preserve disk sync limiter across lease roots Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
@@ -40,6 +40,7 @@ use http::header::{CONTENT_TYPE, HOST};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::collections::BTreeSet;
|
||||
use std::error::Error;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
@@ -48,6 +49,34 @@ use walkdir::WalkDir;
|
||||
|
||||
type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
|
||||
/// Physical `xl.meta` and shard-file census for one object version on one disk.
|
||||
///
|
||||
/// A successful S3 GET only proves that a quorum can serve an object. Replacement
|
||||
/// tests need this lower-level record to prove that the rebuilt target holds the
|
||||
/// `xl.meta` selected for a specific version and every `part.N` it declares.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct VersionShardCensus {
|
||||
pub version_id: Option<String>,
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_numbers: BTreeSet<usize>,
|
||||
}
|
||||
|
||||
impl VersionShardCensus {
|
||||
pub(crate) fn is_complete(&self) -> bool {
|
||||
self.has_xl_meta && self.expected_part_numbers == self.present_part_numbers
|
||||
}
|
||||
|
||||
pub(crate) fn matches_manifest(&self, manifest: &Self) -> bool {
|
||||
self.version_id == manifest.version_id
|
||||
&& self.is_complete()
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
}
|
||||
}
|
||||
|
||||
/// Single-node RustFS server with `disk_count` local volume directories that
|
||||
/// can be faulted individually while the server is running.
|
||||
pub struct DiskFaultHarness {
|
||||
@@ -219,6 +248,78 @@ impl DiskFaultHarness {
|
||||
pub fn object_metadata_exists_on_disk(&self, disk_index: usize, bucket: &str, key: &str) -> bool {
|
||||
self.disks[disk_index].join(bucket).join(key).join("xl.meta").is_file()
|
||||
}
|
||||
|
||||
/// Census the physical files selected by `version_id` on one disk.
|
||||
///
|
||||
/// Missing metadata and missing shard files are represented in the returned
|
||||
/// census rather than as an error so callers can poll replacement progress.
|
||||
/// Invalid metadata or an unknown requested version remains an error: treating
|
||||
/// either as an incomplete rebuild would hide corruption or a wrong-version
|
||||
/// recovery result.
|
||||
pub(crate) fn census_object_version(
|
||||
&self,
|
||||
disk_index: usize,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
) -> ChaosResult<VersionShardCensus> {
|
||||
census_object_version_on_disk(&self.disks[disk_index], bucket, key, version_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Census one physical object version without requiring a single-node harness.
|
||||
/// Cluster replacement tests use the same evidence as the disk-fault tests.
|
||||
pub(crate) fn census_object_version_on_disk(
|
||||
disk: &Path,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
) -> ChaosResult<VersionShardCensus> {
|
||||
let version_id = version_id.map(str::to_owned);
|
||||
let object_dir = disk.join(bucket).join(key);
|
||||
let meta_path = object_dir.join("xl.meta");
|
||||
if !meta_path.is_file() {
|
||||
return Ok(VersionShardCensus {
|
||||
version_id,
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
present_part_numbers: BTreeSet::new(),
|
||||
});
|
||||
}
|
||||
|
||||
let metadata = rustfs_filemeta::FileMeta::load(&std::fs::read(&meta_path)?)?;
|
||||
let file_info = metadata.into_fileinfo(bucket, key, version_id.as_deref().unwrap_or_default(), true, false, true)?;
|
||||
let expected_part_numbers = if file_info.inline_data() {
|
||||
BTreeSet::new()
|
||||
} else {
|
||||
file_info.parts.iter().map(|part| part.number).collect()
|
||||
};
|
||||
let data_dir = file_info.data_dir.map(|id| id.to_string());
|
||||
let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
|
||||
let present_part_numbers = match std::fs::read_dir(&part_dir) {
|
||||
Ok(entries) => entries
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| {
|
||||
entry
|
||||
.file_type()
|
||||
.ok()
|
||||
.filter(|kind| kind.is_file())
|
||||
.and_then(|_| entry.file_name().to_str().map(str::to_owned))
|
||||
})
|
||||
.filter_map(|name| name.strip_prefix("part.").and_then(|number| number.parse::<usize>().ok()))
|
||||
.collect(),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeSet::new(),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
|
||||
Ok(VersionShardCensus {
|
||||
version_id,
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
expected_part_numbers,
|
||||
present_part_numbers,
|
||||
})
|
||||
}
|
||||
|
||||
/// `POST` a signed (SigV4, service `s3`) admin request without relying on the
|
||||
|
||||
@@ -22,15 +22,16 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{DiskFaultHarness, signed_admin_post};
|
||||
use crate::chaos::{DiskFaultHarness, VersionShardCensus, signed_admin_post};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, Instant, interval, timeout};
|
||||
use tracing::info;
|
||||
|
||||
const GET_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
@@ -271,12 +272,17 @@ mod tests {
|
||||
put_and_record(&client, bucket, "heal/nested/large.bin", payload(2 * 1024 * 1024, 34), &mut manifest).await?;
|
||||
|
||||
verify_manifest(&client, bucket, &manifest, "baseline before disk replacement").await?;
|
||||
for (key, _) in &manifest {
|
||||
assert!(
|
||||
harness.object_metadata_exists_on_disk(0, bucket, key),
|
||||
"disk0 should hold xl.meta for {key} before replacement"
|
||||
);
|
||||
}
|
||||
let manifest_keys = manifest.iter().map(|(key, _)| key.clone()).collect::<Vec<_>>();
|
||||
let target_manifest: Vec<(String, VersionShardCensus)> = manifest_keys
|
||||
.iter()
|
||||
.map(|key| {
|
||||
let census = harness.census_object_version(0, bucket, key, None)?;
|
||||
if !census.is_complete() {
|
||||
return Err(format!("disk 0 has incomplete physical census for {key}: {census:?}").into());
|
||||
}
|
||||
Ok((key.clone(), census))
|
||||
})
|
||||
.collect::<Result<_, Box<dyn Error + Send + Sync>>>()?;
|
||||
|
||||
harness.kill_server();
|
||||
harness.replace_disk_with_empty(0)?;
|
||||
@@ -287,21 +293,112 @@ mod tests {
|
||||
signed_admin_post(&heal_url, Some(heal_body), &harness.env.access_key, &harness.env.secret_key).await?;
|
||||
|
||||
let client = harness.env.create_s3_client();
|
||||
let mut remaining: HashSet<String> = manifest.iter().map(|(key, _)| key.clone()).collect();
|
||||
let mut remaining: HashSet<String> = manifest_keys.iter().cloned().collect();
|
||||
let heal_timeout_secs = std::env::var("RUSTFS_RELIABILITY_HEAL_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(120);
|
||||
let deadline = Instant::now() + Duration::from_secs(heal_timeout_secs);
|
||||
let mut retry = interval(Duration::from_secs(1));
|
||||
|
||||
for _ in 0..heal_timeout_secs {
|
||||
remaining.retain(|key| !harness.object_metadata_exists_on_disk(0, bucket, key));
|
||||
loop {
|
||||
remaining.retain(|key| {
|
||||
let expected = target_manifest
|
||||
.iter()
|
||||
.find(|(manifest_key, _)| manifest_key == key)
|
||||
.map(|(_, manifest)| manifest)
|
||||
.expect("every key has a physical manifest");
|
||||
harness
|
||||
.census_object_version(0, bucket, key, None)
|
||||
.map(|census| !census.matches_manifest(expected))
|
||||
.unwrap_or(true)
|
||||
});
|
||||
if remaining.is_empty() {
|
||||
verify_manifest(&client, bucket, &manifest, "after fresh-disk heal completed").await?;
|
||||
return Ok(());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
if Instant::now() >= deadline {
|
||||
break;
|
||||
}
|
||||
retry.tick().await;
|
||||
}
|
||||
|
||||
Err(format!("fresh-disk heal did not rebuild {remaining:?} on the replaced disk within {heal_timeout_secs}s").into())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_versioned_shard_census_selects_each_version_data_dir() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Reliability: physical shard census selects the requested object version");
|
||||
|
||||
let mut harness = DiskFaultHarness::new(4).await?;
|
||||
harness.start_server().await?;
|
||||
let client = harness.env.create_s3_client();
|
||||
let bucket = "reliability-versioned-census";
|
||||
let key = "versions/large.bin";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let first = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(256 * 1024, 41)))
|
||||
.send()
|
||||
.await?;
|
||||
let first_version = first.version_id().ok_or("first PUT did not return a version ID")?;
|
||||
let second = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(256 * 1024, 42)))
|
||||
.send()
|
||||
.await?;
|
||||
let second_version = second.version_id().ok_or("second PUT did not return a version ID")?;
|
||||
let delete = client.delete_object().bucket(bucket).key(key).send().await?;
|
||||
let delete_version = delete.version_id().ok_or("delete marker did not return a version ID")?;
|
||||
|
||||
let first_census = harness.census_object_version(0, bucket, key, Some(first_version))?;
|
||||
let second_census = harness.census_object_version(0, bucket, key, Some(second_version))?;
|
||||
let delete_census = harness.census_object_version(0, bucket, key, Some(delete_version))?;
|
||||
assert!(
|
||||
first_census.is_complete(),
|
||||
"first version physical census is incomplete: {first_census:?}"
|
||||
);
|
||||
assert!(
|
||||
second_census.is_complete(),
|
||||
"second version physical census is incomplete: {second_census:?}"
|
||||
);
|
||||
assert_ne!(
|
||||
first_census.data_dir, second_census.data_dir,
|
||||
"distinct object versions must select distinct physical data directories"
|
||||
);
|
||||
assert_eq!(
|
||||
first_census.expected_part_numbers, second_census.expected_part_numbers,
|
||||
"same single-part shape should expose the same part numbers"
|
||||
);
|
||||
assert!(
|
||||
delete_census.is_complete(),
|
||||
"delete marker physical census is incomplete: {delete_census:?}"
|
||||
);
|
||||
assert!(
|
||||
delete_census.expected_part_numbers.is_empty(),
|
||||
"delete marker must not declare object shards: {delete_census:?}"
|
||||
);
|
||||
assert!(
|
||||
delete_census.present_part_numbers.is_empty(),
|
||||
"delete marker must not select stale object shards: {delete_census:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user