diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 9cb0c690e..8ba420268 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -4073,11 +4073,11 @@ pub fn should_heal_object_on_disk( } if !meta.is_canonical_delete_marker() && !meta.is_remote() { - let err_vec = [CHECK_PART_FILE_NOT_FOUND, CHECK_PART_FILE_CORRUPT]; - for part_err in parts_errs.iter() { - if err_vec.contains(part_err) { - return (true, false, Some(DiskError::PartMissingOrCorrupt)); - } + if parts_errs.contains(&CHECK_PART_FILE_CORRUPT) { + return (true, false, Some(DiskError::FileCorrupt)); + } + if parts_errs.contains(&CHECK_PART_FILE_NOT_FOUND) { + return (true, false, Some(DiskError::PartMissingOrCorrupt)); } } (false, false, None) @@ -6866,8 +6866,9 @@ mod tests { assert!(!should_heal); // Test with part corruption - let (should_heal, _, _) = should_heal_object_on_disk(&None, &[CHECK_PART_FILE_CORRUPT], &meta, &latest_meta); + let (should_heal, _, reason) = should_heal_object_on_disk(&None, &[CHECK_PART_FILE_CORRUPT], &meta, &latest_meta); assert!(should_heal); + assert_eq!(reason, Some(DiskError::FileCorrupt)); } #[tokio::test] diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 5d02a2f5f..ee866fb7e 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -20,6 +20,37 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_HEAL: &str = "heal"; const EVENT_HEAL_OBJECT_RENAME: &str = "heal_object_rename"; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct PartFailureSummary { + part_number: usize, + failed_shards: usize, + bitrot_failure: bool, +} + +fn first_unhealthy_part_summary( + data_errs_by_part: &HashMap>, + parts: &[ObjectPartInfo], +) -> Option { + data_errs_by_part + .iter() + .filter_map(|(part_index, part_errs)| { + let failed_shards = count_part_not_success(part_errs); + if failed_shards == 0 { + return None; + } + Some(( + *part_index, + PartFailureSummary { + part_number: parts.get(*part_index).map(|part| part.number).unwrap_or(part_index + 1), + failed_shards, + bitrot_failure: part_errs.contains(&CHECK_PART_FILE_CORRUPT), + }, + )) + }) + .min_by_key(|(part_index, _)| *part_index) + .map(|(_, summary)| summary) +} + impl SetDisks { #[tracing::instrument(skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))] pub(in crate::set_disk) async fn heal_object( @@ -278,17 +309,53 @@ impl SetDisks { let total_disks = parts_metadata.len(); let healthy_count = total_disks.saturating_sub(disks_to_heal_count); let required_data = total_disks.saturating_sub(latest_meta.erasure.parity_blocks); + let no_parity_failure = + (!latest_meta.deleted && !latest_meta.is_remote() && latest_meta.erasure.parity_blocks == 0) + .then(|| first_unhealthy_part_summary(&data_errs_by_part, &latest_meta.parts)) + .flatten(); + let cannot_heal_err = if no_parity_failure.is_some_and(|failure| failure.bitrot_failure) { + DiskError::FileCorrupt + } else { + DiskError::ErasureReadQuorum + }; - error!( - bucket, - object, - version_id, - required_data_shards = required_data, - healthy_shards = healthy_count, - missing_or_corrupt_shards = disks_to_heal_count, - parity_shards = latest_meta.erasure.parity_blocks, - "Heal object cannot reconstruct with available shards" - ); + if let Some(failure) = no_parity_failure { + result.detail = format!( + "no-parity object is unrecoverable: part {} has {} missing or corrupt data shard(s), bitrot_failure={}, data_blocks={}, parity_blocks=0", + failure.part_number, + failure.failed_shards, + failure.bitrot_failure, + latest_meta.erasure.data_blocks + ); + error!( + bucket, + object, + version_id, + data_shards = latest_meta.erasure.data_blocks, + parity_shards = latest_meta.erasure.parity_blocks, + required_data_shards = required_data, + healthy_shards = healthy_count, + missing_or_corrupt_shards = disks_to_heal_count, + part_number = failure.part_number, + bitrot_failure = failure.bitrot_failure, + "No-parity object failed integrity or availability validation and cannot be reconstructed" + ); + } else { + result.detail = format!( + "object cannot be reconstructed with available shards: required_data_shards={required_data}, healthy_shards={healthy_count}, missing_or_corrupt_shards={disks_to_heal_count}, parity_shards={}", + latest_meta.erasure.parity_blocks + ); + error!( + bucket, + object, + version_id, + required_data_shards = required_data, + healthy_shards = healthy_count, + missing_or_corrupt_shards = disks_to_heal_count, + parity_shards = latest_meta.erasure.parity_blocks, + "Heal object cannot reconstruct with available shards" + ); + } // Allow for dangling deletes, on versions that have DataDir missing etc. // this would end up restoring the correct readable versions. @@ -324,19 +391,10 @@ impl SetDisks { object, version_id, error = %err, + returned_error = %cannot_heal_err, "Heal object dangling cleanup could not prove object deletion" ); - let quorum_err = DiskError::ErasureReadQuorum; - let mut t_errs = Vec::with_capacity(errs.len()); - for _ in 0..errs.len() { - t_errs.push(Some(quorum_err.clone())); - } - - Ok(( - self.default_heal_result(FileInfo::default(), &t_errs, bucket, object, version_id) - .await, - Some(quorum_err), - )) + Ok((result, Some(cannot_heal_err))) } }; } @@ -1258,8 +1316,12 @@ mod heal_result_report_tests { use crate::disk::error::DiskError; use crate::disk::format::FormatV3; use crate::disk::{DiskOption, DiskStore, new_disk}; - use rustfs_common::heal_channel::DriveState; - use rustfs_filemeta::FileInfo; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use crate::{config::storageclass, store::init_format::save_format_file}; + use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode}; + use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo}; use std::sync::Arc; use tempfile::TempDir; use tokio::sync::RwLock; @@ -1300,6 +1362,50 @@ mod heal_result_report_tests { .await } + async fn formatted_single_disk_no_parity_set() -> (TempDir, Arc) { + let format = FormatV3::new(1, 1); + let dir = tempfile::tempdir().expect("tempdir should be created"); + let mut endpoint = + Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("disk should be created"); + + let mut disk_format = format.clone(); + disk_format.erasure.this = format.erasure.sets[0][0]; + save_format_file(&Some(disk.clone()), &Some(disk_format)) + .await + .expect("format should be saved"); + + let set = SetDisks::new( + "test-owner".to_string(), + Arc::new(RwLock::new(vec![Some(disk)])), + 1, + 0, + 0, + 0, + vec![endpoint], + format, + vec![], + ) + .await; + set.set_test_storage_class_config( + storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[1]) + .expect("test storage class should resolve for one local drive"), + ); + (dir, set) + } + // Regression for #955: an offline disk must contribute exactly one drive // record. Before the fix the offline branch fell through and pushed a second // (Corrupt) record for the same disk, so `before/after.drives` grew to @@ -1397,4 +1503,61 @@ mod heal_result_report_tests { assert_eq!(result.before.drives[2].state, DriveState::Offline.to_string()); assert_eq!(result.before.drives[3].state, DriveState::Ok.to_string()); } + + #[tokio::test] + async fn heal_no_parity_bitrot_reports_unrecoverable_integrity_failure() { + let (dir, set) = formatted_single_disk_no_parity_set().await; + let bucket = "bucket-no-parity-bitrot"; + let object = "bad-object.bin"; + let payload = (0..(BLOCK_SIZE_V2 + 17)).map(|idx| (idx % 251) as u8).collect::>(); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut reader = PutObjReader::from_vec(payload); + set.put_object(bucket, object, &mut reader, &opts) + .await + .expect("object should be written"); + + let (fi, _, _) = set + .get_object_fileinfo(bucket, object, &opts, true, false) + .await + .expect("object metadata should resolve"); + assert_eq!(fi.erasure.parity_blocks, 0); + let data_dir = fi.data_dir.expect("non-inline object should have a data directory"); + let part_path = dir.path().join(bucket).join(object).join(data_dir.to_string()).join("part.1"); + let mut part = tokio::fs::read(&part_path).await.expect("part should be readable"); + part[0] ^= 0xff; + tokio::fs::write(&part_path, part) + .await + .expect("part corruption should be written"); + + let (result, err) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("heal should report the unrecoverable object without panicking"); + + assert_eq!(err, Some(DiskError::FileCorrupt)); + assert_eq!(result.bucket, bucket); + assert_eq!(result.object, object); + assert_eq!(result.data_blocks, 1); + assert_eq!(result.parity_blocks, 0); + assert_eq!(result.before.drives[0].state, DriveState::Corrupt.to_string()); + assert!(result.detail.contains("no-parity object is unrecoverable")); + assert!(result.detail.contains("part 1")); + assert!(result.detail.contains("bitrot_failure=true")); + } } diff --git a/docs/operations/durability-modes.md b/docs/operations/durability-modes.md index 5e870ee40..5ef611f9e 100644 --- a/docs/operations/durability-modes.md +++ b/docs/operations/durability-modes.md @@ -201,3 +201,9 @@ by environment variable. Phase 2 (rustfs/backlog#938) adds the per-bucket override described above, configured through the admin API and stored in bucket metadata. `mc admin` integration for the per-bucket tier is a follow-up. + +## Related Recovery Guides + +- [No-parity bitrot recovery](no-parity-bitrot-recovery.md) explains how to + diagnose historical no-parity objects whose raw `part.N` files are present + but fail bitrot validation. diff --git a/docs/operations/no-parity-bitrot-recovery.md b/docs/operations/no-parity-bitrot-recovery.md new file mode 100644 index 000000000..6f5bf123d --- /dev/null +++ b/docs/operations/no-parity-bitrot-recovery.md @@ -0,0 +1,98 @@ +# No-Parity Bitrot Recovery Guide + +This guide covers historical objects written with erasure data shards but no +parity shards, for example an object whose `xl.meta` reports `EcM=1` and +`EcN=0`. PR #5179 prevents RustFS from committing new objects after it detects +this class of no-parity bitrot failure, but operators may still find already +committed objects on disk. + +## Symptom + +An affected object can look surprising during incident response: + +- the raw shard file, such as `part.1`, is visible and readable from the local + filesystem; +- an S3 GET or deep heal reports an integrity failure, commonly through + `FileCorrupt`, `bitrot hash mismatch`, or an unrecoverable heal result; +- there are no parity shards available to reconstruct the corrupted data shard. + +The filesystem-readable `part.N` file is therefore evidence, not trusted object +data. RustFS must not bypass bitrot validation to serve it through S3, because +the stored hash no longer matches the bytes on disk. + +## What To Capture + +Before deleting or moving anything, capture: + +- bucket name, object key, and version ID if versioning is enabled; +- the RustFS version and whether the deployment was running with no parity + (`EcN=0`) at the time the object was written; +- the heal or GET error, including any `FileCorrupt`, `bitrot hash mismatch`, + `ErasureReadQuorum`, or truncated streaming response message; +- `xl.meta` from every shard disk that still has the object; +- the raw `part.N` file from every shard disk that still has the object. + +For local inspection, decode metadata with: + +```bash +cargo run -p rustfs-filemeta --example dump_fileinfo -- /path/to/disk/bucket/object/xl.meta +``` + +Record the erasure geometry (`EcM`, `EcN`), object size, part number, part +logical size, data directory, and checksum algorithm from the decoded metadata. + +## Size Accounting + +RustFS erasure shard files include bitrot hash data in addition to object bytes. +For the default HighwayHash256S checksum, each protected block adds 32 bytes of +hash data to the shard file. A raw `part.1` size can therefore be larger than +the object logical size and still be normal. + +Example: + +```text +logical object bytes: 8,250,370 +protected blocks: 8 +hash overhead: 8 * 32 = 256 bytes +raw part.1 bytes: 8,250,626 +``` + +This size relationship only proves that the file layout is plausible. It does +not prove the bytes are valid. The bitrot reader is the authority for integrity. + +## Recovery Boundary + +If `EcN=0` and a data shard fails bitrot verification, RustFS cannot reconstruct +the object from the erasure set. The valid recovery options are: + +- restore the object from an external backup, replica, upstream source, or a + known-good copy outside the affected erasure set; +- preserve the affected `xl.meta` and `part.N` files as incident evidence, then + delete the object through the normal S3/admin path when retention policy + allows it; +- quarantine by copying evidence out of the live data path first, then remove + or isolate the live object path only after the incident owner confirms the + evidence is no longer needed. + +Do not edit `xl.meta`, rewrite `part.N`, or serve raw shard bytes to clients as +the object. Those actions hide evidence and can convert a detected integrity +failure into silent data corruption. + +If `EcN>0`, this guide is not the primary recovery path. Use normal heal first; +parity may allow RustFS to reconstruct the missing or corrupt shard. + +## Expected Diagnostics + +Deep heal should report no-parity corruption as an unrecoverable integrity +failure rather than only a generic read-quorum problem. The diagnostic context +should include: + +- bucket, object, and version ID; +- erasure data shard count and parity shard count; +- part number; +- whether the failing part had a bitrot failure; +- the number of missing or corrupt shards. + +When the failure is confirmed bitrot on a no-parity object, the heal error is +reported as `FileCorrupt`, and the heal result `detail` states that the +no-parity object is unrecoverable.