diff --git a/Cargo.lock b/Cargo.lock index 699320263..e36abcc99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9893,6 +9893,7 @@ name = "rustfs-filemeta" version = "1.0.0-rc.5" dependencies = [ "arc-swap", + "base64-simd", "byteorder", "bytes", "crc-fast", @@ -9907,6 +9908,7 @@ dependencies = [ "s3s", "serde", "serde_json", + "sha2 0.11.0", "tempfile", "thiserror 2.0.20", "time", diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index b96ae3cde..33507dc26 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -116,6 +116,20 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR /// Default: bitrot verification is enabled on GetObject reads (do not skip). pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false; +/// Create independent shard commitments for new writes after the fleet is upgraded. +/// Existing protected objects and multipart uploads retain their protection. +pub const ENV_SHARD_INTEGRITY_WRITE: &str = "RUSTFS_SHARD_INTEGRITY_WRITE"; +pub const DEFAULT_SHARD_INTEGRITY_WRITE: bool = false; + +/// Operator confirmation that every reader, writer and background coordinator +/// understands independent shard commitments. This is not capability discovery +/// or a fence against an old binary rejoining the fleet. +pub const ENV_SHARD_INTEGRITY_FLEET_CONFIRMED: &str = "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED"; +pub const DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED: bool = false; + +const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_WRITE); +const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED); + /// Request writing the complete remote-tier version state into object metadata. /// /// This remains ineffective until diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 7b295eec7..d84f18d25 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -501,9 +501,9 @@ pub mod object { ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError, - ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len, - lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook, - unregister_get_object_body_cache_hook, unregister_object_mutation_hook, + ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCompletion, + get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, + register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, }; pub use crate::store::{ PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError, diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index abdb9f446..3f7dfaa03 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -5190,6 +5190,7 @@ pub async fn put_restore_opts( // Restore writes stored (possibly encrypted) bytes, so the writer's // computed MD5 is not the object's public plaintext ETag. preserve_etag: oi.etag.clone(), + shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()), //expires: oi.expires, ..Default::default() }) diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 2116f7fb5..c1346f3ff 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -844,6 +844,44 @@ fn spawn_control_channel_prewarm(addr: String) { } impl RemoteDisk { + async fn rename_file_with_durability( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + durable: bool, + ) -> Result<()> { + self.execute_with_timeout( + || async { + let mut client = self.get_client().await?; + let mut request = Request::new(RenameFileRequest { + durable, + disk: self.endpoint.to_string(), + src_volume: src_volume.to_string(), + src_path: src_path.to_string(), + dst_volume: dst_volume.to_string(), + dst_path: dst_path.to_string(), + }); + let canonical_body = rustfs_protos::canonical_rename_file_request_body(request.get_ref()); + attach_mutation_body_digest(&mut request, canonical_body, "rename_file")?; + + let response = client.rename_file(request).await?.into_inner(); + + if !response.success { + return Err(response.error.unwrap_or_default().into()); + } + + if durable && !response.durability_applied { + return Err(DiskError::MethodNotAllowed); + } + Ok(()) + }, + get_max_timeout_duration(), + ) + .await + } + pub(crate) async fn ns_scanner_server_epoch(&self) -> Result> { if self.health.is_faulty() { return Err(DiskError::FaultyDisk); @@ -3401,30 +3439,13 @@ impl DiskAPI for RemoteDisk { "Remote disk RPC started" ); - self.execute_with_timeout( - || async { - let mut client = self.get_client().await?; - let mut request = Request::new(RenameFileRequest { - disk: self.endpoint.to_string(), - src_volume: src_volume.to_string(), - src_path: src_path.to_string(), - dst_volume: dst_volume.to_string(), - dst_path: dst_path.to_string(), - }); - let canonical_body = rustfs_protos::canonical_rename_file_request_body(request.get_ref()); - attach_mutation_body_digest(&mut request, canonical_body, "rename_file")?; + self.rename_file_with_durability(src_volume, src_path, dst_volume, dst_path, false) + .await + } - let response = client.rename_file(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - Ok(()) - }, - get_max_timeout_duration(), - ) - .await + async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { + self.rename_file_with_durability(src_volume, src_path, dst_volume, dst_path, true) + .await } #[tracing::instrument(level = "trace", skip_all)] diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index 2a09bf0c7..b342877de 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -785,6 +785,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets { let put_opts = ObjectOptions { user_defined: dst_opts.user_defined.clone(), + shard_integrity_write_mode: Some(src_info.shard_integrity_write_mode()), versioned: dst_opts.versioned, version_id: dst_opts.version_id.clone(), mod_time: dst_opts.mod_time, diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index eb532c8dc..3c8cba3ac 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -296,6 +296,7 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz preserve_etag: object_info.etag.clone(), src_pool_idx, data_movement: true, + shard_integrity_write_mode: Some(object_info.shard_integrity_write_mode()), ..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok()) } } @@ -476,6 +477,7 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize) versioned: object_info.version_id.is_some(), src_pool_idx, data_movement: true, + shard_integrity_write_mode: Some(object_info.shard_integrity_write_mode()), version_id: object_info.version_id.as_ref().map(|v| v.to_string()), http_preconditions: Some(data_movement_target_precondition()), mod_time: object_info.mod_time, @@ -2109,6 +2111,27 @@ async fn migrate_object_inner( #[cfg(test)] mod tests { use super::*; + + #[test] + fn data_movement_retains_source_shard_integrity_mode() { + use crate::object_api::ShardIntegrityWriteMode; + let mut source = ObjectInfo::default(); + for mode in [ShardIntegrityWriteMode::Legacy, ShardIntegrityWriteMode::Protected] { + if mode == ShardIntegrityWriteMode::Protected { + // A declaration selects protected I/O; the source reader still + // rejects this deliberately incomplete descriptor before commit. + rustfs_utils::http::insert_str( + Arc::make_mut(&mut source.user_defined), + rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY, + "invalid".to_owned(), + ); + } + let put = data_movement_put_object_opts(&source, 0); + let multipart = data_movement_new_multipart_opts(&source, 0); + assert_eq!(put.shard_integrity_write_mode, Some(mode)); + assert_eq!(multipart.shard_integrity_write_mode, Some(mode)); + } + } use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType}; use rustfs_rio::{Checksum, ChecksumType}; use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE}; diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 0d79ca559..ec7b93e1c 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -2396,6 +2396,20 @@ impl DiskAPI for LocalDiskWrapper { .await } + async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { + self.track_disk_health_mutation( + "rename_file", + DiskMetricMutation::Write, + || async { + self.disk + .rename_file_durable(src_volume, src_path, dst_volume, dst_path) + .await + }, + get_max_timeout_duration(), + ) + .await + } + async fn prepare_part_transaction( &self, src_volume: &str, diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index d888e3f7d..d056d8068 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -9354,6 +9354,41 @@ impl DiskAPI for LocalDisk { } let durability = effective_durability(dst_volume); + let part = ObjectPartInfo::unmarshal(&meta)?; + if let Some(integrity) = part.integrity { + integrity.validate()?; + if usize::try_from(integrity.number).map_err(|_| DiskError::FileCorrupt)? != part.number + || usize::try_from(integrity.size).map_err(|_| DiskError::FileCorrupt)? != part.size + || dst_file_path.file_name().and_then(|name| name.to_str()) != Some(format!("part.{}", part.number).as_str()) + { + return Err(DiskError::FileCorrupt); + } + let proof_name = integrity.file_name(); + let source = src_file_path.parent().ok_or(DiskError::FileCorrupt)?.join(&proof_name); + let destination = dst_file_path.parent().ok_or(DiskError::FileCorrupt)?.join(&proof_name); + check_path_length(source.to_string_lossy().as_ref())?; + check_path_length(destination.to_string_lossy().as_ref())?; + let stat = fs::symlink_metadata(&source).await.map_err(to_file_error)?; + if !stat.is_file() || stat.len() != u64::try_from(integrity.index_size()?).map_err(|_| DiskError::FileCorrupt)? { + return Err(DiskError::FileCorrupt); + } + if durability.syncs_data_shards() { + let source = source.clone(); + tokio::task::spawn_blocking(move || os::sync_file(&source)) + .await + .map_err(DiskError::from)? + .map_err(to_file_error)?; + } + // Publish the immutable generation before preparing the part switch. + // Rollback retains the old generation; an unreferenced new index is + // reclaimed with the upload directory if preparation is interrupted. + rename_all(&source, &destination, &dst_volume_dir, &self.publication_root).await?; + if durability.syncs_commit_metadata() { + os::fsync_dir(destination.parent().ok_or(DiskError::FileCorrupt)?) + .await + .map_err(to_file_error)?; + } + } tokio::task::spawn_blocking(move || { let source = std::fs::symlink_metadata(&src_file_path).map_err(to_file_error)?; if !source.is_file() { @@ -9462,6 +9497,41 @@ impl DiskAPI for LocalDisk { let Some(parent) = transaction_path.parent() else { return Err(DiskError::InvalidPath); }; + // Delete only the generation made obsolete by this settled switch. + // Check the published metadata before removing either proof, so a + // repeated or interrupted settlement cannot remove a live index. + let (retained_name, obsolete_name) = match action { + PartTransactionAction::Commit => (PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META), + PartTransactionAction::Rollback => (PART_TRANSACTION_OLD_META, PART_TRANSACTION_NEW_META), + }; + if let (Ok(current), Ok(retained), Ok(obsolete)) = ( + std::fs::read(¤t_meta_path), + std::fs::read(transaction_path.join(retained_name)), + std::fs::read(transaction_path.join(obsolete_name)), + ) && current == retained + && let Ok(obsolete) = ObjectPartInfo::unmarshal(&obsolete) + && let Some(integrity) = obsolete.integrity + && integrity.validate().is_ok() + && usize::try_from(integrity.number).ok() == Some(obsolete.number) + && current_data_path.file_name().and_then(|name| name.to_str()) + == Some(format!("part.{}", obsolete.number).as_str()) + && ObjectPartInfo::unmarshal(&retained) + .ok() + .and_then(|part| part.integrity) + .as_ref() + != Some(&integrity) + { + let obsolete_path = parent.join(integrity.file_name()); + match std::fs::remove_file(&obsolete_path) { + Ok(()) => { + if durability.syncs_commit_metadata() { + os::fsync_dir_std(parent).map_err(to_file_error)?; + } + } + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(error) => return Err(to_file_error(error).into()), + } + } let cleanup_path = parent.join(format!(".part-txn-settled-{}", Uuid::new_v4())); std::fs::rename(&transaction_path, &cleanup_path).map_err(to_file_error)?; if durability.syncs_commit_metadata() { @@ -9735,6 +9805,31 @@ impl DiskAPI for LocalDisk { .await } + async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { + let source = self.io_get_object_path(src_volume, src_path)?; + let destination = self.io_get_object_path(dst_volume, dst_path)?; + check_path_length(source.to_string_lossy().as_ref())?; + check_path_length(destination.to_string_lossy().as_ref())?; + let durability = effective_durability(dst_volume); + let stat = fs::symlink_metadata(&source).await.map_err(to_file_error)?; + if !stat.is_file() { + return Err(DiskError::FileAccessDenied); + } + if durability.syncs_data_shards() { + tokio::task::spawn_blocking(move || os::sync_file(&source)) + .await + .map_err(DiskError::from)? + .map_err(to_file_error)?; + } + self.rename_file(src_volume, src_path, dst_volume, dst_path).await?; + if durability.syncs_commit_metadata() { + os::fsync_dir(destination.parent().ok_or(DiskError::InvalidPath)?) + .await + .map_err(to_file_error)?; + } + Ok(()) + } + #[tracing::instrument(level = "trace", skip_all)] // async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result { async fn append_file(&self, volume: &str, path: &str) -> Result { @@ -12088,7 +12183,15 @@ mod test { ensure_test_volume(&disk, bucket).await; let payload = Bytes::from_static(b"part payload"); - let meta = Bytes::from_static(b"part metadata"); + let meta = Bytes::from( + ObjectPartInfo { + number: 1, + size: payload.len(), + ..Default::default() + } + .marshal_msg() + .expect("legacy part metadata"), + ); disk.write_all(tmp_volume, "upload/part.1", payload.clone()) .await .expect("source part should be written"); @@ -12178,7 +12281,15 @@ mod test { "regression path must cross the traditional Windows MAX_PATH boundary: {deepest_marker:?}" ); let payload = Bytes::from_static(b"part payload"); - let meta = Bytes::from_static(b"part metadata"); + let meta = Bytes::from( + ObjectPartInfo { + number: 1, + size: payload.len(), + ..Default::default() + } + .marshal_msg() + .expect("legacy part metadata"), + ); disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, payload.clone()) .await .expect("source part should be written"); @@ -12207,7 +12318,15 @@ mod test { ); let replacement_payload = Bytes::from_static(b"replacement part payload"); - let replacement_meta = Bytes::from_static(b"replacement part metadata"); + let replacement_meta = Bytes::from( + ObjectPartInfo { + number: 1, + size: replacement_payload.len(), + ..Default::default() + } + .marshal_msg() + .expect("legacy replacement metadata"), + ); disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, replacement_payload.clone()) .await .expect("replacement source part should be written"); @@ -12272,9 +12391,23 @@ mod test { .await .expect("old part metadata should be staged"); - disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", Bytes::from_static(b"new metadata")) - .await - .expect("part transaction should be prepared"); + disk.prepare_part_transaction( + "tmp", + "upload/part.1", + "bucket", + "object/part.1", + Bytes::from( + ObjectPartInfo { + number: 1, + size: 8, + ..Default::default() + } + .marshal_msg() + .expect("legacy part metadata"), + ), + ) + .await + .expect("part transaction should be prepared"); disk.rename_file("tmp", "upload/part.1", "bucket", "object/part.1") .await .expect("data publication should succeed"); @@ -12296,6 +12429,139 @@ mod test { ); } + #[tokio::test] + async fn shard_integrity_part_transaction_keeps_only_the_settled_generation() { + use crate::io_support::shard_integrity::IntegrityBuilder; + use rustfs_filemeta::shard_integrity::IntegrityLayout; + for action in [PartTransactionAction::Commit, PartTransactionAction::Rollback] { + let dir = tempfile::tempdir().expect("fixture"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("disk"); + ensure_test_volume(&disk, "tmp").await; + ensure_test_volume(&disk, "bucket").await; + let mut parts = Vec::new(); + for (volume, directory, byte) in [("bucket", "object", b'a'), ("tmp", "upload", b'b')] { + let mut builder = + IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 1).expect("builder"); + let shard = [byte; 4]; + builder.push([shard.as_slice(); 4].into_iter()).await.expect("stripe"); + let prepared = builder.finish(8).await.expect("proof"); + disk.write_all( + volume, + &format!("{directory}/{}", prepared.part.file_name()), + prepared.inline_bytes().expect("index"), + ) + .await + .expect("index file"); + disk.write_all(volume, &format!("{directory}/part.1"), Bytes::copy_from_slice(&shard)) + .await + .expect("data"); + let part = ObjectPartInfo { + number: 1, + size: 8, + integrity: Some(prepared.part), + ..Default::default() + }; + let meta = Bytes::from(part.marshal_msg().expect("metadata")); + disk.write_all(volume, &format!("{directory}/part.1.meta"), meta.clone()) + .await + .expect("part meta"); + parts.push((part, meta)); + } + disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", parts[1].1.clone()) + .await + .expect("prepare"); + for (part, _) in &parts { + assert!( + disk.read_all("bucket", &format!("object/{}", part.integrity.as_ref().expect("proof").file_name())) + .await + .is_ok(), + "both generations survive preparation" + ); + } + disk.rename_part("tmp", "upload/part.1", "bucket", "object/part.1", parts[1].1.clone()) + .await + .expect("publish"); + disk.settle_part_transaction("bucket", "object/part.1", action) + .await + .expect("settle"); + let retained = usize::from(action == PartTransactionAction::Commit); + assert_eq!( + disk.read_all("bucket", "object/part.1.meta").await.expect("settled metadata"), + parts[retained].1 + ); + for (index, (part, _)) in parts.iter().enumerate() { + let exists = disk + .read_all("bucket", &format!("object/{}", part.integrity.as_ref().expect("proof").file_name())) + .await + .is_ok(); + assert_eq!(exists, index == retained, "obsolete proof is reclaimed after settlement"); + } + } + } + + #[tokio::test] + async fn shard_integrity_settlement_never_deletes_another_parts_index() { + use crate::io_support::shard_integrity::IntegrityBuilder; + use rustfs_filemeta::shard_integrity::IntegrityLayout; + let dir = tempfile::tempdir().expect("fixture"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("disk"); + ensure_test_volume(&disk, "tmp").await; + ensure_test_volume(&disk, "bucket").await; + let mut builder = IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 2).expect("builder"); + builder.push([b"data".as_slice(); 4].into_iter()).await.expect("stripe"); + let prepared = builder.finish(8).await.expect("index"); + let other_index = format!("object/{}", prepared.part.file_name()); + let index_bytes = prepared.inline_bytes().expect("index bytes"); + disk.write_all("bucket", &other_index, index_bytes.clone()) + .await + .expect("part 2 index"); + let corrupt = ObjectPartInfo { + number: 1, + size: 8, + integrity: Some(prepared.part), + ..Default::default() + }; + disk.write_all("bucket", "object/part.1", Bytes::from_static(b"old data")) + .await + .expect("old data"); + disk.write_all( + "bucket", + "object/part.1.meta", + Bytes::from(corrupt.marshal_msg().expect("misdirected metadata")), + ) + .await + .expect("corrupt old metadata"); + disk.write_all("tmp", "upload/part.1", Bytes::from_static(b"new data")) + .await + .expect("new data"); + let replacement = Bytes::from( + ObjectPartInfo { + number: 1, + size: 8, + ..Default::default() + } + .marshal_msg() + .expect("replacement"), + ); + disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", replacement.clone()) + .await + .expect("prepare"); + disk.rename_part("tmp", "upload/part.1", "bucket", "object/part.1", replacement) + .await + .expect("publish"); + disk.settle_part_transaction("bucket", "object/part.1", PartTransactionAction::Commit) + .await + .expect("settle"); + assert_eq!( + disk.read_all("bucket", &other_index) + .await + .expect("unrelated part index retained"), + index_bytes + ); + } + struct BlockingScanWriter { entered_tx: Option>, } diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs index aa32f6fd7..ac064810b 100644 --- a/crates/ecstore/src/disk/local/commit.rs +++ b/crates/ecstore/src/disk/local/commit.rs @@ -304,7 +304,13 @@ impl LocalDisk { let data_dir = fi.data_dir.unwrap_or_default(); fi.parts .iter() - .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) + .flat_map(|part| { + std::iter::once(format!("{dst_path}/{data_dir}/part.{}", part.number)).chain( + part.integrity + .as_ref() + .map(|integrity| format!("{dst_path}/{data_dir}/{}", integrity.file_name())), + ) + }) .collect() }; let src_volume_dir = self.io_get_bucket_path(src_volume)?; diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index a3c4af8d2..6da4b7975 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -577,6 +577,13 @@ impl DiskAPI for Disk { } } + async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { + match self { + Disk::Local(disk) => disk.rename_file_durable(src_volume, src_path, dst_volume, dst_path).await, + Disk::Remote(disk) => disk.rename_file_durable(src_volume, src_path, dst_volume, dst_path).await, + } + } + #[tracing::instrument(level = "trace", skip_all)] async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> { match self { @@ -1175,6 +1182,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result; // ReadFileStream async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>; + async fn rename_file_durable(&self, _src_volume: &str, _src_path: &str, _dst_volume: &str, _dst_path: &str) -> Result<()> { + Err(DiskError::MethodNotAllowed) + } async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()>; async fn prepare_part_transaction( &self, diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index e950644ef..ee4609d6a 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -102,6 +102,7 @@ pin_project! { chunks: Vec, skip_verify: bool, last_verify_duration: Duration, + integrity: Option, } } @@ -119,6 +120,7 @@ where chunks: Vec::new(), skip_verify, last_verify_duration: Duration::ZERO, + integrity: None, } } @@ -126,6 +128,24 @@ where self.last_verify_duration } + pub(crate) fn integrity_proof(&self) -> Option> { + self.integrity + .as_ref() + .map(crate::io_support::shard_integrity::ShardVerifier::proof) + } + + pub(crate) fn set_integrity(&mut self, verifier: crate::io_support::shard_integrity::ShardVerifier) -> std::io::Result<()> { + if self.hash_algo != HashAlgorithm::HighwayHash256S { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "unsupported protected shard framing", + )); + } + self.integrity = Some(verifier); + self.skip_verify = false; + Ok(()) + } + #[cfg(test)] pub(crate) fn inner_ref(&self) -> &R { &self.inner @@ -152,6 +172,9 @@ where let need = self.hash_algo.size() + want; self.read_scratch_block(need, want).await?; let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?; + if let Some(integrity) = &mut self.integrity { + integrity.verify(data).await?; + } out.copy_from_slice(data); self.last_verify_duration = verify; Ok(want) @@ -263,6 +286,18 @@ where /// /// On return `out.len()` has grown by exactly the returned count. pub async fn read_appending(&mut self, out: &mut Vec, want: usize) -> std::io::Result { + let start = out.len(); + let count = self.read_appending_frame(out, want).await?; + if let Some(integrity) = &mut self.integrity + && let Err(error) = integrity.verify(&out[start..]).await + { + out.truncate(start); + return Err(error); + } + Ok(count) + } + + async fn read_appending_frame(&mut self, out: &mut Vec, want: usize) -> std::io::Result { use bytes::BufMut as _; use tokio::io::AsyncReadExt as _; @@ -768,6 +803,7 @@ impl AsyncWrite for CustomWriter { pub struct BitrotWriterWrapper { bitrot_writer: BitrotWriter, writer_type: WriterType, + integrity: Option, } /// Enum to track the type of writer we're using @@ -801,14 +837,22 @@ impl BitrotWriterWrapper { Self { bitrot_writer: BitrotWriter::new(writer, shard_size, checksum_algo), writer_type, + integrity: None, } } /// Write data to the bitrot writer pub async fn write(&mut self, buf: &[u8]) -> std::io::Result { + if let Some(integrity) = &mut self.integrity { + integrity.verify(buf).await?; + } self.bitrot_writer.write(buf).await } + pub(crate) fn set_integrity(&mut self, verifier: crate::io_support::shard_integrity::ShardVerifier) { + self.integrity = Some(verifier); + } + pub async fn shutdown(&mut self) -> std::io::Result<()> { self.bitrot_writer.shutdown().await } diff --git a/crates/ecstore/src/erasure/coding/decode.rs b/crates/ecstore/src/erasure/coding/decode.rs index b0c67c62e..8158a9486 100644 --- a/crates/ecstore/src/erasure/coding/decode.rs +++ b/crates/ecstore/src/erasure/coding/decode.rs @@ -591,6 +591,7 @@ pub(crate) struct ParallelReader { // never consumes the unopened reader reserved for a later stripe. deferred_reopeners: Vec>>, stripe_index: usize, + integrity: Option>, } } @@ -782,6 +783,7 @@ where .map(|index| !demand_bound_lockstep || index < e.data_shards) .collect(); ParallelReader { + integrity: readers.iter().flatten().find_map(BitrotReader::integrity_proof), readers, offset, shard_size, @@ -1998,11 +2000,28 @@ where R: crate::erasure::coding::ShardSource, { async fn read_next_stripe(&mut self) -> Box { + let stripe = self.offset.checked_div(self.shard_size); let mut state = self .stripe_state .take() .unwrap_or_else(|| Box::new(StripeReadState::with_slot_count(self.readers.len(), self.data_shards))); self.read_into_state(&mut state).await; + if state.can_decode() + && let Some(proof) = &self.integrity + { + let result = match stripe { + Some(stripe) => proof.reconstruction_proof(stripe, state.shards_mut()).await, + None => Err(io::Error::new(ErrorKind::InvalidData, "invalid integrity stripe offset")), + }; + match result { + Ok(proof) => state.integrity = proof, + Err(_) => { + let (shards, errors) = state.parts_mut(); + shards.iter_mut().for_each(|shard| *shard = None); + errors.fill(Some(Error::FileCorrupt)); + } + } + } state } @@ -2336,6 +2355,8 @@ impl Erasure { ret_err: &mut Option, stage_metrics_enabled: bool, require_surplus_source: bool, + integrity: Option<&std::sync::Arc>, + request_offset: usize, ) -> StripeFlow where W: AsyncWrite + Send + Sync + Unpin, @@ -2373,6 +2394,20 @@ impl Erasure { // missing data shard and an extra source shard was available, verify // the reconstructed data against that source before streaming bytes. let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + let proof = if let Some(integrity) = integrity { + match integrity + .reconstruction_proof((request_offset + *written) / self.block_size, shards) + .await + { + Ok(proof) => proof, + Err(error) => { + *ret_err = Some(error); + return StripeFlow::Stop; + } + } + } else { + None + }; let decode_result = if require_surplus_source { self.decode_data_with_reconstruction_verification_for_lockstep(shards) } else { @@ -2397,6 +2432,12 @@ impl Erasure { } record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start); + if let Some(proof) = proof + && let Err(error) = proof.verify(shards) + { + *ret_err = Some(error); + return StripeFlow::Stop; + } let emit_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let n = match write_data_blocks(writer, shards, self.data_shards, block_offset, block_length).await { Ok(n) => { @@ -2510,6 +2551,8 @@ impl Erasure { .with_deferred_parity_handles(deferred_handles) .with_deferred_parity_reopeners(deferred_reopeners); + let integrity = reader.integrity.clone(); + let start = offset / self.block_size; let end = end_offset.saturating_sub(1) / self.block_size; @@ -2615,6 +2658,8 @@ impl Erasure { &mut ret_err, stage_metrics_enabled, require_surplus_source, + integrity.as_ref(), + offset, ); tokio::pin!(read_fut); tokio::pin!(emit_fut); @@ -2663,6 +2708,8 @@ impl Erasure { &mut ret_err, stage_metrics_enabled, reader.demand_bound_lockstep, + integrity.as_ref(), + offset, ) .await { @@ -2704,6 +2751,8 @@ impl Erasure { &mut ret_err, stage_metrics_enabled, reader.demand_bound_lockstep, + integrity.as_ref(), + offset, ) .await { diff --git a/crates/ecstore/src/erasure/coding/decode_reader.rs b/crates/ecstore/src/erasure/coding/decode_reader.rs index f8db2dccb..740f3579c 100644 --- a/crates/ecstore/src/erasure/coding/decode_reader.rs +++ b/crates/ecstore/src/erasure/coding/decode_reader.rs @@ -816,6 +816,7 @@ where }; rustfs_io_metrics::record_get_object_reconstruct_outcome(metrics_path, engine.engine_name(), reconstruct_outcome); record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_RECONSTRUCT, reconstruct_stage_start); + state.verify_reconstructed_integrity()?; if state.shards_mut().len() < engine.data_shards() { return Err(io::Error::new( diff --git a/crates/ecstore/src/erasure/coding/encode.rs b/crates/ecstore/src/erasure/coding/encode.rs index 261548c25..20bdbefa2 100644 --- a/crates/ecstore/src/erasure/coding/encode.rs +++ b/crates/ecstore/src/erasure/coding/encode.rs @@ -19,6 +19,7 @@ use crate::disk::error_reduce::{ use crate::erasure::coding::BitrotWriterWrapper; use crate::erasure::coding::Erasure; use crate::erasure::coding::erasure::EncodedBlock; +use crate::io_support::shard_integrity::{IntegrityBuilder, PreparedIntegrity}; use crate::runtime::sources as runtime_sources; use bytes::{Bytes, BytesMut}; use futures::StreamExt; @@ -43,6 +44,13 @@ const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 32; const DEFAULT_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: usize = 4; const DEFAULT_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST: bool = false; +pub(crate) enum IntegrityEncodeMode { + Inline(usize), + SingleBlock(usize), + Streaming, + Batched, +} + /// Cached value of `RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES` env var. /// Read once at first use via `OnceLock` to avoid per-encode syscall. static CACHED_MAX_INFLIGHT_BYTES: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -297,6 +305,7 @@ impl Default for WriteProgressPolicy { pub(crate) struct MultiWriter<'a> { writers: &'a mut [Option], + integrity: Option<&'a mut IntegrityBuilder>, write_quorum: usize, errs: Vec>, policy: WriteProgressPolicy, @@ -314,6 +323,7 @@ impl<'a> MultiWriter<'a> { let length = writers.len(); MultiWriter { writers, + integrity: None, write_quorum, errs: vec![None; length], policy, @@ -375,6 +385,10 @@ impl<'a> MultiWriter<'a> { } async fn write_block(&mut self, block: &EncodedBlock) -> std::io::Result<()> { + if let Some(integrity) = &mut self.integrity { + // Commit every coding index, including disks absent from the write. + integrity.push(block.shards()).await?; + } self.write_shards(block.shards()).await } @@ -508,6 +522,63 @@ impl<'a> MultiWriter<'a> { } impl Erasure { + pub(crate) async fn encode_with_shard_integrity( + self: Arc, + reader: R, + writers: &mut [Option], + quorum: usize, + part_number: usize, + mode: IntegrityEncodeMode, + protected: bool, + ) -> std::io::Result<(R, usize, Option>, Option)> + where + R: AsyncRead + Send + Sync + Unpin + 'static, + { + let mut integrity = if protected { + let layout = rustfs_filemeta::shard_integrity::IntegrityLayout::new( + self.data_shards, + self.parity_shards, + self.block_size, + self.uses_legacy_codec(), + ) + .map_err(std::io::Error::other)?; + Some(IntegrityBuilder::new(layout, part_number)?) + } else { + None + }; + let (reader, size, inline) = match mode { + IntegrityEncodeMode::Inline(hint) => { + let (reader, size, shards) = self + .encode_inline_shards_with_integrity(reader, hint, integrity.as_mut()) + .await?; + (reader, size, Some(shards)) + } + IntegrityEncodeMode::SingleBlock(hint) => { + let (reader, size) = self + .encode_small_direct(reader, writers, quorum, true, hint, integrity.as_mut()) + .await?; + (reader, size, None) + } + IntegrityEncodeMode::Streaming => { + let (reader, size) = self + .encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest(), integrity.as_mut()) + .await?; + (reader, size, None) + } + IntegrityEncodeMode::Batched => { + let (reader, size) = self + .encode_batched_with_integrity(reader, writers, quorum, integrity.as_mut()) + .await?; + (reader, size, None) + } + }; + let integrity = match integrity { + Some(integrity) => Some(integrity.finish(size).await?), + None => None, + }; + Ok((reader, size, inline, integrity)) + } + async fn encode_block(self: Arc, encode_buf: Vec, len: usize) -> std::io::Result<(EncodedBlock, Vec)> { let encode_stage_start = stage_timer_if_enabled(); let encode_once = move || { @@ -564,6 +635,7 @@ impl Erasure { quorum: usize, require_single_block: bool, size_hint: usize, + integrity: Option<&mut IntegrityBuilder>, ) -> std::io::Result<(R, usize)> where R: AsyncRead + Send + Sync + Unpin, @@ -596,6 +668,7 @@ impl Erasure { let block = self.encode_data_owned_block(buf)?; let mut mw = MultiWriter::new(writers, quorum); + mw.integrity = integrity; mw.write_block(&block).await?; mw.shutdown().await?; Ok((reader, total)) @@ -605,10 +678,23 @@ impl Erasure { /// The returned bytes are the same `[hash][shard]` representation produced /// by `BitrotWriter`, ready to be embedded in each disk's staged `xl.meta`. #[hotpath::measure(impl_type = "Erasure")] + #[cfg(test)] pub(crate) async fn encode_inline_shards_with_size_hint( + self: Arc, + reader: R, + size_hint: usize, + ) -> std::io::Result<(R, usize, Vec)> + where + R: AsyncRead + Send + Sync + Unpin, + { + self.encode_inline_shards_with_integrity(reader, size_hint, None).await + } + + async fn encode_inline_shards_with_integrity( self: Arc, mut reader: R, size_hint: usize, + integrity: Option<&mut IntegrityBuilder>, ) -> std::io::Result<(R, usize, Vec)> where R: AsyncRead + Send + Sync + Unpin, @@ -622,6 +708,9 @@ impl Erasure { } let block = self.encode_data_owned_block(buf)?; + if let Some(integrity) = integrity { + integrity.push(block.shards()).await?; + } let mut inline_shards = Vec::with_capacity(block.shards().len()); for shard in block.shards() { let hash = HashAlgorithm::HighwayHash256S.hash_encode(shard); @@ -645,7 +734,7 @@ impl Erasure { R: AsyncRead + Send + Sync + Unpin + 'static, { let use_bytesmut_ingest = use_bytesmut_ingest(); - self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest) + self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest, None) .await } @@ -659,6 +748,7 @@ impl Erasure { writers: &mut [Option], quorum: usize, use_bytesmut_ingest: bool, + integrity: Option<&mut IntegrityBuilder>, ) -> std::io::Result<(R, usize)> where R: AsyncRead + Send + Sync + Unpin + 'static, @@ -766,6 +856,7 @@ impl Erasure { })); let mut writers = MultiWriter::new(writers, quorum); + writers.integrity = integrity; let mut write_err = None; @@ -808,10 +899,23 @@ impl Erasure { #[hotpath::measure(impl_type = "Erasure")] pub async fn encode_batched( + self: Arc, + reader: R, + writers: &mut [Option], + quorum: usize, + ) -> std::io::Result<(R, usize)> + where + R: AsyncRead + Send + Sync + Unpin + 'static, + { + self.encode_batched_with_integrity(reader, writers, quorum, None).await + } + + async fn encode_batched_with_integrity( self: Arc, mut reader: R, writers: &mut [Option], quorum: usize, + integrity: Option<&mut IntegrityBuilder>, ) -> std::io::Result<(R, usize)> where R: AsyncRead + Send + Sync + Unpin + 'static, @@ -892,6 +996,7 @@ impl Erasure { })); let mut writers = MultiWriter::new(writers, quorum); + writers.integrity = integrity; let mut write_err = None; loop { @@ -946,7 +1051,8 @@ impl Erasure { R: AsyncRead + Send + Sync + Unpin, { let size_hint = self.block_size; - self.encode_small_direct(reader, writers, quorum, false, size_hint).await + self.encode_small_direct(reader, writers, quorum, false, size_hint, None) + .await } /// Size-aware inline fast path. `size_hint` only controls the bounded initial @@ -962,7 +1068,8 @@ impl Erasure { where R: AsyncRead + Send + Sync + Unpin, { - self.encode_small_direct(reader, writers, quorum, false, size_hint).await + self.encode_small_direct(reader, writers, quorum, false, size_hint, None) + .await } /// Fast path for single-block non-inline objects: avoids the producer/consumer @@ -978,7 +1085,7 @@ impl Erasure { R: AsyncRead + Send + Sync + Unpin, { let size_hint = self.block_size; - self.encode_small_direct(reader, writers, quorum, true, size_hint).await + self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await } /// Size-aware single-block fast path. `size_hint` only controls the bounded @@ -994,7 +1101,7 @@ impl Erasure { where R: AsyncRead + Send + Sync + Unpin, { - self.encode_small_direct(reader, writers, quorum, true, size_hint).await + self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await } } @@ -1381,10 +1488,10 @@ mod tests { let encode = match pipeline { EncodePipeline::Vec => { - tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await }) + tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await }) } EncodePipeline::BytesMut => { - tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await }) + tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await }) } EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }), }; @@ -1443,8 +1550,8 @@ mod tests { ))]; let result = match pipeline { - EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await, - EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await, + EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await, + EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await, EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await, }; @@ -1525,7 +1632,11 @@ mod tests { BLOCK_SIZE, ))]; let erasure_for_task = erasure.clone(); - let encode = tokio::spawn(async move { erasure_for_task.encode_with_ingest_mode(reader, &mut writers, 1, false).await }); + let encode = tokio::spawn(async move { + erasure_for_task + .encode_with_ingest_mode(reader, &mut writers, 1, false, None) + .await + }); tokio::time::timeout(Duration::from_secs(1), writer_entered) .await @@ -2081,7 +2192,7 @@ mod tests { let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE)); let reader = tokio::io::BufReader::new(Cursor::new(payload.clone())); let (_reader, written) = erasure - .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true) + .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None) .await .expect("BytesMut ingest path should encode the streaming payload"); @@ -2110,7 +2221,7 @@ mod tests { let reader = tokio::io::BufReader::new(Cursor::new(vec![0x5a; block_size * 2])); rustfs_io_metrics::set_put_stage_metrics_enabled(true); - let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await; + let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await; rustfs_io_metrics::set_put_stage_metrics_enabled(false); let (_reader, written) = result.expect("bytesmut streaming encode should complete"); @@ -2728,10 +2839,14 @@ mod tests { let (_reader, total) = match pipeline { EncodePipeline::Vec => { erasure - .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false) + .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false, None) + .await + } + EncodePipeline::BytesMut => { + erasure + .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None) .await } - EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true).await, EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, DATA_SHARDS).await, } .expect("encode should succeed"); diff --git a/crates/ecstore/src/erasure/coding/erasure.rs b/crates/ecstore/src/erasure/coding/erasure.rs index 682ca0cf9..4548816e8 100644 --- a/crates/ecstore/src/erasure/coding/erasure.rs +++ b/crates/ecstore/src/erasure/coding/erasure.rs @@ -657,6 +657,10 @@ pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize { } impl Erasure { + pub(crate) fn uses_legacy_codec(&self) -> bool { + self.uses_legacy + } + /// Create a new Erasure instance /// /// # Arguments diff --git a/crates/ecstore/src/io_support/bitrot.rs b/crates/ecstore/src/io_support/bitrot.rs index d5d5f06de..9d4615f50 100644 --- a/crates/ecstore/src/io_support/bitrot.rs +++ b/crates/ecstore/src/io_support/bitrot.rs @@ -252,6 +252,7 @@ impl DeferredObjectReader { DeferredReaderStripeHandle { state: Arc::clone(&self.state), stripe_stride, + advanced: Arc::new(std::sync::atomic::AtomicUsize::new(0)), } } } @@ -274,9 +275,14 @@ impl DeferredObjectReader { pub(crate) struct DeferredReaderStripeHandle { state: Arc>, stripe_stride: usize, + advanced: Arc, } impl DeferredReaderStripeHandle { + pub(crate) fn integrity_position(&self) -> Arc { + Arc::clone(&self.advanced) + } + /// Advance the pending source by `stripes` full stripes. /// /// Returns `false` when the reader has already been opened (or failed): @@ -297,8 +303,12 @@ impl DeferredReaderStripeHandle { let Some(offset) = source.offset.checked_add(delta) else { return false; }; + let Some(advanced) = self.advanced.load(std::sync::atomic::Ordering::Acquire).checked_add(stripes) else { + return false; + }; source.offset = offset; source.length = source.length.saturating_sub(delta); + self.advanced.store(advanced, std::sync::atomic::Ordering::Release); true } _ => false, diff --git a/crates/ecstore/src/io_support/mod.rs b/crates/ecstore/src/io_support/mod.rs index 040371161..f7411849c 100644 --- a/crates/ecstore/src/io_support/mod.rs +++ b/crates/ecstore/src/io_support/mod.rs @@ -17,3 +17,4 @@ pub(crate) mod bitrot; pub(crate) mod compress; pub(crate) mod rio; +pub(crate) mod shard_integrity; diff --git a/crates/ecstore/src/io_support/shard_integrity.rs b/crates/ecstore/src/io_support/shard_integrity.rs new file mode 100644 index 000000000..b47c5dcd7 --- /dev/null +++ b/crates/ecstore/src/io_support/shard_integrity.rs @@ -0,0 +1,861 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::disk::{DiskAPI, DiskStore}; +use bytes::Bytes; +use futures::{StreamExt, stream::FuturesUnordered}; +use rustfs_filemeta::FileInfo; +use rustfs_filemeta::shard_integrity::{ + IntegrityLayout, MAX_INLINE_PROOF_BYTES, PartIntegrity, SUFFIX_INLINE_INTEGRITY, node_digest, +}; +use rustfs_utils::http::{contains_key_str, get_consistent_str, insert_str}; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::Mutex; + +const MEMORY_LIMIT: usize = 1024 * 1024; +const COPY_CHUNK_SIZE: usize = 64 * 1024; + +fn corrupt() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, "shard integrity verification failed") +} + +/// Only encoded digest rows are retained. Payload bytes never enter the spool. +/// Large parts spill to an anonymous temporary file; cancellation drops it. +pub(crate) struct IntegrityBuilder { + part: PartIntegrity, + rows: Vec, + spill: Option, +} + +impl IntegrityBuilder { + pub(crate) fn new(layout: IntegrityLayout, number: usize) -> io::Result { + Ok(Self { + part: PartIntegrity::new(layout, number).map_err(|_| corrupt())?, + rows: Vec::new(), + spill: None, + }) + } + + pub(crate) async fn push<'a>(&mut self, shards: impl ExactSizeIterator) -> io::Result<()> { + if shards.len() != self.part.layout.shards() { + return Err(corrupt()); + } + let mut row = Vec::with_capacity(shards.len() * 32); + for (index, payload) in shards.enumerate() { + row.extend_from_slice( + &self + .part + .shard_digest(self.part.stripes, index, payload) + .map_err(|_| corrupt())?, + ); + } + self.append(&row).await?; + self.part.stripes = self.part.stripes.checked_add(1).ok_or_else(corrupt)?; + Ok(()) + } + + async fn append(&mut self, bytes: &[u8]) -> io::Result<()> { + self.rows.extend_from_slice(bytes); + if self.rows.len() >= MEMORY_LIMIT { + if self.spill.is_none() { + let file = tokio::task::spawn_blocking(tempfile::tempfile) + .await + .map_err(io::Error::other)??; + self.spill = Some(tokio::fs::File::from_std(file)); + } + if let Some(file) = &mut self.spill { + file.write_all(&self.rows).await?; + self.rows.clear(); + } + } + Ok(()) + } + + async fn into_spool(self) -> io::Result { + if let Some(mut file) = self.spill { + file.write_all(&self.rows).await?; + file.flush().await?; + Ok(Spool::File(file.into_std().await)) + } else { + Ok(Spool::Memory(self.rows)) + } + } + + pub(crate) async fn finish(mut self, size: usize) -> io::Result { + self.part.size = size.try_into().map_err(|_| corrupt())?; + self.part.validate().map_err(|_| corrupt())?; + let part = self.part.clone(); + let rows = self.into_spool().await?; + if part.stripes < 16 { + build_index(part, rows) + } else { + tokio::task::spawn_blocking(move || build_index(part, rows)) + .await + .map_err(io::Error::other)? + } + } +} + +enum Spool { + Memory(Vec), + File(std::fs::File), +} + +impl Spool { + fn append(&mut self, bytes: &[u8]) -> io::Result<()> { + if let Self::Memory(buffer) = self + && buffer.len() + bytes.len() > MEMORY_LIMIT + { + let mut file = tempfile::tempfile()?; + file.write_all(buffer)?; + *self = Self::File(file); + } + match self { + Self::Memory(buffer) => buffer.extend_from_slice(bytes), + Self::File(file) => { + file.seek(SeekFrom::End(0))?; + file.write_all(bytes)?; + } + } + Ok(()) + } + + fn read_at(&mut self, offset: u64, out: &mut [u8]) -> io::Result<()> { + match self { + Self::Memory(buffer) => { + let offset = usize::try_from(offset).map_err(|_| corrupt())?; + let end = offset.checked_add(out.len()).ok_or_else(corrupt)?; + out.copy_from_slice(buffer.get(offset..end).ok_or_else(corrupt)?); + Ok(()) + } + Self::File(file) => { + file.seek(SeekFrom::Start(offset))?; + file.read_exact(out) + } + } + } +} + +fn build_index(mut part: PartIntegrity, mut rows: Spool) -> io::Result { + let leaves = u64::from(part.stripes.max(1)).next_power_of_two(); + let mut tree = Spool::Memory(Vec::new()); + let row_size = part.layout.shards() * 32; + let mut row = vec![0; row_size]; + for stripe in 0..leaves { + let stripe_number = u32::try_from(stripe).map_err(|_| corrupt())?; + let digest = if stripe < u64::from(part.stripes) { + rows.read_at(stripe * u64::try_from(row_size).map_err(|_| corrupt())?, &mut row)?; + part.leaf_digest(stripe_number, &row).map_err(|_| corrupt())? + } else { + part.padding_digest(stripe_number) + }; + tree.append(&digest)?; + } + let mut levels = vec![0u64]; + let mut count = leaves; + let mut level_offset = 0; + let mut next_offset = leaves * 32; + let mut pair = [0; 64]; + while count > 1 { + for pair_number in 0..count / 2 { + tree.read_at(level_offset + pair_number * 64, &mut pair)?; + let left = pair[..32].try_into().map_err(|_| corrupt())?; + let right = pair[32..].try_into().map_err(|_| corrupt())?; + tree.append(&node_digest(left, right))?; + } + levels.push(next_offset); + level_offset = next_offset; + count /= 2; + next_offset += count * 32; + } + let mut root = [0; 32]; + tree.read_at(level_offset, &mut root)?; + part.root = part.root_digest(&root); + let mut index = Spool::Memory(Vec::new()); + index.append(&part.index_header())?; + let mut sibling = [0; 32]; + for stripe in 0..part.stripes { + rows.read_at(u64::from(stripe) * u64::try_from(row_size).map_err(|_| corrupt())?, &mut row)?; + index.append(&row)?; + let mut position = u64::from(stripe); + for &offset in levels.iter().take(levels.len() - 1) { + tree.read_at(offset + (position ^ 1) * 32, &mut sibling)?; + index.append(&sibling)?; + position >>= 1; + } + } + Ok(PreparedIntegrity { part, index }) +} + +pub(crate) struct PreparedIntegrity { + pub(crate) part: PartIntegrity, + index: Spool, +} + +impl PreparedIntegrity { + pub(crate) async fn copy_from(proof: &PartProofReader) -> io::Result { + let mut builder = IntegrityBuilder { + part: proof.part.clone(), + rows: proof.part.index_header().to_vec(), + spill: None, + }; + for stripe in 0..proof.part.stripes { + builder.append(&proof.record(stripe).await?).await?; + } + let index = builder.into_spool().await?; + Ok(Self { + part: proof.part.clone(), + index, + }) + } + pub(crate) fn inline_bytes(&self) -> io::Result { + match &self.index { + Spool::Memory(bytes) if bytes.len() <= MAX_INLINE_PROOF_BYTES => Ok(Bytes::copy_from_slice(bytes)), + _ => Err(corrupt()), + } + } + + pub(crate) fn set_inline_metadata(&self, fi: &mut FileInfo) -> io::Result<()> { + insert_str( + &mut fi.metadata, + SUFFIX_INLINE_INTEGRITY, + base64_simd::STANDARD.encode_to_string(self.inline_bytes()?), + ); + Ok(()) + } + + /// A disk survives only if BOTH payload and index writes succeed. The caller + /// supplies the surviving payload disks and rechecks the commit quorum. + pub(crate) async fn write( + self, + disks: &mut [Option], + original_volume: &str, + volume: &str, + directory: &str, + ) -> io::Result { + let length = self.part.index_size().map_err(|_| corrupt())?; + let path = format!("{directory}/{}", self.part.file_name()); + let mut writers = futures::future::join_all(disks.iter().map(|disk| async { + match disk { + Some(disk) => disk + .create_file(original_volume, volume, &path, i64::try_from(length).map_err(|_| corrupt())?) + .await + .map(Some) + .map_err(io::Error::other), + None => Ok(None), + } + })) + .await + .into_iter() + .map(Result::ok) + .map(Option::flatten) + .collect::>(); + let mut source: Box = match self.index { + Spool::Memory(bytes) => Box::new(std::io::Cursor::new(bytes)), + Spool::File(mut file) => { + // The file has a single owner; duplicated descriptors share offsets. + file = tokio::task::spawn_blocking(move || { + file.seek(SeekFrom::Start(0))?; + io::Result::Ok(file) + }) + .await + .map_err(io::Error::other)??; + Box::new(tokio::fs::File::from_std(file)) + } + }; + let mut buffer = vec![0; COPY_CHUNK_SIZE.min(length)]; + loop { + let n = source.read(&mut buffer).await?; + if n == 0 { + break; + } + futures::future::join_all(writers.iter_mut().map(|writer| { + let bytes = &buffer[..n]; + async move { + if let Some(inner) = writer + && !matches!( + tokio::time::timeout(std::time::Duration::from_secs(60), inner.write_all(bytes)).await, + Ok(Ok(())) + ) + { + *writer = None; + } + } + })) + .await; + } + futures::future::join_all(writers.iter_mut().zip(disks.iter_mut()).map(|(writer, disk)| async move { + let success = if let Some(writer) = writer { + matches!( + tokio::time::timeout(std::time::Duration::from_secs(60), writer.shutdown()).await, + Ok(Ok(())) + ) + } else { + false + }; + if !success { + *disk = None; + } + })) + .await; + Ok(self.part) + } +} + +/// A shared, two-stripe cache coalesces the N readers' proof fetches. The only +/// async lock is held across a bounded read from at most N replicas; it never +/// nests another lock. A missing local index may use any authenticated replica. +pub(crate) struct PartProofReader { + pub(crate) part: PartIntegrity, + sources: Vec<(DiskStore, String)>, + volume: String, + inline: Option, + cached: Mutex, +} + +#[derive(Default)] +struct ProofCache { + records: std::collections::VecDeque<(u32, Bytes)>, + preferred: usize, +} + +impl PartProofReader { + pub(crate) async fn reconstruction_proof( + &self, + stripe: usize, + shards: &[Option>], + ) -> io::Result> { + let mut missing = 0u16; + for (index, shard) in shards.iter().take(usize::from(self.part.layout.data)).enumerate() { + if shard.is_none() { + missing |= 1 << index; + } + } + if missing == 0 { + return Ok(None); + } + let stripe = u32::try_from(stripe).map_err(|_| corrupt())?; + Ok(Some(ReconstructionProof { + part: self.part.clone(), + stripe, + record: self.record(stripe).await?, + missing, + })) + } + + pub(crate) fn new( + part: PartIntegrity, + files: &[FileInfo], + disks: &[Option], + volume: &str, + object: &str, + ) -> io::Result> { + part.validate().map_err(|_| corrupt())?; + let mut sources = Vec::with_capacity(disks.len()); + let mut inline = None; + for (file, disk) in files.iter().zip(disks) { + let part_index = usize::try_from(part.number) + .ok() + .and_then(|number| file.parts.binary_search_by_key(&number, |p| p.number).ok()); + if part_index.and_then(|index| file.parts[index].integrity.as_ref()) != Some(&part) { + continue; + } + if inline.is_none() && contains_key_str(&file.metadata, SUFFIX_INLINE_INTEGRITY) { + let candidate = (|| { + let value = get_consistent_str(&file.metadata, SUFFIX_INLINE_INTEGRITY)?; + if value.len() > MAX_INLINE_PROOF_BYTES.div_ceil(3) * 4 { + return None; + } + let decoded = base64_simd::STANDARD.decode_to_vec(value).ok()?; + if decoded.len() != part.index_size().ok()? || !decoded.starts_with(&part.index_header()) { + return None; + } + for stripe in 0..part.stripes { + let offset = part.record_offset(stripe).ok()?; + part.verify_record(stripe, decoded.get(offset..offset + part.record_size())?) + .ok()?; + } + Some(Bytes::from(decoded)) + })(); + if let Some(candidate) = candidate { + inline = Some(candidate); + } + } + if let (Some(disk), Some(directory)) = (disk, file.data_dir.filter(|id| !id.is_nil())) { + sources.push((disk.clone(), format!("{object}/{directory}/{}", part.file_name()))); + } + } + Ok(Arc::new(Self { + part, + sources, + volume: volume.to_owned(), + inline, + cached: Mutex::new(ProofCache::default()), + })) + } + + pub(crate) async fn record(&self, stripe: u32) -> io::Result { + let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?; + let mut cached = self.cached.lock().await; + if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) { + return Ok(bytes.clone()); + } + let size = self.part.record_size(); + let record = if let Some(inline) = &self.inline { + inline.slice(offset..offset + size) + } else { + let read = |index: usize| async move { + let (disk, path) = &self.sources[index]; + let result = tokio::time::timeout(std::time::Duration::from_secs(15), async { + let mut reader = disk + .read_file_stream(&self.volume, path, offset, size) + .await + .map_err(io::Error::other)?; + let mut bytes = vec![0; size]; + reader.read_exact(&mut bytes).await?; + self.part.verify_record(stripe, &bytes).map_err(|_| corrupt())?; + io::Result::Ok(Bytes::from(bytes)) + }) + .await; + (index, result) + }; + if self.sources.is_empty() { + return Err(corrupt()); + } + let preferred = cached.preferred % self.sources.len(); + let mut pending = FuturesUnordered::new(); + pending.push(read(preferred)); + let first = tokio::select! { + result = pending.next() => result, + () = tokio::time::sleep(std::time::Duration::from_millis(50)) => None, + }; + let mut found = match first { + Some((index, Ok(Ok(bytes)))) => Some((index, bytes)), + _ => None, + }; + if found.is_none() { + for index in 0..self.sources.len() { + if index != preferred { + pending.push(read(index)); + } + } + while let Some((index, result)) = pending.next().await { + if let Ok(Ok(bytes)) = result { + found = Some((index, bytes)); + break; + } + } + } + let (index, bytes) = found.ok_or_else(corrupt)?; + cached.preferred = index; + bytes + }; + self.part.verify_record(stripe, &record).map_err(|_| corrupt())?; + if cached.records.len() == 2 { + cached.records.pop_front(); + } + cached.records.push_back((stripe, record.clone())); + Ok(record) + } + + pub(crate) async fn verify(&self, stripe: u32, index: usize, payload: &[u8]) -> io::Result<()> { + let record = self.record(stripe).await?; + // Authentication of the shared record is cached; payload digests are + // always recomputed, including skip_verify and repaired-shard paths. + let expected = record.get(index * 32..(index + 1) * 32).ok_or_else(corrupt)?; + if self + .part + .shard_digest(stripe, index, payload) + .map_err(|_| corrupt())? + .as_slice() + != expected + { + return Err(corrupt()); + } + Ok(()) + } +} + +#[derive(Clone)] +pub(crate) struct ShardVerifier { + proof: Arc, + coding_index: usize, + first_stripe: u32, + consumed: u32, + advanced: Option>, +} + +impl ShardVerifier { + pub(crate) fn proof(&self) -> Arc { + Arc::clone(&self.proof) + } + pub(crate) fn new( + proof: Arc, + coding_index: usize, + first_stripe: usize, + advanced: Option>, + ) -> io::Result { + if coding_index >= proof.part.layout.shards() { + return Err(corrupt()); + } + Ok(Self { + proof, + coding_index, + first_stripe: first_stripe.try_into().map_err(|_| corrupt())?, + consumed: 0, + advanced, + }) + } + + pub(crate) async fn verify(&mut self, payload: &[u8]) -> io::Result<()> { + let advanced = self.advanced.as_ref().map_or(0, |value| value.load(Ordering::Acquire)); + let stripe = self + .first_stripe + .checked_add(u32::try_from(advanced).map_err(|_| corrupt())?) + .and_then(|value| value.checked_add(self.consumed)) + .ok_or_else(corrupt)?; + self.proof.verify(stripe, self.coding_index, payload).await?; + self.consumed = self.consumed.checked_add(1).ok_or_else(corrupt)?; + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ReconstructionProof { + part: PartIntegrity, + stripe: u32, + record: Bytes, + missing: u16, +} + +impl ReconstructionProof { + pub(crate) fn verify(&self, shards: &[Option>]) -> io::Result<()> { + for index in 0..usize::from(self.part.layout.data) { + if self.missing & (1 << index) == 0 { + continue; + } + let payload = shards.get(index).and_then(Option::as_deref).ok_or_else(corrupt)?; + let actual = self.part.shard_digest(self.stripe, index, payload).map_err(|_| corrupt())?; + if self.record.get(index * 32..(index + 1) * 32) != Some(actual.as_slice()) { + return Err(corrupt()); + } + } + Ok(()) + } +} + +/// Deep scans verify stripes across all disks in lockstep so a single bounded +/// proof fetch serves all N payload checks. VerifyFile from an older peer is +/// deliberately not treated as evidence of this independent commitment. +pub(crate) async fn verify_deep_parts( + files: &[FileInfo], + disks: &[Option], + expected: &FileInfo, + volume: &str, + object: &str, + statuses: &mut std::collections::HashMap>, +) -> crate::disk::error::Result<()> { + use crate::disk::error::DiskError; + use crate::disk::{CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, conv_part_err_to_int}; + use crate::io_support::bitrot::create_bitrot_reader_from_bytes; + let erasure = crate::erasure::coding::Erasure::try_new_with_options( + expected.erasure.data_blocks, + expected.erasure.parity_blocks, + expected.erasure.block_size, + expected.uses_legacy_checksum, + ) + .map_err(DiskError::from)?; + let read_timeout = crate::disk::disk_store::get_object_disk_read_timeout(); + for (part_index, part) in expected.parts.iter().enumerate() { + let commitment = part.integrity.as_ref().ok_or(DiskError::FileCorrupt)?; + let proof = PartProofReader::new(commitment.clone(), files, disks, volume, object).map_err(DiskError::from)?; + if commitment.stripes == 0 && commitment.root != commitment.root_digest(&commitment.padding_digest(0)) { + return Err(DiskError::FileCorrupt); + } + let part_status = statuses.get_mut(&part_index).ok_or(DiskError::FileCorrupt)?; + let length = erasure.shard_file_offset(0, part.size, part.size); + let mut readers = Vec::with_capacity(disks.len()); + for (index, disk) in disks.iter().enumerate() { + if part_status[index] != CHECK_PART_UNKNOWN { + readers.push(None); + continue; + } + let file = &files[index]; + if file.erasure.get_checksum_info(part.number).algorithm != rustfs_utils::HashAlgorithm::HighwayHash256S { + return Err(DiskError::BitrotHashAlgoInvalid); + } + let path = format!("{object}/{}/part.{}", file.data_dir.unwrap_or_default(), part.number); + let result = create_bitrot_reader_from_bytes( + file.data.clone(), + disk.as_ref(), + volume, + &path, + 0, + length, + erasure.shard_size(), + rustfs_utils::HashAlgorithm::HighwayHash256S, + false, + false, + ) + .await; + match result { + Ok(Some(mut reader)) => { + let coding_index = expected + .erasure + .distribution + .get(index) + .and_then(|index| index.checked_sub(1)) + .ok_or(DiskError::FileCorrupt)?; + reader + .set_integrity(ShardVerifier::new(Arc::clone(&proof), coding_index, 0, None).map_err(DiskError::from)?) + .map_err(DiskError::from)?; + readers.push(Some(reader)); + part_status[index] = CHECK_PART_SUCCESS; + } + Ok(None) => { + readers.push(None); + part_status[index] = CHECK_PART_FILE_CORRUPT; + } + Err(error) => { + readers.push(None); + part_status[index] = conv_part_err_to_int(&Some(error)); + } + } + } + let mut remaining = length; + let mut buffers = vec![vec![0; erasure.shard_size().min(length)]; disks.len()]; + while remaining > 0 { + let want = remaining.min(erasure.shard_size()); + let results = futures::future::join_all(readers.iter_mut().zip(&mut buffers).map(|(reader, buffer)| async move { + let Some(reader) = reader else { return Ok(()) }; + let read = reader.read(&mut buffer[..want]); + if read_timeout.is_zero() { + read.await.map(|_| ()) + } else { + tokio::time::timeout(read_timeout, read) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "integrity scan timed out"))? + .map(|_| ()) + } + })) + .await; + for (index, result) in results.into_iter().enumerate() { + if result.is_err() { + part_status[index] = CHECK_PART_FILE_CORRUPT; + readers[index] = None; + } + } + remaining -= want; + } + } + Ok(()) +} + +async fn verify_index_file(disk: &DiskStore, volume: &str, path: &str, part: &PartIntegrity) -> io::Result<()> { + let size = part.index_size().map_err(|_| corrupt())?; + let source = disk.read_file_stream(volume, path, 0, size).await.map_err(io::Error::other)?; + let mut source = tokio::io::BufReader::with_capacity(COPY_CHUNK_SIZE, source); + let mut header = [0; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE]; + source.read_exact(&mut header).await?; + if header != part.index_header() { + return Err(corrupt()); + } + let mut record = vec![0; part.record_size()]; + for stripe in 0..part.stripes { + source.read_exact(&mut record).await?; + part.verify_record(stripe, &record).map_err(|_| corrupt())?; + } + Ok(()) +} + +/// Restore only untrusted/missing proof replicas, under the existing root and +/// the caller's exclusive object lock. This does not rewrite payload or xl.meta. +pub(crate) async fn restore_proof_replicas( + expected: &FileInfo, + disks: &[Option], + volume: &str, + object: &str, +) -> crate::disk::error::Result { + use crate::disk::DeleteOptions; + use crate::disk::RUSTFS_META_TMP_BUCKET; + use crate::disk::error::DiskError; + if contains_key_str(&expected.metadata, SUFFIX_INLINE_INTEGRITY) || expected.deleted || expected.is_remote() { + return Ok(0); + } + let directory = expected.data_dir.filter(|id| !id.is_nil()).ok_or(DiskError::FileCorrupt)?; + let mut repaired = 0; + for part in &expected.parts { + let part = part.integrity.as_ref().ok_or(DiskError::FileCorrupt)?; + let path = format!("{object}/{directory}/{}", part.file_name()); + let checks = futures::future::join_all(disks.iter().map(|disk| async { + let Some(disk) = disk else { return true }; + matches!( + tokio::time::timeout(std::time::Duration::from_secs(60), verify_index_file(disk, volume, &path, part)).await, + Ok(Ok(())) + ) + })) + .await; + if checks.iter().all(|valid| *valid) { + continue; + } + let proof = PartProofReader { + part: part.clone(), + sources: disks.iter().flatten().map(|disk| (disk.clone(), path.clone())).collect(), + volume: volume.to_owned(), + inline: None, + cached: Mutex::new(ProofCache::default()), + }; + let prepared = PreparedIntegrity::copy_from(&proof).await.map_err(DiskError::from)?; + let temporary = format!("integrity-{}", uuid::Uuid::new_v4()); + let mut targets: Vec<_> = disks + .iter() + .zip(&checks) + .map(|(disk, valid)| if *valid { None } else { disk.clone() }) + .collect(); + let result: crate::disk::error::Result<()> = async { + prepared + .write(&mut targets, volume, RUSTFS_META_TMP_BUCKET, &temporary) + .await + .map_err(DiskError::from)?; + let source_path = format!("{temporary}/{}", part.file_name()); + for (index, valid) in checks.iter().enumerate() { + if *valid { + continue; + } + let disk = targets[index].as_ref().ok_or(DiskError::ErasureWriteQuorum)?; + disk.rename_file_durable(RUSTFS_META_TMP_BUCKET, &source_path, volume, &path) + .await?; + repaired += 1; + } + Ok(()) + } + .await; + for disk in disks.iter().flatten() { + let _ = disk + .delete( + RUSTFS_META_TMP_BUCKET, + &temporary, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await; + } + result?; + } + Ok(repaired) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn integrity_index_rejects_donor_payload_record_position_and_generation() { + let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout"); + let mut builder = IntegrityBuilder::new(layout, 1).expect("builder"); + builder.part.generation = uuid::Uuid::parse_str("00112233-4455-4677-8899-aabbccddeeff").expect("fixed generation"); + let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"]; + builder.push(shards.into_iter()).await.expect("first stripe"); + builder.push(shards.into_iter()).await.expect("second stripe"); + let prepared = builder.finish(16).await.expect("index"); + let bytes = prepared.inline_bytes().expect("inline index"); + let part = prepared.part; + // Frozen independently with Python hashlib from the v1 byte layout. + let expected_root = [ + 33, 254, 159, 226, 214, 235, 239, 175, 182, 132, 172, 81, 6, 234, 125, 187, 25, 98, 190, 6, 251, 241, 171, 33, 150, + 188, 179, 250, 26, 60, 47, 198, + ]; + assert_eq!(part.root, expected_root); + let offset = part.record_offset(0).expect("offset"); + let record = &bytes[offset..offset + part.record_size()]; + part.verify_shard(0, 0, shards[0], record).expect("target payload"); + assert!(part.verify_shard(0, 0, shards[1], record).is_err()); + assert!(part.verify_record(1, record).is_err()); + let mut other = part.clone(); + other.generation = uuid::Uuid::new_v4(); + assert!(other.verify_record(0, record).is_err()); + for position in [0, 32, record.len() - 1] { + let mut corrupt_record = record.to_vec(); + corrupt_record[position] ^= 1; + assert!(part.verify_record(0, &corrupt_record).is_err()); + } + let proof = Arc::new(PartProofReader { + part, + sources: vec![], + volume: "bucket".to_owned(), + inline: Some(bytes), + cached: Mutex::new(ProofCache::default()), + }); + let mut verifier = ShardVerifier::new(proof, 0, 0, None).expect("verifier"); + verifier.verify(shards[0]).await.expect("stripe 0"); + verifier.verify(shards[0]).await.expect("stripe 1"); + assert!(verifier.verify(shards[0]).await.is_err(), "no clean success past committed stripes"); + } + + #[tokio::test] + async fn integrity_reconstruction_rejects_wrong_decoder_output() { + let mut builder = IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 1).expect("builder"); + builder + .push([b"abcd".as_slice(), b"efgh", b"ijkl", b"mnop"].into_iter()) + .await + .expect("stripe"); + let prepared = builder.finish(8).await.expect("index"); + let index = prepared.inline_bytes().expect("bytes"); + let proof = PartProofReader { + part: prepared.part, + inline: Some(index), + sources: Vec::new(), + volume: String::new(), + cached: Mutex::new(ProofCache::default()), + }; + let mut shards = vec![None, Some(b"efgh".to_vec()), Some(b"ijkl".to_vec()), Some(b"mnop".to_vec())]; + let verification = proof + .reconstruction_proof(0, &shards) + .await + .expect("proof") + .expect("missing data"); + shards[0] = Some(b"donr".to_vec()); + assert!(verification.verify(&shards).is_err()); + shards[0] = Some(b"abcd".to_vec()); + verification.verify(&shards).expect("verified reconstruction"); + } + + #[tokio::test] + async fn integrity_builder_spills_without_changing_proofs() { + let layout = IntegrityLayout::new(12, 4, 12, false).expect("layout"); + let mut builder = IntegrityBuilder::new(layout, 7).expect("builder"); + let shards = [b"x".as_slice(); 16]; + for _ in 0..2050 { + builder.push(shards.into_iter()).await.expect("stripe"); + } + assert!(builder.spill.is_some(), "digest rows exceed memory bound"); + let mut prepared = builder.finish(2050 * 12).await.expect("spilled tree"); + assert!(matches!(prepared.index, Spool::File(_)), "materialized index is also bounded"); + let mut record = vec![0; prepared.part.record_size()]; + for stripe in [0, 1025, 2049] { + let offset = prepared.part.record_offset(stripe).expect("offset"); + prepared + .index + .read_at(u64::try_from(offset).expect("offset fits"), &mut record) + .expect("proof"); + prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof"); + } + } +} diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index de492f2bf..c4556f023 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -882,6 +882,15 @@ pub enum WriteCompletion { TailDrained, } +/// Storage-owned write mode inherited by physical rewrites. This selects the +/// destination format; the source reader must still validate every source byte. +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ShardIntegrityWriteMode { + Legacy, + Protected, +} + #[derive(Default, Clone)] pub struct ObjectOptions { // Use the maximum parity (N/2), used when saving server configuration files @@ -942,6 +951,10 @@ pub struct ObjectOptions { pub data_movement: bool, pub raw_data_movement_read: bool, + /// Internal, in-memory protection context. Never populated from S3 metadata. + /// None selects the rollout default only for a new write, not for a rewrite. + #[doc(hidden)] + pub shard_integrity_write_mode: Option, /// Durable reservation identity carried only by decommission writes. Other /// data-movement users, including rebalance, leave it unset. Keep this /// context boxed because `ObjectOptions` is passed by value through deep @@ -1043,6 +1056,25 @@ pub enum ReplicationStatusWritebackMode { } impl ObjectOptions { + pub(crate) fn shard_integrity_write_enabled(&self) -> bool { + match self.shard_integrity_write_mode { + Some(ShardIntegrityWriteMode::Protected) => true, + Some(ShardIntegrityWriteMode::Legacy) => false, + None if self.data_movement => false, + None => { + rustfs_utils::get_env_bool(rustfs_config::ENV_SHARD_INTEGRITY_WRITE, rustfs_config::DEFAULT_SHARD_INTEGRITY_WRITE) + && rustfs_utils::get_env_bool( + rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, + rustfs_config::DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED, + ) + } + } + } + + pub(crate) fn inherit_shard_integrity(&mut self, source: &ObjectInfo) { + self.shard_integrity_write_mode = Some(source.shard_integrity_write_mode()); + } + pub(crate) fn with_capacity_expected_data_bytes(expected_data_bytes: Option) -> Self { Self { decommission_capacity: expected_data_bytes.map(|expected_data_bytes| { @@ -1438,6 +1470,23 @@ impl Clone for ObjectInfo { } impl ObjectInfo { + pub(crate) fn shard_integrity_write_mode(&self) -> ShardIntegrityWriteMode { + // Any declaration requires protection. Malformed declarations remain + // errors in the source reader and must never select the legacy path. + if self.parts.iter().any(|part| part.integrity.is_some()) + || [ + rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY, + rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY, + ] + .iter() + .any(|suffix| rustfs_utils::http::contains_key_str(&self.user_defined, suffix)) + { + ShardIntegrityWriteMode::Protected + } else { + ShardIntegrityWriteMode::Legacy + } + } + /// Capture the source mutation snapshot used by replication workers when /// publishing terminal status. The semantic fingerprint is recomputed at /// the storage CAS boundary, so an older writer that preserves an unknown @@ -1840,6 +1889,7 @@ impl ObjectInfo { checksums: part.checksums.clone(), number: part.number, error: part.error.clone(), + integrity: part.integrity.clone(), }) .collect::>(); diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 9f12d53d8..91a66a6c3 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -924,7 +924,7 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses( responses: &[Option>], read_quorum: usize, ) -> disk::error::Result { - let mut part_quorum: HashMap<(&str, usize, usize, i64), (usize, &ObjectPartInfo)> = HashMap::new(); + let mut part_quorum = HashMap::new(); let mut present_count = 0usize; let mut missing_count = 0usize; let mut transient_error_count = 0usize; @@ -943,7 +943,7 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses( if !parts[part_idx].etag.is_empty() { present_count += 1; let part = &parts[part_idx]; - let key = (part.etag.as_str(), part.number, part.size, part.actual_size); + let key = (part.etag.as_str(), part.number, part.size, part.actual_size, part.integrity.as_ref()); let (count, _) = part_quorum.entry(key).or_insert((0, part)); *count += 1; continue; @@ -1420,6 +1420,40 @@ pub(in crate::set_disk) fn get_bitrot_reader_setup_strategy( } impl BitrotReaderSetup { + pub(in crate::set_disk) fn bind_integrity( + &mut self, + expected: Option<&rustfs_filemeta::shard_integrity::PartIntegrity>, + files: &[FileInfo], + disks: &[Option], + bucket: &str, + object: &str, + first_stripe: usize, + ) -> std::io::Result<()> { + use crate::io_support::shard_integrity::{PartProofReader, ShardVerifier}; + let Some(expected) = expected else { return Ok(()) }; + let proof = PartProofReader::new(expected.clone(), files, disks, bucket, object)?; + for (index, reader) in self.readers.iter_mut().enumerate() { + if let Some(reader) = reader { + let advanced = self.deferred_stripe_handles[index] + .as_ref() + .map(DeferredReaderStripeHandle::integrity_position); + reader.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, first_stripe, advanced)?)?; + } + if let Some(reopen) = self.deferred_reopeners[index].take() { + let proof = Arc::clone(&proof); + self.deferred_reopeners[index] = Some(Arc::new(move |stripe| { + let mut reader = reopen(stripe)?; + let first = first_stripe.checked_add(stripe)?; + reader + .set_integrity(ShardVerifier::new(Arc::clone(&proof), index, first, None).ok()?) + .ok()?; + Some(reader) + })); + } + } + Ok(()) + } + pub(in crate::set_disk) fn new(shards: usize) -> Self { Self { readers: (0..shards).map(|_| None).collect(), @@ -5698,6 +5732,8 @@ impl SetDisks { quorum_context: Option>, ) -> disk::error::Result>> { self.recover_part_transaction(dst_object, write_quorum).await?; + let part = ObjectPartInfo::unmarshal(&meta)?; + let integrity = part.integrity.map(Arc::new); let src_bucket = Arc::new(src_bucket.to_string()); let src_object = Arc::new(src_object.to_string()); @@ -5711,12 +5747,39 @@ impl SetDisks { let dst_bucket = dst_bucket.clone(); let dst_object = dst_object.clone(); let meta = meta.clone(); + let integrity = integrity.clone(); async move { let disk = disk?; - Some( - disk.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta) - .await, - ) + let prepared = disk + .prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta) + .await; + if let Err(error) = prepared { + return Some(Err(error)); + } + if let Some(integrity) = integrity { + let Some((directory, _)) = dst_object.rsplit_once('/') else { + return Some(Err(DiskError::FileCorrupt)); + }; + let path = format!("{directory}/{}", integrity.file_name()); + // Older peers may return Ok from PreparePart without moving + // the index. They must not enter the protected write quorum. + let result = async { + let mut reader = disk + .read_file_stream(&dst_bucket, &path, 0, rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE) + .await?; + let mut header = [0; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE]; + tokio::io::AsyncReadExt::read_exact(&mut reader, &mut header) + .await + .map_err(DiskError::from)?; + if header != integrity.index_header() { + return Err(DiskError::FileCorrupt); + } + Ok(()) + } + .await; + return Some(result); + } + Some(Ok(())) } }); let prepare_results = join_all(prepare_tasks).await; diff --git a/crates/ecstore/src/set_disk/metadata.rs b/crates/ecstore/src/set_disk/metadata.rs index 60ad041da..54d398a16 100644 --- a/crates/ecstore/src/set_disk/metadata.rs +++ b/crates/ecstore/src/set_disk/metadata.rs @@ -488,6 +488,7 @@ impl SetDisks { } pub(crate) fn hydrate_selected_fileinfo_part_checksums(fi: &mut FileInfo) -> disk::error::Result<()> { + fi.hydrate_shard_integrity().map_err(DiskError::from)?; fi.hydrate_data_movement_part_checksums().map_err(DiskError::from)?; for part in &fi.parts { let Some(checksums) = part.checksums.as_ref() else { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index fa46f6110..1b5c6b0e1 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -6426,6 +6426,24 @@ async fn disks_with_all_parts( } } + if scan_mode == HealScanMode::Deep + && !latest_meta.deleted + && !latest_meta.is_remote() + && latest_meta.parts.iter().any(|part| part.integrity.is_some()) + { + crate::io_support::shard_integrity::verify_deep_parts( + parts_metadata, + online_disks, + latest_meta, + bucket, + object, + &mut data_errs_by_part, + ) + .await?; + populate_data_errs_by_disk(&mut data_errs_by_disk, &data_errs_by_part); + return Ok((data_errs_by_disk, data_errs_by_part)); + } + // Check data for each disk for (index, disk) in online_disks.iter().enumerate() { if meta_errs[index].is_some() { @@ -6858,6 +6876,7 @@ fn completed_multipart_object_part(part_num: usize, ext_part: &ObjectPartInfo) - actual_size: ext_part.actual_size, index: ext_part.index.clone(), checksums: ext_part.checksums.clone(), + integrity: ext_part.integrity.clone(), ..Default::default() } } diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 65ae1bd36..8cdd9b05f 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -823,6 +823,13 @@ impl SetDisks { match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) { Ok(mut latest_meta) => { Self::hydrate_selected_fileinfo_part_checksums(&mut latest_meta)?; + let protected = + !latest_meta.parts.is_empty() && latest_meta.parts.iter().all(|part| part.integrity.is_some()); + result.integrity_verified = protected + && !latest_meta.deleted + && !latest_meta.is_remote() + && opts.scan_mode == HealScanMode::Deep + && !read_repair_uses_shared_lock; trace!( event = EVENT_SET_DISK_HEAL, component = LOG_COMPONENT_ECSTORE, @@ -926,7 +933,31 @@ impl SetDisks { }); } + if !latest_meta.deleted && !latest_meta.is_remote() && !protected { + result.detail = + "Legacy object uses standard repair; independent object identity remains unverified".to_owned(); + } + if disks_to_heal_count == 0 { + if result.integrity_verified && !opts.dry_run { + match crate::io_support::shard_integrity::restore_proof_replicas( + &latest_meta, + &disks, + bucket, + object, + ) + .await + { + Ok(repaired) if repaired > 0 => { + result.detail = format!("Restored {repaired} independent integrity indexes") + } + Ok(_) => {} + Err(error) => { + result.integrity_verified = false; + return Ok((result, Some(error))); + } + } + } // The object is already healthy: no disk needs healing. // This is the common case for the very objects PR #4356 // targets — a valid `xl.meta` plus a leaked pre-#3510 @@ -1215,6 +1246,15 @@ impl SetDisks { let mut writer_failure_warned = false; for (part_index, part) in latest_meta.parts.iter().enumerate() { + use crate::io_support::shard_integrity::{PartProofReader, PreparedIntegrity, ShardVerifier}; + let proof = part + .integrity + .as_ref() + .map(|part| { + PartProofReader::new(part.clone(), &parts_metadata, &latest_disks, bucket, object) + }) + .transpose() + .map_err(DiskError::from)?; let till_offset = erasure.shard_file_offset(0, part.size, part.size); let use_mmap_read = object_mmap_read_enabled(); @@ -1265,7 +1305,15 @@ impl SetDisks { ) .await { - Ok(Some(reader)) => { + Ok(Some(mut reader)) => { + if let Some(proof) = &proof { + reader + .set_integrity( + ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None) + .map_err(DiskError::from)?, + ) + .map_err(DiskError::from)?; + } readers.push(Some(reader)); } Ok(None) => { @@ -1324,6 +1372,13 @@ impl SetDisks { continue; } }; + let mut writer = writer; + if let Some(proof) = &proof { + writer.set_integrity( + ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None) + .map_err(DiskError::from)?, + ); + } writers.push(Some(writer)); } else { writers.push(None); @@ -1363,6 +1418,26 @@ impl SetDisks { return Err(e); } // close_bitrot_writers(&mut writers).await?; + if !is_inline_buffer && let Some(proof) = &proof { + let before = out_dated_disks.iter().filter(|disk| disk.is_some()).count(); + let proof_result = async { + PreparedIntegrity::copy_from(proof) + .await? + .write( + &mut out_dated_disks, + bucket, + RUSTFS_META_TMP_BUCKET, + &format!("{tmp_id}/{dst_data_dir}"), + ) + .await + } + .await; + if let Err(error) = proof_result { + let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; + return Err(error.into()); + } + disks_to_heal_count -= before - out_dated_disks.iter().filter(|disk| disk.is_some()).count(); + } for (index, disk_op) in out_dated_disks.iter_mut().enumerate() { if disk_op.is_none() { @@ -1385,6 +1460,13 @@ impl SetDisks { part.index.clone(), part.checksums.clone(), ); + if let Some(updated) = parts_metadata[index] + .parts + .iter_mut() + .find(|updated| updated.number == part.number) + { + updated.integrity.clone_from(&part.integrity); + } if is_inline_buffer { if let Some(writer) = writers[index].take() { // if let Some(w) = writer.as_any().downcast_ref::() { @@ -1560,6 +1642,15 @@ impl SetDisks { )); } + if result.integrity_verified + && let Err(error) = + crate::io_support::shard_integrity::restore_proof_replicas(&latest_meta, &disks, bucket, object) + .await + { + result.integrity_verified = false; + return Ok((result, Some(error))); + } + // The object is healthy here; sweep any data dirs left behind // by pre-#3510 unversioned overwrites, which the dangling paths // above never touch (issues #3231, #3191). Best effort — a @@ -1678,6 +1769,17 @@ impl SetDisks { let Some(candidate) = candidates.first().copied() else { return Ok(false); }; + let protected = rustfs_filemeta::shard_integrity::descriptor_from_metadata(&candidate.metadata) + .map_err(DiskError::from)? + .is_some() + || candidate.parts.iter().any(|part| part.integrity.is_some()); + // A minority metadata copy cannot establish an independent commitment. + // Legacy recovery retains its existing bounds without certifying identity. + if protected + && (candidates.len() < candidate.erasure.data_blocks || candidate.parts.iter().any(|part| part.integrity.is_none())) + { + return Ok(false); + } let identity = Self::file_info_quorum_hash(candidate); if candidates .iter() @@ -1711,6 +1813,31 @@ impl SetDisks { return Ok(false); } + let mut verified_disks = vec![true; disks.len()]; + if protected { + let mut proof_files = parts_metadata.to_vec(); + for (index, file) in proof_files.iter_mut().enumerate() { + if matches!( + errs.get(index).and_then(Option::as_ref), + Some(DiskError::FileNotFound | DiskError::FileVersionNotFound) + ) { + *file = candidate.clone(); + file.erasure.index = candidate.erasure.distribution[index]; + } + } + let mut verified = (0..candidate.parts.len()) + .map(|part| (part, vec![crate::disk::CHECK_PART_UNKNOWN; disks.len()])) + .collect(); + crate::io_support::shard_integrity::verify_deep_parts(&proof_files, disks, candidate, bucket, object, &mut verified) + .await?; + for (index, valid) in verified_disks.iter_mut().enumerate() { + *valid = verified.values().all(|parts| parts.get(index) == Some(&CHECK_PART_SUCCESS)); + } + if verified_disks.iter().filter(|valid| **valid).count() < candidate.erasure.data_blocks { + return Ok(false); + } + } + let mut wrote = 0usize; for (index, disk) in disks.iter().enumerate() { let Some(disk) = disk else { @@ -1720,7 +1847,7 @@ impl SetDisks { errs.get(index).and_then(Option::as_ref), Some(DiskError::FileNotFound | DiskError::FileVersionNotFound) ); - if !metadata_absent { + if !metadata_absent || !verified_disks[index] { continue; } let Some(&shard_index) = candidate.erasure.distribution.get(index) else { @@ -5439,3 +5566,756 @@ mod heal_result_report_tests { } } } + +#[cfg(test)] +mod shard_integrity_rollout_tests; + +#[cfg(test)] +mod shard_identity_tests { + use crate::disk::{DiskAPI as _, ReadOptions}; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated; + use crate::storage_api_contracts::multipart::MultipartOperations as _; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use crate::storage_api_contracts::range::HTTPRangeSpec; + use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode}; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn shard_integrity_heal_rejects_complete_donor_shards_ec12_4() { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(16, 0, 4).await; + let bucket = "donor-shard-identity"; + for disk in &disks { + disk.make_volume(bucket).await.expect("create fixture bucket"); + } + let expected = vec![0x5a; 5 * 1024 * 1024 + 123]; + let donor_body = vec![0xa6; expected.len()]; + let options = ObjectOptions { + shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected), + no_lock: true, + versioned: true, + ..Default::default() + }; + for (case, (coding_indexes, replace_descriptor)) in [ + (vec![1], false), + (vec![6], false), + (vec![12], false), + (vec![13], false), + (vec![16], false), + (vec![1, 2, 3, 4], false), + (vec![1, 2, 3, 4, 5], false), + (vec![1], true), + ] + .into_iter() + .enumerate() + { + let target = format!("target-{case}"); + let donor = format!("donor-{case}"); + set.put_object(bucket, &target, &mut PutObjReader::from_vec(expected.clone()), &options) + .await + .expect("commit every target shard"); + set.put_object(bucket, &donor, &mut PutObjReader::from_vec(donor_body.clone()), &options) + .await + .expect("commit every donor shard"); + let mut target_meta = Vec::new(); + let mut donor_meta = Vec::new(); + for disk in &disks { + target_meta.push( + disk.read_version("", bucket, &target, "", &ReadOptions::default()) + .await + .expect("target metadata"), + ); + donor_meta.push( + disk.read_version("", bucket, &donor, "", &ReadOptions::default()) + .await + .expect("donor metadata"), + ); + } + assert_eq!((target_meta[0].erasure.data_blocks, target_meta[0].erasure.parity_blocks), (12, 4)); + let mut before = Vec::new(); + let mut paths = Vec::new(); + let mut meta_before = Vec::new(); + for (slot, meta) in target_meta.iter().enumerate() { + let object_path = dirs[slot].path().join(bucket).join(&target); + let path = object_path + .join(meta.data_dir.expect("external data dir").to_string()) + .join("part.1"); + before.push(tokio::fs::read(&path).await.expect("original shard")); + meta_before.push(tokio::fs::read(object_path.join("xl.meta")).await.expect("original xl.meta")); + paths.push(path); + } + for index in &coding_indexes { + let target_slot = target_meta + .iter() + .position(|m| m.erasure.index == *index) + .expect("target coding index"); + let donor_slot = donor_meta + .iter() + .position(|m| m.erasure.index == *index) + .expect("donor coding index"); + let path = dirs[donor_slot] + .path() + .join(bucket) + .join(&donor) + .join(donor_meta[donor_slot].data_dir.expect("donor data dir").to_string()) + .join("part.1"); + let replacement = tokio::fs::read(path).await.expect("complete valid donor"); + assert_eq!( + replacement.len(), + before[target_slot].len(), + "the adversarial file must have the same physical length" + ); + assert_ne!(replacement, before[target_slot]); + tokio::fs::write(&paths[target_slot], replacement) + .await + .expect("replace the complete shard without changing metadata"); + if replace_descriptor { + let mut altered = target_meta[target_slot].clone(); + let suffix = rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY; + let descriptor = rustfs_utils::http::get_consistent_str(&donor_meta[donor_slot].metadata, suffix) + .expect("donor descriptor"); + rustfs_utils::http::insert_str(&mut altered.metadata, suffix, descriptor.to_owned()); + disks[target_slot] + .write_metadata("", bucket, &target, altered) + .await + .expect("a minority descriptor must not authorize its own donor payload"); + } + } + let version = target_meta[0].version_id.expect("versioned fixture").to_string(); + if coding_indexes.len() <= 4 { + let mut reader = set + .get_object_reader(bucket, &target, None, Default::default(), &options) + .await + .expect("recoverable GET before explicit heal"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("verify and reconstruct every GET block"); + assert_eq!(body, expected, "GET must not expose donor data before explicit heal"); + } + let heal = set + .heal_object( + bucket, + &target, + &version, + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await; + if coding_indexes.len() <= 4 { + let (result, error) = heal.expect("recoverable donor mismatch"); + assert!(error.is_none(), "{error:?}"); + assert_eq!( + result.drives_healed(), + Some(coding_indexes.len()), + "every complete donor must be detected" + ); + for (slot, path) in paths.iter().enumerate() { + assert_eq!( + tokio::fs::read(path).await.expect("post-heal shard"), + before[slot], + "reconstruct exact original bytes at slot {slot}" + ); + } + for (range, expected_range) in [ + (None, expected.as_slice()), + ( + Some(HTTPRangeSpec { + start: 1024 * 1024 + 17, + end: 2 * 1024 * 1024 + 99, + is_suffix_length: false, + }), + &expected[1024 * 1024 + 17..2 * 1024 * 1024 + 100], + ), + ] { + let mut reader = set + .get_object_reader( + bucket, + &target, + range, + Default::default(), + &ObjectOptions { + version_id: Some(version.clone()), + ..options.clone() + }, + ) + .await + .expect("read repaired version"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("complete verified GET body"); + assert_eq!(body, expected_range, "full and range GET must match original bytes"); + } + } else { + assert!( + heal.as_ref().map_or(true, |(_, error)| error.is_some()), + "eleven authoritative sources must not produce a successful heal" + ); + let read = set + .get_object_reader(bucket, &target, None, Default::default(), &options) + .await; + if let Ok(mut reader) = read { + let mut body = Vec::new(); + assert!(reader.stream.read_to_end(&mut body).await.is_err(), "quorum-minus-one must fail the body"); + assert!(body.is_empty(), "unverified first-stripe bytes must not escape"); + } + for (slot, meta) in target_meta.iter().enumerate() { + if !coding_indexes.contains(&meta.erasure.index) { + assert_eq!(tokio::fs::read(&paths[slot]).await.expect("retained correct shard"), before[slot]); + } + assert_eq!( + tokio::fs::read(dirs[slot].path().join(bucket).join(&target).join("xl.meta")) + .await + .expect("retained metadata"), + meta_before[slot] + ); + } + } + } + } + + #[tokio::test] + async fn shard_integrity_inline_donor_repair_and_late_external_get() { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "bound-inline-and-late"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let options = ObjectOptions { + shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected), + no_lock: true, + ..Default::default() + }; + let read_options = ReadOptions { + read_data: true, + ..Default::default() + }; + for (size, late) in [(4113usize, false), (3 * 1024 * 1024 + 123, true)] { + let target = format!("target-{size}"); + let donor = format!("donor-{size}"); + let expected = vec![0x4d; size]; + set.put_object(bucket, &target, &mut PutObjReader::from_vec(expected.clone()), &options) + .await + .expect("target PUT"); + set.put_object(bucket, &donor, &mut PutObjReader::from_vec(vec![0x9a; size]), &options) + .await + .expect("donor PUT"); + let mut target_parts = Vec::new(); + let mut donor_parts = Vec::new(); + for disk in &disks { + target_parts.push( + disk.read_version("", bucket, &target, "", &read_options) + .await + .expect("target meta"), + ); + donor_parts.push( + disk.read_version("", bucket, &donor, "", &read_options) + .await + .expect("donor meta"), + ); + } + let slot = target_parts + .iter() + .position(|fi| fi.erasure.index == 1) + .expect("target data slot"); + let donor_slot = donor_parts + .iter() + .position(|fi| fi.erasure.index == 1) + .expect("donor data slot"); + if !late { + let original = target_parts[slot].data.clone().expect("inline target"); + let donor_bytes = donor_parts[donor_slot].data.as_ref().expect("inline donor"); + assert_eq!(donor_bytes.len(), original.len()); + let path = dirs[slot].path().join(bucket).join(&target).join("xl.meta"); + let mut raw = tokio::fs::read(&path).await.expect("physical inline metadata"); + let offsets: Vec<_> = raw + .windows(original.len()) + .enumerate() + .filter_map(|(offset, bytes)| (bytes == original.as_ref()).then_some(offset)) + .collect(); + assert_eq!(offsets.len(), 1, "unique inline payload range"); + raw[offsets[0]..offsets[0] + original.len()].copy_from_slice(donor_bytes); + tokio::fs::write(path, raw) + .await + .expect("replace inline bytes with target identity retained"); + let (result, error) = set + .heal_object( + bucket, + &target, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("inline heal"); + assert!(error.is_none(), "{error:?}"); + assert_eq!(result.drives_healed(), Some(1)); + let repaired = disks[slot] + .read_version("", bucket, &target, "", &read_options) + .await + .expect("repaired inline"); + assert_eq!(repaired.data, Some(original)); + } else { + let target_path = dirs[slot] + .path() + .join(bucket) + .join(&target) + .join(target_parts[slot].data_dir.expect("target dir").to_string()) + .join("part.1"); + let donor_path = dirs[donor_slot] + .path() + .join(bucket) + .join(&donor) + .join(donor_parts[donor_slot].data_dir.expect("donor dir").to_string()) + .join("part.1"); + let mut bytes = tokio::fs::read(&target_path).await.expect("target bytes"); + let donor_bytes = tokio::fs::read(donor_path).await.expect("donor bytes"); + let frame = 32 + target_parts[slot].erasure.shard_size(); + bytes[frame..2 * frame].copy_from_slice(&donor_bytes[frame..2 * frame]); + tokio::fs::write(target_path, bytes) + .await + .expect("replace a later complete frame"); + } + let mut reader = set + .get_object_reader(bucket, &target, None, Default::default(), &options) + .await + .expect("GET after corruption"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("late corruption must reconstruct without leaking donor bytes"); + assert_eq!(body, expected); + } + } + + #[tokio::test] + async fn shard_integrity_legacy_requires_target_digest_not_only_parity_consistency() { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "legacy-shard-authority"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let options = ObjectOptions { + shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected), + no_lock: true, + ..Default::default() + }; + let expected = vec![0x4d; 1024 * 1024 + 123]; + let mut parts = Vec::new(); + let mut paths = Vec::new(); + for (object, body) in [("target", expected.clone()), ("donor", vec![0x9a; expected.len()])] { + set.put_object(bucket, object, &mut PutObjReader::from_vec(body), &options) + .await + .expect("seed legacy fixture"); + let mut object_parts = Vec::new(); + let mut object_paths = Vec::new(); + for (slot, disk) in disks.iter().enumerate() { + let mut fi = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("source meta"); + let path = dirs[slot] + .path() + .join(bucket) + .join(object) + .join(fi.data_dir.expect("data dir").to_string()) + .join("part.1"); + let encoded = tokio::fs::read(&path).await.expect("encoded source"); + let mut legacy = Vec::with_capacity(encoded.len()); + for frame in encoded.chunks(32 + fi.erasure.shard_size()) { + let payload = &frame[32..]; + legacy.extend_from_slice(rustfs_utils::HashAlgorithm::HighwayHash256S.hash_encode(payload).as_ref()); + legacy.extend_from_slice(payload); + } + rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut fi.metadata); + for part in &mut fi.parts { + part.integrity = None; + } + tokio::fs::write(&path, legacy).await.expect("legacy framing"); + disk.write_metadata("", bucket, object, fi.clone()) + .await + .expect("legacy metadata"); + object_parts.push(fi); + object_paths.push(path); + } + parts.push(object_parts); + paths.push(object_paths); + } + let mut reader = set + .get_object_reader(bucket, "target", None, Default::default(), &options) + .await + .expect("healthy legacy object remains readable"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("legacy body"); + assert_eq!(body, expected); + for (slot, fi) in parts[0].iter().enumerate() { + let source = parts[1] + .iter() + .position(|donor| donor.erasure.index == fi.erasure.index) + .expect("coding index"); + tokio::fs::copy(&paths[1][source], &paths[0][slot]) + .await + .expect("install a complete parity-consistent foreign codeword"); + } + // Compatibility keeps legacy GET semantics. A complete foreign codeword + // remains an explicit residual risk until a trusted source rewrites it. + let mut legacy = set + .get_object_reader(bucket, "target", None, Default::default(), &options) + .await + .expect("legacy GET remains available"); + let mut foreign = Vec::new(); + legacy + .stream + .read_to_end(&mut foreign) + .await + .expect("legacy donor is internally consistent"); + assert_ne!(foreign, expected); + for scan_mode in [HealScanMode::Normal, HealScanMode::Deep] { + let heal = set + .heal_object( + bucket, + "target", + "", + &HealOpts { + no_lock: true, + scan_mode, + ..Default::default() + }, + ) + .await; + let (result, error) = heal.expect("legacy scan"); + assert!(error.is_none(), "legacy scan remains observable"); + assert!(!result.integrity_verified, "a foreign codeword must not be certified"); + assert_eq!(result.drives_healed(), Some(0), "legacy scan must not rewrite data"); + } + } + + #[tokio::test] + async fn shard_integrity_multipart_replacement_and_metadata_copy_preserve_identity() { + use crate::set_disk::{CompletePart, RUSTFS_META_MULTIPART_BUCKET, SetDisks}; + use rustfs_filemeta::ObjectPartInfo; + + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "bound-multipart-copy"; + let object = "target"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let options = ObjectOptions { + shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected), + no_lock: true, + versioned: true, + ..Default::default() + }; + let upload = set.new_multipart_upload(bucket, object, &options).await.expect("new upload"); + let (upload_meta, _) = set + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .expect("upload metadata"); + let upload_path = SetDisks::get_multipart_upload_dir(bucket, object, &upload.upload_id, false); + let part_path = dirs[0] + .path() + .join(RUSTFS_META_MULTIPART_BUCKET) + .join(upload_path) + .join(upload_meta.data_dir.expect("staging directory").to_string()) + .join("part.2"); + let first = vec![0x36; 5 * 1024 * 1024]; + let second = vec![0x69; 1024 * 1024 + 123]; + let p1 = set + .put_object_part(bucket, object, &upload.upload_id, 1, &mut PutObjReader::from_vec(first.clone()), &options) + .await + .expect("first part"); + set.put_object_part( + bucket, + object, + &upload.upload_id, + 2, + &mut PutObjReader::from_vec(vec![0xa5; second.len()]), + &options, + ) + .await + .expect("original second part"); + let old_frame = tokio::fs::read(&part_path).await.expect("old part shard"); + let old_meta: ObjectPartInfo = rmp_serde::from_slice( + &tokio::fs::read(part_path.with_extension("2.meta")) + .await + .expect("old part descriptor"), + ) + .expect("decode old part descriptor"); + let p2 = set + .put_object_part( + bucket, + object, + &upload.upload_id, + 2, + &mut PutObjReader::from_vec(second.clone()), + &options, + ) + .await + .expect("replace second part"); + let mut completed = set + .clone() + .complete_multipart_upload( + bucket, + object, + &upload.upload_id, + vec![ + CompletePart { + part_num: p1.part_num, + etag: p1.etag, + ..Default::default() + }, + CompletePart { + part_num: p2.part_num, + etag: p2.etag, + ..Default::default() + }, + ], + &options, + ) + .await + .expect("complete replacement"); + let original = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("completed descriptor"); + assert_ne!( + original.parts[1].integrity.as_ref().map(|proof| proof.generation), + old_meta.integrity.as_ref().map(|proof| proof.generation) + ); + assert_ne!( + original.parts[0].integrity.as_ref().map(|proof| proof.generation), + original.parts[1].integrity.as_ref().map(|proof| proof.generation) + ); + let final_path = dirs[0] + .path() + .join(bucket) + .join(object) + .join(original.data_dir.expect("completed directory").to_string()) + .join("part.2"); + assert_eq!(old_frame.len(), tokio::fs::metadata(&final_path).await.unwrap().len() as usize); + tokio::fs::write(final_path, old_frame) + .await + .expect("replay replaced part shard"); + let (result, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("heal replayed part"); + assert!(error.is_none(), "{error:?}"); + assert_eq!(result.drives_healed(), Some(1)); + + // REPLACE may omit every internal field; metadata-only COPY must keep + // the existing payload domain even when it creates another version. + completed.metadata_only = true; + completed.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "x-amz-meta-label".to_string(), + "replacement".to_string(), + )])); + set.copy_object(bucket, object, bucket, object, &mut completed, &options, &options) + .await + .expect("metadata replacement copy"); + let copied = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("copied descriptor"); + for part in [1, 2] { + assert_eq!(copied.parts[part - 1].integrity, original.parts[part - 1].integrity); + } + let expected = [first, second].concat(); + for version in [None, original.version_id.map(|id| id.to_string())] { + for range in [ + None, + Some(HTTPRangeSpec { + start: 5 * 1024 * 1024 - 17, + end: 5 * 1024 * 1024 + 99, + is_suffix_length: false, + }), + ] { + let expected_bytes = if range.is_some() { + &expected[5 * 1024 * 1024 - 17..5 * 1024 * 1024 + 100] + } else { + &expected[..] + }; + let mut reader = set + .get_object_reader( + bucket, + object, + range, + Default::default(), + &ObjectOptions { + version_id: version.clone(), + ..options.clone() + }, + ) + .await + .expect("multipart GET"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("verified multipart body"); + assert_eq!(actual, expected_bytes); + } + } + } + #[tokio::test] + async fn shard_integrity_index_repair_preserves_payload_and_metadata() { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "integrity-index-repair"; + let object = "target"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket"); + } + let expected = vec![0x6a; 2 * 1024 * 1024 + 123]; + let opts = ObjectOptions { + shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected), + no_lock: true, + ..Default::default() + }; + set.put_object(bucket, object, &mut PutObjReader::from_vec(expected.clone()), &opts) + .await + .expect("PUT"); + let mut snapshots = Vec::new(); + for (dir, disk) in dirs.iter().zip(&disks) { + let info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("metadata"); + let root = dir.path().join(bucket).join(object); + let data = root.join(info.data_dir.expect("data dir").to_string()); + let proof = data.join(info.parts[0].integrity.as_ref().expect("commitment").file_name()); + snapshots.push(( + data.join("part.1"), + tokio::fs::read(data.join("part.1")).await.expect("payload"), + root.join("xl.meta"), + tokio::fs::read(root.join("xl.meta")).await.expect("xl.meta"), + proof.clone(), + tokio::fs::read(proof).await.expect("index"), + )); + } + tokio::fs::remove_file(&snapshots[0].4).await.expect("missing index"); + let mut bad = snapshots[1].5.clone(); + bad[70] ^= 1; + tokio::fs::write(&snapshots[1].4, bad).await.expect("corrupt index"); + let mut reader = set + .get_object_reader(bucket, object, None, Default::default(), &opts) + .await + .expect("GET with proof fallback"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("verified fallback body"); + assert_eq!(body, expected); + let (healed, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("index repair"); + assert!(error.is_none(), "{error:?}"); + assert!(healed.integrity_verified); + for ((path, payload, meta_path, meta, proof_path, proof), (disk, dir)) in snapshots.iter().zip(disks.iter().zip(&dirs)) { + assert_eq!(tokio::fs::read(path).await.expect("payload"), *payload); + assert_eq!(tokio::fs::read(meta_path).await.expect("metadata"), *meta); + assert_eq!(tokio::fs::read(proof_path).await.expect("restored proof"), *proof); + let relative = proof_path.strip_prefix(dir.path().join(bucket)).expect("relative proof path"); + disk.delete(bucket, relative.to_str().expect("proof path"), crate::disk::DeleteOptions::default()) + .await + .expect("lose all independent proofs"); + } + // All missing proofs cannot be recreated by trusting the surviving + // payload's self-contained bitrot checksum. + if let Ok(mut reader) = set.get_object_reader(bucket, object, None, Default::default(), &opts).await { + let mut body = Vec::new(); + assert!(reader.stream.read_to_end(&mut body).await.is_err()); + } + if let Ok((result, error)) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + { + assert!(error.is_some() || !result.integrity_verified); + } + for (path, payload, meta_path, meta, proof_path, _) in snapshots { + assert_eq!(tokio::fs::read(path).await.expect("retained payload"), payload); + assert_eq!(tokio::fs::read(meta_path).await.expect("retained metadata"), meta); + assert!(!proof_path.exists(), "no proof minted from unverified payload"); + } + } + #[tokio::test] + async fn shard_integrity_empty_and_no_parity_objects_remain_usable() { + for parity in [0, 2] { + let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, parity).await; + let bucket = "integrity-empty-no-parity"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket"); + } + for size in [0, 1, 1024 * 1024 + 123] { + let object = format!("target-{size}"); + let expected = vec![0x51; size]; + let opts = ObjectOptions { + shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected), + no_lock: true, + ..Default::default() + }; + set.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &opts) + .await + .expect("PUT"); + let mut reader = set + .get_object_reader(bucket, &object, None, Default::default(), &opts) + .await + .expect("GET"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("body"); + assert_eq!(body, expected); + let (result, error) = set + .heal_object( + bucket, + &object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("Deep scan"); + assert!(error.is_none(), "{parity} {size}: {error:?}"); + assert!(result.integrity_verified); + } + } + } +} diff --git a/crates/ecstore/src/set_disk/ops/heal/shard_integrity_rollout_tests.rs b/crates/ecstore/src/set_disk/ops/heal/shard_integrity_rollout_tests.rs new file mode 100644 index 000000000..637e80a31 --- /dev/null +++ b/crates/ecstore/src/set_disk/ops/heal/shard_integrity_rollout_tests.rs @@ -0,0 +1,396 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::disk::{DiskAPI as _, ReadOptions}; +use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode}; +use crate::set_disk::CompletePart; +use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated; +use crate::storage_api_contracts::multipart::MultipartOperations as _; +use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; +use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode}; +use tokio::io::AsyncReadExt; + +fn rollout_vars(requested: bool, confirmed: bool) -> [(&'static str, Option<&'static str>); 2] { + [ + (rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some(if requested { "true" } else { "false" })), + ( + rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, + Some(if confirmed { "true" } else { "false" }), + ), + ] +} + +#[tokio::test] +async fn remote_metadata_heal_does_not_certify_unread_tier_payload() { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "remote-integrity"; + let object = "transitioned"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]), + &ObjectOptions { + no_lock: true, + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), + ..Default::default() + }, + ) + .await + .expect("protected source"); + for (disk, dir) in disks.iter().zip(&dirs) { + let mut info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("source metadata"); + assert!(info.parts[0].integrity.is_some()); + info.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_owned(); + info.transition_tier = "WARM".to_owned(); + info.transitioned_objname = "remote/transitioned".to_owned(); + let data_dir = info.data_dir.expect("external shard directory"); + disk.write_metadata("", bucket, object, info) + .await + .expect("publish remote metadata"); + tokio::fs::remove_dir_all(dir.path().join(bucket).join(object).join(data_dir.to_string())) + .await + .expect("tier transition releases local payload and proof indexes"); + } + let (item, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("remote metadata scan"); + assert!(error.is_none(), "{error:?}"); + assert_eq!(item.drives_healed(), Some(0)); + assert!(!item.integrity_verified, "local metadata presence cannot certify unread remote bytes"); +} + +#[tokio::test] +#[serial_test::serial(shard_integrity_rollout)] +async fn rollout_requires_both_flags_for_new_puts() { + let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "integrity-rollout"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + for (requested, confirmed, protected) in [ + (false, false, false), + (true, false, false), + (false, true, false), + (true, true, true), + ] { + for size in [4096, 1024 * 1024 + 19] { + let object = format!("put-{requested}-{confirmed}-{size}"); + let expected = vec![0x53; size]; + temp_env::async_with_vars(rollout_vars(requested, confirmed), async { + set.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &opts) + .await + .expect("write at rollout gate"); + }) + .await; + for disk in &disks { + let meta = disk + .read_version( + "", + bucket, + &object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("published metadata"); + assert_eq!(meta.parts[0].integrity.is_some(), protected); + assert_eq!( + rustfs_utils::http::contains_key_str( + &meta.metadata, + rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY + ), + protected, + "the public gate must control the committed extension" + ); + assert_eq!(meta.erasure.get_checksum_info(1).algorithm, rustfs_utils::HashAlgorithm::HighwayHash256S); + } + let mut reader = temp_env::async_with_vars( + rollout_vars(false, false), + set.get_object_reader(bucket, &object, None, Default::default(), &opts), + ) + .await + .expect("read after disabling new protection"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("complete exact body"); + assert_eq!(actual, expected); + } + } +} + +#[tokio::test] +async fn legacy_shard_repair_and_explicit_version_metadata_recovery_remain_available() { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "legacy-recovery"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let opts = ObjectOptions { + no_lock: true, + versioned: true, + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy), + ..Default::default() + }; + let expected = vec![0x67; 1024 * 1024 + 37]; + for case in ["missing-part", "bad-bitrot", "missing-version-metadata"] { + let object = case; + let info = set + .put_object(bucket, object, &mut PutObjReader::from_vec(expected.clone()), &opts) + .await + .expect("write legacy fixture"); + let version = info.version_id.expect("versioned fixture").to_string(); + let meta = disks[0] + .read_version("", bucket, object, &version, &ReadOptions::default()) + .await + .expect("legacy metadata"); + assert!(meta.parts.iter().all(|part| part.integrity.is_none())); + let data_dir = meta.data_dir.expect("external part directory").to_string(); + let mut original = Vec::new(); + for dir in &dirs { + original.push( + tokio::fs::read(dir.path().join(bucket).join(object).join(&data_dir).join("part.1")) + .await + .expect("original framed shard"), + ); + } + match case { + "missing-part" => { + tokio::fs::remove_file(dirs[0].path().join(bucket).join(object).join(&data_dir).join("part.1")) + .await + .expect("remove one shard"); + } + "bad-bitrot" => { + let mut damaged = original[0].clone(); + damaged[32] ^= 1; + tokio::fs::write(dirs[0].path().join(bucket).join(object).join(&data_dir).join("part.1"), damaged) + .await + .expect("corrupt one stored block"); + } + _ => { + for dir in dirs.iter().skip(1) { + tokio::fs::remove_file(dir.path().join(bucket).join(object).join("xl.meta")) + .await + .expect("retain one consistent version metadata copy"); + } + } + } + let (result, error) = set + .heal_object( + bucket, + object, + &version, + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("legacy recovery must run"); + assert!(error.is_none(), "{case}: {error:?}"); + assert!(!result.integrity_verified, "traditional recovery cannot certify original identity"); + if case != "missing-version-metadata" { + assert_eq!(result.drives_healed(), Some(1), "physical repair must remain observable"); + } + for (index, disk) in disks.iter().enumerate() { + let restored = disk + .read_version("", bucket, object, &version, &ReadOptions::default()) + .await + .expect("metadata must be restored for the explicit version"); + assert_eq!(restored.version_id, info.version_id); + assert!(restored.parts.iter().all(|part| part.integrity.is_none())); + let bytes = tokio::fs::read( + dirs[index] + .path() + .join(bucket) + .join(object) + .join(restored.data_dir.expect("restored data directory").to_string()) + .join("part.1"), + ) + .await + .expect("restored payload"); + assert_eq!(bytes, original[index], "recover the original framed bytes on every drive"); + } + let mut reader = set + .get_object_reader( + bucket, + object, + None, + Default::default(), + &ObjectOptions { + version_id: Some(version), + ..opts.clone() + }, + ) + .await + .expect("read recovered historical version"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("complete recovered body"); + assert_eq!(actual, expected); + } +} + +#[tokio::test] +#[serial_test::serial(shard_integrity_rollout)] +async fn multipart_mode_survives_rollout_switch_changes() { + let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "multipart-rollout"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + for protected in [false, true] { + let object = format!("upload-{protected}"); + let upload = + temp_env::async_with_vars(rollout_vars(protected, protected), set.new_multipart_upload(bucket, &object, &opts)) + .await + .expect("initiate upload"); + let expected = vec![0x72; 128 * 1024 + 7]; + let complete = temp_env::async_with_vars(rollout_vars(!protected, !protected), async { + let part = set + .put_object_part( + bucket, + &object, + &upload.upload_id, + 1, + &mut PutObjReader::from_vec(expected.clone()), + &opts, + ) + .await + .expect("upload after switch change"); + set.clone() + .complete_multipart_upload( + bucket, + &object, + &upload.upload_id, + vec![CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &opts, + ) + .await + .expect("complete using persisted upload mode") + }) + .await; + assert_eq!( + complete.shard_integrity_write_mode(), + if protected { + ShardIntegrityWriteMode::Protected + } else { + ShardIntegrityWriteMode::Legacy + } + ); + let meta = disks[0] + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("complete metadata"); + assert_eq!(meta.parts[0].integrity.is_some(), protected); + assert!(!rustfs_utils::http::contains_key_str( + &meta.metadata, + rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY + )); + let mut reader = set + .get_object_reader(bucket, &object, None, Default::default(), &opts) + .await + .expect("completed upload"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("full multipart body"); + assert_eq!(actual, expected); + } +} + +#[tokio::test] +#[serial_test::serial(shard_integrity_rollout)] +async fn physical_copy_inherits_source_mode_instead_of_current_switch() { + let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "rewrite-rollout"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + for protected in [false, true] { + let mode = if protected { + ShardIntegrityWriteMode::Protected + } else { + ShardIntegrityWriteMode::Legacy + }; + let object = format!("source-{protected}"); + let expected = vec![0x39; 1024 * 1024 + 11]; + let seed_opts = ObjectOptions { + no_lock: true, + shard_integrity_write_mode: Some(mode), + ..Default::default() + }; + let mut source = set + .put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &seed_opts) + .await + .expect("source fixture"); + source.metadata_only = false; + source.put_object_reader = Some(PutObjReader::from_vec(expected.clone())); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + let copied = temp_env::async_with_vars( + rollout_vars(!protected, !protected), + set.copy_object(bucket, &object, bucket, &object, &mut source, &opts, &opts), + ) + .await + .expect("materialized self-copy"); + assert_eq!( + copied.shard_integrity_write_mode(), + mode, + "a legacy rewrite is not a trusted migration and a protected rewrite cannot downgrade" + ); + let meta = disks[0] + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("rewritten metadata"); + assert_eq!(meta.parts[0].integrity.is_some(), protected); + let mut reader = set + .get_object_reader(bucket, &object, None, Default::default(), &opts) + .await + .expect("read physical rewrite"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("complete rewritten body"); + assert_eq!(actual, expected); + } +} diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 482093090..39bd5d1cc 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -1626,17 +1626,20 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { }; let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); - let (reader, w_size) = match write_path { - SmallWritePath::SingleBlockNonInline => { - Arc::clone(&erasure) - .encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint) - .await? - } - SmallWritePath::PipelineBatchedLarge => { - Arc::clone(&erasure).encode_batched(stream, &mut writers, write_quorum).await? - } - SmallWritePath::Inline | SmallWritePath::Pipeline => Arc::clone(&erasure).encode(stream, &mut writers, write_quorum).await?, + let upload_suffix = rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY; + let protected_upload = rustfs_utils::http::get_consistent_str(&fi.metadata, upload_suffix) == Some("1"); + if rustfs_utils::http::contains_key_str(&fi.metadata, upload_suffix) && !protected_upload { + return Err(DiskError::FileCorrupt.into()); + } + use crate::erasure::coding::encode::IntegrityEncodeMode; + let mode = match write_path { + SmallWritePath::SingleBlockNonInline => IntegrityEncodeMode::SingleBlock(small_size_hint), + SmallWritePath::PipelineBatchedLarge => IntegrityEncodeMode::Batched, + SmallWritePath::Inline | SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming, }; + let (reader, w_size, _, integrity) = Arc::clone(&erasure) + .encode_with_shard_integrity(stream, &mut writers, write_quorum, part_id, mode, protected_upload) + .await?; if let Some(stage_start) = encode_stage_start { rustfs_io_metrics::record_put_object_stage_duration( @@ -1674,6 +1677,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { )))?; } + let part_integrity = if let Some(integrity) = integrity { + Some(integrity.write(&mut shuffle_disks, bucket, RUSTFS_META_TMP_BUCKET, &tmp_part).await?) + } else { None }; + if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum { + return Err(Error::ErasureWriteQuorum); + } + let index_op = data .stream .try_get_index() @@ -1706,6 +1716,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { actual_size, index: index_op, checksums: if checksums.is_empty() { None } else { Some(checksums) }, + integrity: part_integrity, ..Default::default() }; @@ -2065,6 +2076,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { #[tracing::instrument(skip(self))] async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { crate::hp_guard!("SetDisks::new_multipart_upload"); + let protect_upload = opts.shard_integrity_write_enabled(); let storage_class_config = self.storage_class_config_snapshot(); let mut _object_lock_guard = None; @@ -2086,6 +2098,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let disks = disks.clone(); let mut user_defined = opts.user_defined.clone(); + rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut user_defined); + if protect_upload { + rustfs_utils::http::insert_str( + &mut user_defined, + rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY, + "1".to_owned(), + ); + } rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_PART_CHECKSUMS); if !opts.data_movement { rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD); @@ -2579,6 +2599,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { part.index.clone(), part.checksums.clone(), ); + let inserted = fi + .parts + .iter_mut() + .find(|entry| entry.number == part.number) + .ok_or(Error::FileCorrupt)?; + inserted.integrity.clone_from(&part.integrity); } let (shuffle_disks, mut parts_metadatas) = Self::shuffle_disks_and_parts_metadata_by_index(&disks, &files_metas, &fi); @@ -2949,6 +2975,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } } + if rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY) + && (rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY) + != Some("1") + || fi.parts.iter().any(|part| part.integrity.is_none())) + { + return Err(Error::PartMissingOrCorrupt); + } + rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY); + fi.persist_shard_integrity()?; + for meta in parts_metadatas.iter_mut() { if meta.has_valid_erasure_geometry() { meta.size = fi.size; @@ -5355,7 +5391,16 @@ mod tests { upload_path, upload_meta.data_dir.expect("multipart upload should have a data directory") ); - let retry_meta = Bytes::from_static(b"interrupted retry metadata"); + let retry_meta = Bytes::from( + ObjectPartInfo { + number: 1, + size: 23, + etag: "interrupted-retry".to_owned(), + ..Default::default() + } + .marshal_msg() + .expect("valid legacy retry metadata"), + ); for (index, disk) in disk_stores.iter().enumerate().take(3) { let retry_path = format!("{}/part.1", Uuid::new_v4()); @@ -5423,7 +5468,16 @@ mod tests { &src_path, RUSTFS_META_MULTIPART_BUCKET, &dst_path, - Bytes::from_static(b"retry metadata"), + Bytes::from( + ObjectPartInfo { + number: 1, + size: 9, + etag: "retry".to_owned(), + ..Default::default() + } + .marshal_msg() + .expect("valid legacy part metadata"), + ), 3, None, ) diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 4b685a4f8..e36ccdd78 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -3417,6 +3417,7 @@ impl SetDisks { opts: &ObjectOptions, mut publication_fence: Option, ) -> Result<(ObjectInfo, Option)> { + let protect_write = opts.shard_integrity_write_enabled(); if publication_fence.is_none() && opts.data_movement && rustfs_utils::http::metadata_compat::contains_key_str( @@ -3461,6 +3462,7 @@ impl SetDisks { let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?; let mut user_defined = opts.user_defined.clone(); + rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut user_defined); if let Some(eval_metadata) = &opts.eval_metadata { merge_evaluated_metadata(&mut user_defined, eval_metadata)?; } @@ -3556,7 +3558,9 @@ impl SetDisks { let put_object_size = known_put_object_storage_size(data.size()); let shard_file_size_raw = erasure.shard_file_size(put_object_size); - let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned); + let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned) + && put_object_size >= 0 + && usize::try_from(put_object_size).is_ok_and(|size| size <= erasure.block_size); let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled(); let shard_file_size = shard_file_size_raw; @@ -3677,48 +3681,16 @@ impl SetDisks { }; let encode_stage_start = collect_stage_timing.then(Instant::now); - let mut inline_shards = None; - let (reader, w_size) = match write_path { - SmallWritePath::Inline => match Arc::clone(&erasure) - .encode_inline_shards_with_size_hint(stream, small_size_hint) - .await - { - Ok((r, w, shards)) => { - inline_shards = Some(shards); - (r, w) - } - Err(e) => { - error!("encode_inline_small err {:?}", e); - return Err(e.into()); - } - }, - SmallWritePath::SingleBlockNonInline => match Arc::clone(&erasure) - .encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint) - .await - { - Ok((r, w)) => (r, w), - Err(e) => { - error!("encode_single_block_non_inline err {:?}", e); - return Err(e.into()); - } - }, - SmallWritePath::PipelineBatchedLarge => { - match Arc::clone(&erasure).encode_batched(stream, &mut writers, write_quorum).await { - Ok((r, w)) => (r, w), - Err(e) => { - error!("encode_batched err {:?}", e); - return Err(e.into()); - } - } - } - SmallWritePath::Pipeline => match Arc::clone(&erasure).encode(stream, &mut writers, write_quorum).await { - Ok((r, w)) => (r, w), - Err(e) => { - error!("encode err {:?}", e); - return Err(e.into()); - } - }, + use crate::erasure::coding::encode::IntegrityEncodeMode; + let mode = match write_path { + SmallWritePath::Inline => IntegrityEncodeMode::Inline(small_size_hint), + SmallWritePath::SingleBlockNonInline => IntegrityEncodeMode::SingleBlock(small_size_hint), + SmallWritePath::PipelineBatchedLarge => IntegrityEncodeMode::Batched, + SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming, }; + let (reader, w_size, inline_shards, integrity) = Arc::clone(&erasure) + .encode_with_shard_integrity(stream, &mut writers, write_quorum, 1, mode, protect_write) + .await?; let encode_elapsed = encode_stage_start.map(|stage_start| stage_start.elapsed()); let encode_ms = encode_elapsed.map(|elapsed| elapsed.as_millis() as u64).unwrap_or_default(); if let Some(encode_elapsed) = encode_elapsed { @@ -3820,6 +3792,35 @@ impl SetDisks { ))); } + let part_integrity = if let Some(integrity) = integrity { + Some(if is_inline_buffer { + integrity.set_inline_metadata(&mut fi)?; + integrity.part + } else { + integrity + .write( + &mut shuffle_disks, + bucket, + RUSTFS_META_TMP_BUCKET, + &format!("{tmp_dir}/{}", fi.data_dir.ok_or(Error::FileCorrupt)?), + ) + .await? + }) + } else { + None + }; + if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum { + return Err(Error::ErasureWriteQuorum); + } + if let Some(inline_proof) = + rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY) + { + insert_str( + &mut user_defined, + rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY, + inline_proof.to_owned(), + ); + } fi.metadata = user_defined; if fi.version_id.is_none_or(|id| id.is_nil()) && !opts.data_movement && expected_restore_operation_id.is_none() { // Every disk must publish the same cleanup owner alongside a @@ -3832,6 +3833,8 @@ impl SetDisks { fi.size = w_size as i64; fi.versioned = opts.versioned || opts.version_suspended; fi.add_object_part(1, etag, w_size, mod_time, actual_size, index_op, None); + fi.parts[0].integrity = part_integrity; + fi.persist_shard_integrity()?; if opts.data_movement { fi.set_data_moved(); } @@ -7446,7 +7449,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // Self-copy with a data reader: write tier data back locally (de-tiering). // Handles `mc cp --storage-class STANDARD obj obj` on a transitioned object. if let Some(mut put_reader) = src_info.put_object_reader.take() { - return self.put_object(dst_bucket, dst_object, &mut put_reader, dst_opts).await; + let mut put_opts = dst_opts.clone(); + put_opts.inherit_shard_integrity(src_info); + return self.put_object(dst_bucket, dst_object, &mut put_reader, &put_opts).await; } // Same-key tiered copy without a pre-fetched reader: fall through to the metadata // path so the caller gets a disk/quorum error rather than NotImplemented. @@ -7613,6 +7618,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { None }; let mut replacement_metadata = (*src_info.user_defined).clone(); + rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut replacement_metadata); + for suffix in [ + rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY, + rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY, + ] { + if rustfs_utils::http::contains_key_str(&fi.metadata, suffix) { + let value = rustfs_utils::http::get_consistent_str(&fi.metadata, suffix).ok_or(Error::FileCorrupt)?; + rustfs_utils::http::insert_str(&mut replacement_metadata, suffix, value.to_owned()); + } + } if let Some(part_checksums) = preserved_part_checksums { rustfs_utils::http::insert_str(&mut replacement_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums); } diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 2e8f58ead..0c1c8392c 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -725,6 +725,15 @@ impl SetDisks { skip_verify_bitrot, ) .await?; + if let Some(expected) = part.integrity.as_ref() { + use crate::io_support::shard_integrity::{PartProofReader, ShardVerifier}; + let proof = PartProofReader::new(expected.clone(), files, disks, bucket, object)?; + for (index, reader) in readers.iter_mut().enumerate() { + if let Some(reader) = reader { + reader.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, 0, None)?)?; + } + } + } let reader_setup_elapsed = reader_setup_stage_start.elapsed(); rustfs_io_metrics::record_get_object_shard_reader_setup_duration(reader_setup_elapsed.as_secs_f64()); rustfs_io_metrics::record_get_object_stage_duration_by_size( @@ -777,6 +786,7 @@ impl SetDisks { erasure.data_shards, ) .await; + reader_setup.bind_integrity(part.integrity.as_ref(), &files, &disks, bucket, object, 0)?; let reader_setup_elapsed = reader_setup_stage_start.elapsed(); rustfs_io_metrics::record_get_object_shard_reader_setup_duration(reader_setup_elapsed.as_secs_f64()); rustfs_io_metrics::record_get_object_stage_duration_by_size( @@ -978,6 +988,7 @@ impl SetDisks { let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(&disks)); let sync_spec = PartReaderSetupSpec { + integrity: fi.parts[current_part].integrity.clone(), part_number, read_offset, read_length, @@ -1032,6 +1043,7 @@ impl SetDisks { let next_size = fi.parts[next_part].size; let next_length = next_size.min(remaining_after_current); let spec = PartReaderSetupSpec { + integrity: fi.parts[next_part].integrity.clone(), part_number: next_number, read_offset: 0, read_length: erasure.shard_file_offset(0, next_length, next_size), @@ -1657,7 +1669,7 @@ impl SetDisks { }); let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(disks)); - let reader_setup = create_bitrot_readers_until_quorum_with_preference( + let mut reader_setup = create_bitrot_readers_until_quorum_with_preference( files, disks, bucket, @@ -1681,6 +1693,12 @@ impl SetDisks { }), ) .await; + let expected = fi + .parts + .iter() + .find(|part| part.number == part_number) + .and_then(|part| part.integrity.as_ref()); + reader_setup.bind_integrity(expected, files, disks, bucket, object, part_offset / erasure.block_size)?; record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_SETUP, reader_setup_stage_start); let available_shards = reader_setup.available_shards(); @@ -1773,6 +1791,7 @@ impl SetDisks { /// Per-part parameters for a multipart bitrot reader setup. struct PartReaderSetupSpec { + integrity: Option, part_number: usize, read_offset: usize, read_length: usize, @@ -1914,7 +1933,7 @@ async fn setup_multipart_part_readers( metrics_size_bucket: &'static str, ) -> (BitrotReaderSetup, Duration) { let started = Instant::now(); - let setup = create_bitrot_readers_until_quorum_with_preference( + let mut setup = create_bitrot_readers_until_quorum_with_preference( files, disks, bucket, @@ -1938,6 +1957,13 @@ async fn setup_multipart_part_readers( }), ) .await; + if setup + .bind_integrity(spec.integrity.as_ref(), files, disks, bucket, object, spec.read_offset / shard_size) + .is_err() + { + setup = BitrotReaderSetup::new(disks.len()); + setup.errors.fill(Some(DiskError::FileCorrupt)); + } (setup, started.elapsed()) } diff --git a/crates/ecstore/src/set_disk/shard_source.rs b/crates/ecstore/src/set_disk/shard_source.rs index 5dae946e4..5d6b17f61 100644 --- a/crates/ecstore/src/set_disk/shard_source.rs +++ b/crates/ecstore/src/set_disk/shard_source.rs @@ -56,6 +56,7 @@ pub(crate) struct StripeReadState { shards: ShardBuffers, errors: ShardErrors, read_quorum: usize, + pub(crate) integrity: Option, } impl StripeReadState { @@ -70,6 +71,7 @@ impl StripeReadState { shards, errors, read_quorum, + integrity: None, } } @@ -78,6 +80,7 @@ impl StripeReadState { shards: SmallVec::new(), errors: SmallVec::new(), read_quorum, + integrity: None, }; state.reset(slot_count, read_quorum); state @@ -89,6 +92,7 @@ impl StripeReadState { self.errors.clear(); self.errors.resize_with(slot_count, || None); self.read_quorum = read_quorum; + self.integrity = None; } pub(crate) fn available_shards(&self) -> usize { @@ -124,6 +128,13 @@ impl StripeReadState { &mut self.shards } + pub(crate) fn verify_reconstructed_integrity(&self) -> std::io::Result<()> { + if let Some(proof) = &self.integrity { + proof.verify(&self.shards)?; + } + Ok(()) + } + pub(crate) fn into_parts(self) -> (ShardBuffers, ShardErrors) { (self.shards, self.errors) } diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 22bb521da..bd384b804 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -4336,6 +4336,7 @@ impl ECStore { let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]); let mut dst_opts = dst_opts.clone(); + dst_opts.inherit_shard_integrity(src_info); if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() { dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?); } @@ -4455,6 +4456,7 @@ impl ECStore { let mut put_opts = ObjectOptions { user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()), versioned: dst_opts.versioned, + shard_integrity_write_mode: dst_opts.shard_integrity_write_mode, version_id: dst_opts.version_id.clone(), no_lock: dst_opts.no_lock, mod_time: dst_opts.mod_time, @@ -4493,6 +4495,7 @@ impl ECStore { let mut put_opts = ObjectOptions { user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()), versioned: dst_opts.versioned, + shard_integrity_write_mode: dst_opts.shard_integrity_write_mode, version_id: dst_opts.version_id.clone(), no_lock: dst_opts.no_lock, mod_time: dst_opts.mod_time, @@ -4541,6 +4544,7 @@ impl ECStore { let mut put_opts = ObjectOptions { user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()), versioned: dst_opts.versioned, + shard_integrity_write_mode: dst_opts.shard_integrity_write_mode, version_id: dst_opts.version_id.clone(), no_lock: dst_opts.no_lock, mod_time: dst_opts.mod_time, diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index 369978881..ef0d49c3b 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -51,6 +51,8 @@ thiserror.workspace = true s3s = { workspace = true, features = ["minio"] } regex.workspace = true arc-swap.workspace = true +sha2.workspace = true +base64-simd.workspace = true [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index 27721300d..72078f62c 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -35,7 +35,7 @@ use uuid::Uuid; pub const ERASURE_ALGORITHM: &str = "rs-vandermonde"; pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M -const MAX_ERASURE_SHARDS: usize = 16; +pub(crate) const MAX_ERASURE_SHARDS: usize = 16; const MAX_FILEINFO_PARTS: usize = 10_000; const MAX_FILEINFO_CHECKSUMS: usize = 10_000; const FILEINFO_PART_BITMAP_WORD_BITS: usize = std::mem::size_of::() * 8; @@ -57,7 +57,7 @@ const ERR_RESTORE_HDR_MALFORMED: &str = "x-amz-restore header malformed"; const RFC1123: &[FormatItem<'_>] = format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT"); -#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] +#[derive(Deserialize, Debug, PartialEq, Clone, Default)] pub struct ObjectPartInfo { pub etag: String, pub number: usize, @@ -69,6 +69,28 @@ pub struct ObjectPartInfo { // Checksums holds checksums of the part pub checksums: Option>, pub error: Option, + #[serde(default)] + pub integrity: Option, +} + +impl Serialize for ObjectPartInfo { + fn serialize(&self, serializer: S) -> std::result::Result { + // Named fields let release readers ignore optional extensions. Appending + // a ninth element to the old positional array makes those readers fail. + let mut map = serializer.serialize_map(Some(8 + usize::from(self.integrity.is_some())))?; + map.serialize_entry("etag", &self.etag)?; + map.serialize_entry("number", &self.number)?; + map.serialize_entry("size", &self.size)?; + map.serialize_entry("actual_size", &self.actual_size)?; + map.serialize_entry("mod_time", &self.mod_time)?; + map.serialize_entry("index", &self.index)?; + map.serialize_entry("checksums", &self.checksums)?; + map.serialize_entry("error", &self.error)?; + if let Some(integrity) = &self.integrity { + map.serialize_entry("integrity", integrity)?; + } + map.end() + } } impl ObjectPartInfo { @@ -639,7 +661,11 @@ impl<'de> Deserialize<'de> for FileInfo { } } - deserializer.deserialize_struct("FileInfo", FILE_INFO_FIELDS, FileInfoVisitor) + let mut file = deserializer.deserialize_struct("FileInfo", FILE_INFO_FIELDS, FileInfoVisitor)?; + if !file.parts.is_empty() { + file.hydrate_shard_integrity().map_err(de::Error::custom)?; + } + Ok(file) } } @@ -1090,6 +1116,7 @@ impl FileInfo { index, checksums, error: None, + integrity: None, }; for p in self.parts.iter_mut() { @@ -2172,6 +2199,7 @@ mod tests { index, checksums, error, + integrity: None, }) } @@ -2379,6 +2407,7 @@ mod tests { .collect(), ), error: Some("part-error".to_string()), + integrity: None, }], erasure: ErasureInfo { algorithm: "erasure-algorithm".to_string(), diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index 42aa5a74d..554b2373b 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -2034,6 +2034,7 @@ impl From for ObjectPartInfo { index: value.index, checksums: value.checksums, error: value.error, + integrity: None, } } } @@ -2711,6 +2712,33 @@ impl MetaObject { if all_parts && include_part_checksums { file_info.hydrate_data_movement_part_checksums()?; } + if !self.part_numbers.is_empty() + && rustfs_utils::http::contains_key_str(&file_info.metadata, crate::shard_integrity::SUFFIX_UPLOAD_INTEGRITY) + { + return Err(Error::FileCorrupt); + } + if let Some(commitments) = crate::shard_integrity::descriptor_from_metadata(&file_info.metadata)? { + let layout = crate::shard_integrity::IntegrityLayout::new( + self.erasure_m, + self.erasure_n, + self.erasure_block_size, + file_info.uses_legacy_checksum, + )?; + if commitments.len() != self.part_numbers.len() || commitments.len() != self.part_sizes.len() { + return Err(Error::FileCorrupt); + } + for (i, commitment) in commitments.into_iter().enumerate() { + if commitment.layout != layout + || usize::try_from(commitment.number).map_err(|_| Error::FileCorrupt)? != self.part_numbers[i] + || usize::try_from(commitment.size).map_err(|_| Error::FileCorrupt)? != self.part_sizes[i] + { + return Err(Error::FileCorrupt); + } + if all_parts { + file_info.parts[i].integrity = Some(commitment); + } + } + } Ok(file_info) } diff --git a/crates/filemeta/src/lib.rs b/crates/filemeta/src/lib.rs index 3fca95ea6..16d6d3a71 100644 --- a/crates/filemeta/src/lib.rs +++ b/crates/filemeta/src/lib.rs @@ -28,3 +28,4 @@ pub use filemeta::*; pub use filemeta_inline::*; pub use metacache::*; pub use replication::*; +pub mod shard_integrity; diff --git a/crates/filemeta/src/shard_integrity.rs b/crates/filemeta/src/shard_integrity.rs new file mode 100644 index 000000000..f7f9866ca --- /dev/null +++ b/crates/filemeta/src/shard_integrity.rs @@ -0,0 +1,517 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Optional, quorum-owned SHA-256 commitments. The existing part payload and +//! HighwayHash framing are unchanged. A proof file is only an untrusted index; +//! authority comes from the root stored in the selected object's metadata. + +use crate::{Error, FileInfo, Result}; +use rustfs_utils::http::{contains_key_str, get_consistent_str, insert_str, remove_str}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use uuid::Uuid; + +pub const SUFFIX_SHARD_INTEGRITY: &str = "shard-integrity-v1"; +pub const SUFFIX_INLINE_INTEGRITY: &str = "shard-integrity-inline-v1"; +pub const SUFFIX_UPLOAD_INTEGRITY: &str = "shard-integrity-upload-v1"; +pub const MAX_PARTS: usize = 10_000; +pub const MAX_DESCRIPTOR_BYTES: usize = 1024 * 1024; +pub const MAX_INLINE_PROOF_BYTES: usize = 64 * 1024; +pub const INDEX_HEADER_SIZE: usize = 64; +const TABLE_HEADER_SIZE: usize = 32; +const PART_RECORD_SIZE: usize = 64; +const TABLE_MAGIC: &[u8; 8] = b"RFSI\x01\0\0\0"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct IntegrityLayout { + pub data: u16, + pub parity: u16, + pub block_size: u32, + pub legacy: bool, +} + +impl IntegrityLayout { + pub fn new(data: usize, parity: usize, block_size: usize, legacy: bool) -> Result { + let layout = Self { + data: data.try_into().map_err(|_| Error::FileCorrupt)?, + parity: parity.try_into().map_err(|_| Error::FileCorrupt)?, + block_size: block_size.try_into().map_err(|_| Error::FileCorrupt)?, + legacy, + }; + layout.validate()?; + Ok(layout) + } + + pub fn validate(&self) -> Result<()> { + if self.data == 0 + || self.data < self.parity + || self.shards() > crate::fileinfo::MAX_ERASURE_SHARDS + || self.block_size == 0 + || self.block_size > 64 * 1024 * 1024 + { + return Err(Error::FileCorrupt); + } + Ok(()) + } + + pub fn shards(&self) -> usize { + usize::from(self.data) + usize::from(self.parity) + } + + fn bytes(&self) -> [u8; 9] { + let mut out = [0; 9]; + out[..2].copy_from_slice(&self.data.to_le_bytes()); + out[2..4].copy_from_slice(&self.parity.to_le_bytes()); + out[4..8].copy_from_slice(&self.block_size.to_le_bytes()); + out[8] = u8::from(self.legacy); + out + } +} + +/// One immutable UploadPart/PUT generation. A metadata-only copy retains this +/// identity; overwriting even an identical part creates a new generation. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct PartIntegrity { + pub layout: IntegrityLayout, + pub number: u32, + pub generation: Uuid, + pub size: u64, + pub stripes: u32, + pub root: [u8; 32], +} + +impl PartIntegrity { + pub fn new(layout: IntegrityLayout, number: usize) -> Result { + let part = Self { + layout, + number: number.try_into().map_err(|_| Error::FileCorrupt)?, + generation: Uuid::new_v4(), + size: 0, + stripes: 0, + root: [0; 32], + }; + part.validate()?; + Ok(part) + } + + pub fn validate(&self) -> Result<()> { + self.layout.validate()?; + if self.number == 0 + || usize::try_from(self.number).map_err(|_| Error::FileCorrupt)? > MAX_PARTS + || self.generation.is_nil() + || self.size > i64::MAX.unsigned_abs() + || self.size.div_ceil(u64::from(self.layout.block_size)) != u64::from(self.stripes) + { + return Err(Error::FileCorrupt); + } + Ok(()) + } + + pub fn file_name(&self) -> String { + format!("part.{}.integrity.{}", self.number, self.generation) + } + + pub fn height(&self) -> u32 { + u32::BITS - self.stripes.max(1).saturating_sub(1).leading_zeros() + } + + pub fn record_size(&self) -> usize { + // At most 16 digests and 32 siblings, independent of untrusted sizes. + 32 * (self.layout.shards() + usize::try_from(self.height()).unwrap_or(32)) + } + + pub fn index_size(&self) -> Result { + usize::try_from(self.stripes) + .ok() + .and_then(|n| n.checked_mul(self.record_size())) + .and_then(|n| n.checked_add(INDEX_HEADER_SIZE)) + .ok_or(Error::FileCorrupt) + } + + pub fn record_offset(&self, stripe: u32) -> Result { + if stripe >= self.stripes { + return Err(Error::FileCorrupt); + } + usize::try_from(stripe) + .ok() + .and_then(|n| n.checked_mul(self.record_size())) + .and_then(|n| n.checked_add(INDEX_HEADER_SIZE)) + .ok_or(Error::FileCorrupt) + } + + pub fn index_header(&self) -> [u8; INDEX_HEADER_SIZE] { + let mut out = [0; INDEX_HEADER_SIZE]; + out[..8].copy_from_slice(b"RFSP\x01\0\0\0"); + out[8..17].copy_from_slice(&self.layout.bytes()); + out[20..24].copy_from_slice(&self.number.to_le_bytes()); + out[24..40].copy_from_slice(self.generation.as_bytes()); + out[40..48].copy_from_slice(&self.size.to_le_bytes()); + out[48..52].copy_from_slice(&self.stripes.to_le_bytes()); + out + } + + fn hash_context(&self, domain: &[u8]) -> Sha256 { + let mut hash = Sha256::new(); + hash.update(domain); + hash.update(self.layout.bytes()); + hash.update(self.number.to_le_bytes()); + hash.update(self.generation.as_bytes()); + hash + } + + pub fn shard_digest(&self, stripe: u32, coding_index: usize, payload: &[u8]) -> Result<[u8; 32]> { + if coding_index >= self.layout.shards() { + return Err(Error::FileCorrupt); + } + let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/payload\0"); + hash.update(stripe.to_le_bytes()); + hash.update(u16::try_from(coding_index).map_err(|_| Error::FileCorrupt)?.to_le_bytes()); + hash.update(u64::try_from(payload.len()).map_err(|_| Error::FileCorrupt)?.to_le_bytes()); + hash.update(payload); + Ok(hash.finalize().into()) + } + + pub fn leaf_digest(&self, stripe: u32, digests: &[u8]) -> Result<[u8; 32]> { + if digests.len() != self.layout.shards() * 32 { + return Err(Error::FileCorrupt); + } + let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/leaf\0"); + hash.update(stripe.to_le_bytes()); + hash.update(digests); + Ok(hash.finalize().into()) + } + + pub fn padding_digest(&self, stripe: u32) -> [u8; 32] { + let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/padding\0"); + hash.update(stripe.to_le_bytes()); + hash.finalize().into() + } + + pub fn root_digest(&self, tree_root: &[u8; 32]) -> [u8; 32] { + let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/root\0"); + hash.update(self.size.to_le_bytes()); + hash.update(self.stripes.to_le_bytes()); + hash.update(tree_root); + hash.finalize().into() + } + + pub fn verify_record(&self, stripe: u32, record: &[u8]) -> Result<()> { + self.record_offset(stripe)?; + if record.len() != self.record_size() { + return Err(Error::FileCorrupt); + } + let (digests, siblings) = record.split_at(self.layout.shards() * 32); + let mut digest = self.leaf_digest(stripe, digests)?; + let mut position = stripe; + for sibling in siblings.as_chunks::<32>().0 { + digest = if position & 1 == 0 { + node_digest(&digest, sibling) + } else { + node_digest(sibling, &digest) + }; + position >>= 1; + } + if self.root_digest(&digest) != self.root { + return Err(Error::FileCorrupt); + } + Ok(()) + } + + pub fn verify_shard(&self, stripe: u32, coding_index: usize, payload: &[u8], record: &[u8]) -> Result<()> { + self.verify_record(stripe, record)?; + let digest = self.shard_digest(stripe, coding_index, payload)?; + if record.get(coding_index * 32..(coding_index + 1) * 32) != Some(digest.as_slice()) { + return Err(Error::FileCorrupt); + } + Ok(()) + } +} + +pub fn node_digest(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { + let mut hash = Sha256::new(); + hash.update(b"rustfs/shard-integrity/v1/node\0"); + hash.update(left); + hash.update(right); + hash.finalize().into() +} + +pub fn encode_descriptor(parts: &[PartIntegrity]) -> Result { + let Some(first) = parts.first() else { return Err(Error::FileCorrupt) }; + if parts.len() > MAX_PARTS { + return Err(Error::FileCorrupt); + } + let mut out = vec![0; TABLE_HEADER_SIZE]; + out[..8].copy_from_slice(TABLE_MAGIC); + out[8..17].copy_from_slice(&first.layout.bytes()); + out[28..32].copy_from_slice(&u32::try_from(parts.len()).map_err(|_| Error::FileCorrupt)?.to_le_bytes()); + let mut previous = 0; + for part in parts { + part.validate()?; + if part.layout != first.layout || part.number <= previous { + return Err(Error::FileCorrupt); + } + previous = part.number; + out.extend_from_slice(&part.number.to_le_bytes()); + out.extend_from_slice(part.generation.as_bytes()); + out.extend_from_slice(&part.size.to_le_bytes()); + out.extend_from_slice(&part.stripes.to_le_bytes()); + out.extend_from_slice(&part.root); + } + Ok(base64_simd::STANDARD.encode_to_string(out)) +} + +pub fn decode_descriptor(value: &str) -> Result> { + if value.len() > MAX_DESCRIPTOR_BYTES { + return Err(Error::FileCorrupt); + } + let bytes = base64_simd::STANDARD.decode_to_vec(value).map_err(|_| Error::FileCorrupt)?; + // Reject non-canonical encodings as well as unsupported versions/reserved bits. + if bytes.len() < TABLE_HEADER_SIZE + || &bytes[..8] != TABLE_MAGIC + || bytes[16] > 1 + || bytes[17..28].iter().any(|v| *v != 0) + || base64_simd::STANDARD.encode_to_string(&bytes) != value + { + return Err(Error::FileCorrupt); + } + let u32_at = |start| -> Result { + Ok(u32::from_le_bytes( + bytes + .get(start..start + 4) + .ok_or(Error::FileCorrupt)? + .try_into() + .map_err(|_| Error::FileCorrupt)?, + )) + }; + let count = usize::try_from(u32_at(28)?).map_err(|_| Error::FileCorrupt)?; + if count == 0 || count > MAX_PARTS || bytes.len() != TABLE_HEADER_SIZE + count * PART_RECORD_SIZE { + return Err(Error::FileCorrupt); + } + let layout = IntegrityLayout { + data: u16::from_le_bytes([bytes[8], bytes[9]]), + parity: u16::from_le_bytes([bytes[10], bytes[11]]), + block_size: u32_at(12)?, + legacy: bytes[16] == 1, + }; + layout.validate()?; + let mut parts = Vec::with_capacity(count); + let mut previous = 0; + for record in bytes[TABLE_HEADER_SIZE..].as_chunks::().0 { + let part = PartIntegrity { + layout, + number: u32::from_le_bytes(record[..4].try_into().map_err(|_| Error::FileCorrupt)?), + generation: Uuid::from_slice(&record[4..20]) + .ok() + .filter(|id| !id.is_nil()) + .ok_or(Error::FileCorrupt)?, + size: u64::from_le_bytes(record[20..28].try_into().map_err(|_| Error::FileCorrupt)?), + stripes: u32::from_le_bytes(record[28..32].try_into().map_err(|_| Error::FileCorrupt)?), + root: record[32..64].try_into().map_err(|_| Error::FileCorrupt)?, + }; + part.validate()?; + if part.number <= previous { + return Err(Error::FileCorrupt); + } + previous = part.number; + parts.push(part); + } + Ok(parts) +} + +pub fn descriptor_from_metadata(metadata: &HashMap) -> Result>> { + if !contains_key_str(metadata, SUFFIX_SHARD_INTEGRITY) { + // An orphaned inline proof must not silently turn a protected object into legacy. + if contains_key_str(metadata, SUFFIX_INLINE_INTEGRITY) { + return Err(Error::FileCorrupt); + } + return Ok(None); + } + decode_descriptor(get_consistent_str(metadata, SUFFIX_SHARD_INTEGRITY).ok_or(Error::FileCorrupt)?).map(Some) +} + +pub fn clear_integrity_metadata(metadata: &mut HashMap) { + for suffix in [SUFFIX_SHARD_INTEGRITY, SUFFIX_INLINE_INTEGRITY, SUFFIX_UPLOAD_INTEGRITY] { + remove_str(metadata, suffix); + } +} + +impl FileInfo { + /// Decode once at the metadata boundary and retain each part's validated + /// commitment. Reading part n must not reparse a 10,000-part descriptor. + pub fn hydrate_shard_integrity(&mut self) -> Result<()> { + if !self.parts.is_empty() && contains_key_str(&self.metadata, SUFFIX_UPLOAD_INTEGRITY) { + return Err(Error::FileCorrupt); + } + let Some(parts) = descriptor_from_metadata(&self.metadata)? else { + if self.parts.iter().any(|part| part.integrity.is_some()) { + return Err(Error::FileCorrupt); + } + return Ok(()); + }; + let layout = IntegrityLayout::new( + self.erasure.data_blocks, + self.erasure.parity_blocks, + self.erasure.block_size, + self.uses_legacy_checksum, + )?; + if parts.len() != self.parts.len() { + return Err(Error::FileCorrupt); + } + for (expected, actual) in parts.into_iter().zip(&mut self.parts) { + if expected.layout != layout + || usize::try_from(expected.number).map_err(|_| Error::FileCorrupt)? != actual.number + || usize::try_from(expected.size).map_err(|_| Error::FileCorrupt)? != actual.size + || actual.integrity.as_ref().is_some_and(|value| value != &expected) + { + return Err(Error::FileCorrupt); + } + actual.integrity = Some(expected); + } + Ok(()) + } + + pub fn persist_shard_integrity(&mut self) -> Result<()> { + let parts: Vec<_> = self.parts.iter().filter_map(|part| part.integrity.clone()).collect(); + if parts.is_empty() { + clear_integrity_metadata(&mut self.metadata); + return Ok(()); + } + if parts.len() != self.parts.len() { + return Err(Error::FileCorrupt); + } + insert_str(&mut self.metadata, SUFFIX_SHARD_INTEGRITY, encode_descriptor(&parts)?); + self.hydrate_shard_integrity() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ObjectPartInfo; + + fn part(number: usize) -> PartIntegrity { + let mut part = + PartIntegrity::new(IntegrityLayout::new(12, 4, 1024 * 1024, false).expect("layout"), number).expect("part"); + part.size = 1024 * 1024 + 123; + part.stripes = 2; + part.root = [7; 32]; + part + } + + #[test] + fn integrity_descriptor_is_bounded_canonical_and_complete() { + let parts: Vec<_> = (1..=MAX_PARTS).map(part).collect(); + let encoded = encode_descriptor(&parts).expect("maximum multipart descriptor"); + assert_eq!(encoded.len(), 853_376); + assert_eq!(decode_descriptor(&encoded).expect("decode maximum table"), parts); + assert!(encode_descriptor(&[part(2), part(1)]).is_err()); + assert!(encode_descriptor(&[part(1), part(1)]).is_err()); + assert!(decode_descriptor(&format!("{encoded} ")).is_err()); + assert!(decode_descriptor(&"A".repeat(MAX_DESCRIPTOR_BYTES + 1)).is_err()); + let mut bytes = base64_simd::STANDARD + .decode_to_vec(encode_descriptor(&[part(1)]).expect("encode")) + .expect("base64"); + for offset in [4, 17, 28, 36] { + let mut invalid = bytes.clone(); + if offset == 36 { + invalid[36..52].fill(0); + } else { + invalid[offset] = 255; + } + assert!( + decode_descriptor(&base64_simd::STANDARD.encode_to_string(invalid)).is_err(), + "offset {offset}" + ); + } + bytes.push(0); + assert!(decode_descriptor(&base64_simd::STANDARD.encode_to_string(bytes)).is_err()); + } + + #[test] + fn integrity_metadata_distinguishes_absence_corruption_and_part_mismatch() { + let mut file = FileInfo::new("object", 12, 4); + file.add_object_part(1, String::new(), 1024 * 1024 + 123, None, 0, None, None); + assert!(descriptor_from_metadata(&file.metadata).expect("legacy absence").is_none()); + file.parts[0].integrity = Some(part(1)); + file.persist_shard_integrity().expect("persist"); + let mut decoded = file.clone(); + decoded.parts[0].integrity = None; + decoded.hydrate_shard_integrity().expect("hydrate old peer response"); + assert_eq!(decoded.parts[0].integrity, file.parts[0].integrity); + decoded.parts[0].size += 1; + assert!(decoded.hydrate_shard_integrity().is_err()); + file.metadata + .insert(format!("x-minio-internal-{SUFFIX_SHARD_INTEGRITY}"), "invalid".to_owned()); + assert!(descriptor_from_metadata(&file.metadata).is_err()); + } + + #[test] + fn integrity_descriptor_matches_independent_v1_vector() { + // Header, UUID byte order, dimensions and root frozen independently + // with Python struct/hashlib, rather than a Rust round trip. + let encoded = "UkZTSQEAAAACAAIACAAAAAAAAAAAAAAAAAAAAAEAAAABAAAAABEiM0RVRneImaq7zN3u/xAAAAAAAAAAAgAAACH+n+LW6++vtoSsUQbqfbsZYr4G+/GrIZa8s/oaPC/G"; + let parts = decode_descriptor(encoded).expect("known descriptor"); + assert_eq!(parts.len(), 1); + let part = &parts[0]; + assert_eq!(part.layout, IntegrityLayout::new(2, 2, 8, false).expect("layout")); + assert_eq!(part.generation.to_string(), "00112233-4455-4677-8899-aabbccddeeff"); + assert_eq!((part.number, part.size, part.stripes), (1, 16, 2)); + assert_eq!( + part.root, + [ + 33, 254, 159, 226, 214, 235, 239, 175, 182, 132, 172, 81, 6, 234, 125, 187, 25, 98, 190, 6, 251, 241, 171, 33, + 150, 188, 179, 250, 26, 60, 47, 198 + ] + ); + assert_eq!(encode_descriptor(&parts).expect("encode known descriptor"), encoded); + } + + #[derive(Serialize, Deserialize)] + struct ReleasePart { + etag: String, + number: usize, + size: usize, + actual_size: i64, + mod_time: Option, + index: Option, + checksums: Option>, + error: Option, + } + + #[test] + fn integrity_part_extension_is_readable_by_release_decoder_and_reads_old_arrays() { + let old = ReleasePart { + etag: "same-etag".to_owned(), + number: 1, + size: 5, + actual_size: 5, + mod_time: None, + index: None, + checksums: None, + error: None, + }; + let tuple = rmp_serde::to_vec(&old).expect("release positional writer"); + assert_eq!(tuple[0], 0x98, "release wrote eight positional fields"); + let mut new = ObjectPartInfo::unmarshal(&tuple).expect("read release part"); + assert!(new.integrity.is_none()); + new.integrity = Some(part(1)); + let bytes = new.marshal_msg().expect("new writer"); + assert_eq!(bytes[0], 0x89, "extension uses nine named fields"); + let decoded: ReleasePart = rmp_serde::from_slice(&bytes).expect("release decoder ignores unknown named field"); + assert_eq!(decoded.etag, old.etag); + assert_eq!(decoded.size, 5); + assert_eq!(ObjectPartInfo::unmarshal(&bytes).expect("new decoder").integrity, new.integrity); + } +} diff --git a/crates/heal/src/heal/mrf_queue.rs b/crates/heal/src/heal/mrf_queue.rs index 8ea4e0efe..2dfe4e09a 100644 --- a/crates/heal/src/heal/mrf_queue.rs +++ b/crates/heal/src/heal/mrf_queue.rs @@ -456,7 +456,7 @@ fn warn_mrf_journal_write(err: &super::DiskError) { /// Translate an intent into the prioritized heal request the issue specifies: /// decode failures go Urgent ECDecode, metadata corruption goes High -/// Metadata, partial writes go Normal object heal. +/// Metadata, partial writes go Normal-priority object heal with Deep verification. pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest { let bucket = intent.bucket.to_string(); let object = intent.object.to_string(); @@ -484,6 +484,11 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest { ), }; let mut options = HealOptions::default(); + if matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::PartialWrite) { + // Presence-only repair cannot discharge a protected object's durable + // obligation. Verify payloads even after new protection is disabled. + options.scan_mode = rustfs_heal_contracts::heal_channel::HealScanMode::Deep; + } if !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) && let Some(scope) = intent.scope { @@ -2716,5 +2721,6 @@ mod tests { }); assert!(matches!(partial.heal_type, HealType::Object { .. })); assert_eq!(partial.priority, HealPriority::Normal); + assert_eq!(partial.options.scan_mode, rustfs_heal_contracts::heal_channel::HealScanMode::Deep); } } diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 559ed2e68..e02471798 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -1169,7 +1169,7 @@ impl HealStorageAPI for ECStoreHealStorage { } else { None } - } else if error.is_none() && !opts.dry_run { + } else if error.is_none() && !opts.dry_run && item.integrity_verified { let ok_drive_state = DriveState::Ok.to_string(); let all_after_drives_ok = item.after.drives.iter().all(|drive| drive.state == ok_drive_state); match ( diff --git a/crates/heal/tests/heal_b920_subquorum_union_test.rs b/crates/heal/tests/heal_b920_subquorum_union_test.rs index c3cfe3905..052de72b1 100644 --- a/crates/heal/tests/heal_b920_subquorum_union_test.rs +++ b/crates/heal/tests/heal_b920_subquorum_union_test.rs @@ -762,10 +762,12 @@ mod absence_receipt_regressions { let outcome = task.get_outcome().await; assert_eq!(outcome.counters.processed, 3); assert_eq!(outcome.counters.healed, 1, "completed cleanup must be repaired: {outcome:?}"); - assert_eq!(outcome.counters.unchanged, 2); - assert_eq!(outcome.counters.unknown, 0); + // Exact historical absence has its own proof. The two live legacy + // versions remain readable but carry no independent payload receipt. + assert_eq!(outcome.counters.unchanged, 0); + assert_eq!(outcome.counters.unknown, 2); assert_eq!(outcome.counters.failed, 0); - assert_eq!(outcome.counters.skipped, 0); + assert_eq!(outcome.counters.skipped, 2); assert_versions(&store, bucket, &old, ¤t).await; } diff --git a/crates/heal/tests/heal_truncated_xlmeta_test.rs b/crates/heal/tests/heal_truncated_xlmeta_test.rs index e3de63233..6645dfc89 100644 --- a/crates/heal/tests/heal_truncated_xlmeta_test.rs +++ b/crates/heal/tests/heal_truncated_xlmeta_test.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![recursion_limit = "256"] + use http::HeaderMap; use rustfs_heal::heal::{ outcome::{HealObjectDisposition, HealTraversalCoverage}, @@ -24,7 +26,8 @@ use tokio::io::AsyncReadExt as _; mod storage_api; use storage_api::integration::{ - DiskAPI as _, DiskError, DiskOption, Endpoint, NamespaceLocking as _, ObjectIO as _, ReadOptions, new_disk, + DiskAPI as _, DiskError, DiskOption, Endpoint, NamespaceLocking as _, ObjectIO as _, ReadOptions, ShardIntegrityWriteMode, + new_disk, }; fn deep_heal_task(storage: &Arc, bucket: &str, object: &str, dry_run: bool) -> HealTask { @@ -75,6 +78,7 @@ async fn deep_heal_truncated_xlmeta_scenario() { let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); let bucket = "truncated-xlmeta"; env.make_bucket(bucket, false).await; + env.make_bucket("truncated-xlmeta-versioned", true).await; let payload = vec![0x7b; 4 * 1024 * 1024]; let mut endpoint = Endpoint::try_from(env.disk_paths[0].to_str().expect("UTF-8 disk path")).expect("target endpoint"); endpoint.set_pool_index(0); @@ -88,13 +92,13 @@ async fn deep_heal_truncated_xlmeta_scenario() { ..Default::default() }; - for damage in ["length-prefix", "metadata-body", "crc-tail", "versioned"] { + for (damage, protected) in ["length-prefix", "metadata-body", "crc-tail", "versioned"] + .into_iter() + .flat_map(|damage| [false, true].map(|protected| (damage, protected))) + { let versioned = damage == "versioned"; let bucket = if versioned { "truncated-xlmeta-versioned" } else { bucket }; - if versioned { - env.make_bucket(bucket, true).await; - } - let object = format!("{damage}/object.bin"); + let object = format!("{damage}-{protected}/object.bin"); let mut reader = PutObjReader::from_vec(payload.clone()); env.ecstore .put_object( @@ -103,6 +107,11 @@ async fn deep_heal_truncated_xlmeta_scenario() { &mut reader, &ObjectOptions { versioned, + shard_integrity_write_mode: Some(if protected { + ShardIntegrityWriteMode::Protected + } else { + ShardIntegrityWriteMode::Legacy + }), ..Default::default() }, ) @@ -168,15 +177,26 @@ async fn deep_heal_truncated_xlmeta_scenario() { let outcome = task.get_outcome().await; assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); assert_eq!(outcome.counters.processed, 1); - assert_eq!(outcome.counters.healed, 1, "{damage}: {outcome:?}"); - assert_eq!(outcome.counters.unknown, 0); - assert_eq!(outcome.counters.skipped, 0); + // Both modes restore metadata and shards. Only independently verified + // payloads authorize a strong repair receipt. + assert_eq!(outcome.counters.healed, u64::from(protected), "{damage}: {outcome:?}"); + assert_eq!(outcome.counters.unknown, u64::from(!protected)); + assert_eq!(outcome.counters.skipped, u64::from(!protected)); assert_eq!(outcome.counters.failed, 0); assert_eq!(outcome.counters.attempt_failures, 0); - assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Repaired); + assert_eq!( + outcome.objects[0].disposition, + if protected { + HealObjectDisposition::Repaired + } else { + HealObjectDisposition::Unknown + } + ); assert_eq!(read_error, DiskError::FileCorrupt); let results = task.get_result_items().await; assert_eq!(results.len(), 1); + assert_eq!(results[0].drives_healed(), Some(1)); + assert_eq!(results[0].integrity_verified, protected); assert_eq!(results[0].before.drives.len(), 16); assert_eq!(results[0].after.drives.len(), 16); assert_eq!(results[0].before.drives[0].state, DriveState::Corrupt.to_string()); @@ -217,8 +237,8 @@ async fn deep_heal_truncated_xlmeta_scenario() { repeat.execute().await.expect("repeated heal completes"); let repeat_outcome = repeat.get_outcome().await; assert_eq!(repeat_outcome.counters.healed, 0); - assert_eq!(repeat_outcome.counters.unchanged, 1); - assert_eq!(repeat_outcome.counters.unknown, 0); + assert_eq!(repeat_outcome.counters.unchanged, u64::from(protected)); + assert_eq!(repeat_outcome.counters.unknown, u64::from(!protected)); assert_eq!(tokio::fs::read(&target_meta).await.expect("read repeated-heal target"), healed_bytes); let mut reader = env .ecstore @@ -273,8 +293,9 @@ async fn deep_heal_truncated_xlmeta_scenario() { let outcome = quorum_task.get_outcome().await; if damaged == 4 { result.expect("twelve authoritative members must repair four damaged copies"); - assert_eq!(outcome.counters.healed, 1); - assert_eq!(outcome.counters.unknown, 0); + assert_eq!(outcome.counters.healed, u64::from(protected)); + assert_eq!(outcome.counters.unknown, u64::from(!protected)); + assert_eq!(quorum_task.get_result_items().await[0].drives_healed(), Some(4)); assert!( quorum_task.get_result_items().await[0] .after diff --git a/crates/heal/tests/mrf_partial_write_test.rs b/crates/heal/tests/mrf_partial_write_test.rs index 24b444030..771995043 100644 --- a/crates/heal/tests/mrf_partial_write_test.rs +++ b/crates/heal/tests/mrf_partial_write_test.rs @@ -16,6 +16,8 @@ //! tests never start a scanner, auto-heal, or an explicit Admin Heal request. //! Run with nextest: channel and storage singletons require separate processes. +#![recursion_limit = "256"] + use rustfs_heal::heal::{ manager::{HealConfig, HealManager}, mrf_queue::{self, snapshot::inspect_local_committed_snapshot}, @@ -178,6 +180,8 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() { [ ("RUSTFS_HEAL_MRF_ENABLE", Some("true")), ("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", Some("false")), + ("RUSTFS_SHARD_INTEGRITY_WRITE", Some("false")), + ("RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED", Some("false")), ], async { let root = tempfile::tempdir().expect("test directory should be created"); @@ -313,14 +317,21 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() { ); assert!( wait_until(|| async { - inspect_local_committed_snapshot(SNAPSHOT_LIMIT) - .await - .expect("cleanup snapshot must validate") - .is_none() + let snapshot = manager.operations_snapshot().await; + snapshot.queue_length == 0 && snapshot.active_tasks == 0 }) .await, - "verified repair must release durable responsibility" + "legacy repair attempts must finish without inventing verification" ); + assert!(snapshot_contains("new.bin").await); + assert!(snapshot_contains("versioned.bin").await); + manager.stop().await.expect("test manager should stop"); + let before_purge = inspect_local_committed_snapshot(SNAPSHOT_LIMIT) + .await + .expect("legacy checkpoint must validate") + .expect("unverified legacy responsibility must remain") + .payload() + .to_vec(); // Deleting an existing marker by VersionId removes a version; it // must not create a new partial-write repair responsibility. for path in &env.disk_paths[12..] { @@ -346,12 +357,14 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() { ) .await .expect("explicit marker purge should retain quorum"); - assert!( + assert_eq!( inspect_local_committed_snapshot(SNAPSHOT_LIMIT) .await - .expect("purge must not create a checkpoint") - .is_none(), - "a physical marker purge must not be admitted as a marker creation repair" + .expect("purge checkpoint must validate") + .expect("existing legacy checkpoint must remain") + .payload(), + before_purge, + "a physical marker purge must not add a marker creation repair" ); for (path, disk) in env.disk_paths[12..].iter().zip(&all[12..]) { tokio::fs::remove_file(path).await.expect("remove purge outage sentinel"); @@ -361,7 +374,6 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() { disk.reset_health_for_store_init_retry(); } *set.disks.write().await = all.iter().cloned().map(Some).collect(); - manager.stop().await.expect("test manager should stop"); }, ) .await; @@ -391,6 +403,15 @@ fn partial_write_crash_fixture() { #[tokio::test] async fn partial_write_sigkill_replay_rearms_and_repairs() { + partial_write_sigkill_replay_scenario(true).await; +} + +#[tokio::test] +async fn legacy_sigkill_replay_repairs_without_releasing_unverified_responsibility() { + partial_write_sigkill_replay_scenario(false).await; +} + +async fn partial_write_sigkill_replay_scenario(protected: bool) { use std::process::{Command, Stdio}; let root = tempfile::tempdir().expect("crash test directory"); let log = std::fs::File::create(root.path().join("child.log")).expect("child log"); @@ -399,6 +420,8 @@ async fn partial_write_sigkill_replay_rearms_and_repairs() { .env("RUSTFS_TEST_PARTIAL_WRITE_CRASH_ROOT", root.path()) .env("RUSTFS_HEAL_MRF_ENABLE", "true") .env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false") + .env("RUSTFS_SHARD_INTEGRITY_WRITE", protected.to_string()) + .env("RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED", protected.to_string()) .stdout(Stdio::from(log.try_clone().expect("clone child log"))) .stderr(Stdio::from(log)) .spawn() @@ -432,15 +455,27 @@ async fn partial_write_sigkill_replay_rearms_and_repairs() { "replayed responsibility must heal the returning member" ); assert_payload(&env, "partial-crash", "crash.bin", None, b"durable partial write across SIGKILL").await; - assert!( - wait_until(|| async { - inspect_local_committed_snapshot(SNAPSHOT_LIMIT) - .await - .expect("valid checkpoint") - .is_none() - }) - .await, - "replayed responsibility must be released only after verified repair" - ); + if protected { + assert!( + wait_until(|| async { + inspect_local_committed_snapshot(SNAPSHOT_LIMIT) + .await + .expect("valid checkpoint") + .is_none() + }) + .await, + "replayed responsibility must be released only after verified repair" + ); + } else { + assert!( + wait_until(|| async { + let snapshot = manager.operations_snapshot().await; + snapshot.queue_length == 0 && snapshot.active_tasks == 0 + }) + .await, + "legacy replay attempts must finish" + ); + assert!(snapshot_contains("crash.bin").await, "unverified legacy responsibility must remain"); + } manager.stop().await.expect("restarted manager should stop"); } diff --git a/crates/heal/tests/shard_identity_receipt_test.rs b/crates/heal/tests/shard_identity_receipt_test.rs new file mode 100644 index 000000000..fafa88256 --- /dev/null +++ b/crates/heal/tests/shard_identity_receipt_test.rs @@ -0,0 +1,210 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![recursion_limit = "256"] + +use rustfs_heal::heal::{ + outcome::HealObjectDisposition, + storage::{ECStoreHealStorage, HealStorageAPI}, +}; +use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode}; +use rustfs_test_utils::TestECStoreEnv; +use serial_test::serial; +use tokio::io::AsyncReadExt as _; + +mod storage_api; +use storage_api::integration::{DiskAPI, ObjectIO, ObjectOptions, PutObjReader, ReadOptions, ShardIntegrityWriteMode}; + +#[tokio::test] +#[serial] +async fn legacy_repair_reports_execution_without_strong_receipt() { + let root = tempfile::tempdir().expect("legacy receipt fixture"); + let env = TestECStoreEnv::builder() + .base_dir(root.path()) + .prefix("legacy_receipt") + .build() + .await; + let bucket = "legacy-receipt"; + let object = "missing-shard"; + env.make_bucket(bucket, false).await; + let expected = vec![0x5c; 1024 * 1024 + 37]; + env.ecstore + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(expected.clone()), + &ObjectOptions { + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy), + ..Default::default() + }, + ) + .await + .expect("legacy object"); + let set = env.ecstore.pools[0].get_disks(0); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + let meta = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("metadata"); + tokio::fs::remove_file( + env.disk_paths[0] + .join(bucket) + .join(object) + .join(meta.data_dir.expect("external directory").to_string()) + .join("part.1"), + ) + .await + .expect("remove one shard"); + let result = ECStoreHealStorage::new(env.ecstore.clone()) + .heal_object_with_receipt( + bucket, + object, + None, + &HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("legacy repair"); + assert!(result.error.is_none(), "{:?}", result.error); + assert_eq!(result.item.drives_healed(), Some(1)); + assert!(!result.item.integrity_verified); + assert!(result.receipt.is_none(), "physical repair does not prove original identity"); + let mut reader = env + .ecstore + .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default()) + .await + .expect("read repaired object"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("exact recovered body"); + assert_eq!(actual, expected); +} + +#[tokio::test] +#[serial] +async fn receipt_requires_independent_deep_verification() { + let root = tempfile::tempdir().expect("receipt fixture"); + let env = TestECStoreEnv::builder() + .base_dir(root.path()) + .prefix("shard_identity_receipt") + .build() + .await; + let bucket = "shard-identity-receipt"; + env.make_bucket(bucket, false).await; + let set = env.ecstore.pools[0].get_disks(0); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + let storage = ECStoreHealStorage::new(env.ecstore.clone()); + for corrupt_count in [1usize, 3] { + let object = format!("target-{corrupt_count}"); + let donor = format!("donor-{corrupt_count}"); + for (name, byte) in [(&object, 0x3c), (&donor, 0xa9)] { + env.ecstore + .put_object( + bucket, + name, + &mut PutObjReader::from_vec(vec![byte; 1024 * 1024 + 123]), + &ObjectOptions { + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("commit all fixture shards"); + } + let mut target_meta = Vec::new(); + let mut donor_meta = Vec::new(); + for disk in &disks { + target_meta.push( + disk.read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("target metadata"), + ); + donor_meta.push( + disk.read_version("", bucket, &donor, "", &ReadOptions::default()) + .await + .expect("donor metadata"), + ); + } + let mut retained = Vec::new(); + for (slot, target) in target_meta.iter().enumerate() { + let target_path = env.disk_paths[slot] + .join(bucket) + .join(&object) + .join(target.data_dir.expect("target directory").to_string()) + .join("part.1"); + let original = tokio::fs::read(&target_path).await.expect("original shard"); + if target.erasure.index <= corrupt_count { + let donor_slot = donor_meta + .iter() + .position(|part| part.erasure.index == target.erasure.index) + .expect("same donor coding index"); + let donor_path = env.disk_paths[donor_slot] + .join(bucket) + .join(&donor) + .join(donor_meta[donor_slot].data_dir.expect("donor directory").to_string()) + .join("part.1"); + let replacement = tokio::fs::read(donor_path).await.expect("complete donor shard"); + assert_eq!(replacement.len(), original.len()); + tokio::fs::write(&target_path, replacement) + .await + .expect("replace intact shard"); + } else { + retained.push((target_path, original)); + } + } + let normal = storage + .heal_object_with_receipt( + bucket, + &object, + None, + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Normal, + ..Default::default() + }, + ) + .await + .expect("normal presence scan"); + assert!(normal.receipt.is_none(), "a presence scan cannot certify payload integrity"); + let result = storage + .heal_object_with_receipt( + bucket, + &object, + None, + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await; + if corrupt_count == 1 { + let result = result.expect("recoverable repair"); + assert!(result.error.is_none()); + assert_eq!( + result.receipt.expect("verified repair receipt").disposition, + HealObjectDisposition::Repaired, + "presence alone must not produce VerifiedHealthy" + ); + } else if let Ok(result) = result { + assert!(result.error.is_some(), "below quorum cannot be a success"); + assert!(result.receipt.is_none(), "no completion receipt without authoritative sources"); + } + for (path, original) in retained { + assert_eq!(tokio::fs::read(path).await.expect("retained correct shard"), original); + } + } +} diff --git a/crates/heal/tests/storage_api.rs b/crates/heal/tests/storage_api.rs index ae9f3449c..9927054f1 100644 --- a/crates/heal/tests/storage_api.rs +++ b/crates/heal/tests/storage_api.rs @@ -23,7 +23,7 @@ pub(crate) mod integration { pub(crate) use rustfs_ecstore::api::disk::{ DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, RUSTFS_META_BUCKET, ReadOptions, new_disk, }; - pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader}; + pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode}; pub(crate) use rustfs_ecstore::api::storage::ECStore; pub(crate) use rustfs_storage_api::BucketOperations; pub(crate) use rustfs_storage_api::MakeBucketOptions; diff --git a/crates/madmin/src/heal_commands.rs b/crates/madmin/src/heal_commands.rs index faa2763b1..ac4340bec 100644 --- a/crates/madmin/src/heal_commands.rs +++ b/crates/madmin/src/heal_commands.rs @@ -36,6 +36,10 @@ const DRIVE_STATE_OK: &str = "ok"; #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct HealResultItem { + /// Only the coordinator's authenticated full scan may set this. Transported + /// or old-peer results default to unproven and cannot advance heal receipts. + #[serde(skip)] + pub integrity_verified: bool, #[serde(rename = "resultId")] pub result_index: usize, #[serde(rename = "type")] diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 0ee5014ad..1654d3582 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -288,6 +288,9 @@ pub struct RenameFileRequest { pub dst_volume: ::prost::alloc::string::String, #[prost(string, tag = "5")] pub dst_path: ::prost::alloc::string::String, + /// Apply the destination's payload/metadata durability policy before success. + #[prost(bool, tag = "6")] + pub durable: bool, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RenameFileResponse { @@ -295,6 +298,8 @@ pub struct RenameFileResponse { pub success: bool, #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, + #[prost(bool, tag = "3")] + pub durability_applied: bool, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WriteRequest { diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 48307708b..3b416559b 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -1139,6 +1139,11 @@ pub fn canonical_rename_file_request_body( body.push_str(&request.src_path)?; body.push_str(&request.dst_volume)?; body.push_str(&request.dst_path)?; + // Preserve the release signature for ordinary renames. An older server + // computes a different digest for a durable request and rejects mutation. + if request.durable { + body.push_bool(true); + } Ok(body.finish()) } @@ -1450,6 +1455,7 @@ mod disk_mutation_canonical_tests { assert_all_distinct(&bodies); let rename_file = RenameFileRequest { + durable: false, disk: "d".into(), src_volume: "sv".into(), src_path: "sp".into(), @@ -1458,6 +1464,7 @@ mod disk_mutation_canonical_tests { }; let mut bodies = vec![canonical_rename_file_request_body(&rename_file).unwrap()]; for mutate in [ + |r: &mut RenameFileRequest| r.durable = true, |r: &mut RenameFileRequest| r.disk = "d2".into(), |r: &mut RenameFileRequest| r.src_volume = "sv2".into(), |r: &mut RenameFileRequest| r.src_path = "sp2".into(), @@ -1632,10 +1639,28 @@ mod disk_mutation_canonical_tests { ); } + #[test] + fn durable_rename_extension_preserves_old_wire_defaults_and_signatures() { + use crate::proto_gen::node_service::RenameFileResponse; + use prost::Message; + // Release wire messages omitted request tag 6 and response tag 3. + let mut request = + RenameFileRequest::decode(b"\x0a\x01d\x12\x01s\x1a\x01p\x22\x01v\x2a\x01q".as_slice()).expect("release request"); + assert!(!request.durable); + let old_body = b"rustfs-rename-file-request-v1\0\0\0\0\0\0\0\0\x01d\0\0\0\0\0\0\0\x01s\0\0\0\0\0\0\0\x01p\0\0\0\0\0\0\0\x01v\0\0\0\0\0\0\0\x01q"; + assert_eq!(canonical_rename_file_request_body(&request).expect("body"), old_body); + request.durable = true; + assert_ne!(canonical_rename_file_request_body(&request).expect("durable body"), old_body); + let old_success = RenameFileResponse::decode(b"\x08\x01".as_slice()).expect("release response"); + assert!(old_success.success); + assert!(!old_success.durability_applied, "old success cannot certify durable publication"); + } + #[test] fn disk_mutation_canonical_domains_are_distinct_per_message() { // The same field values must never authenticate one RPC's request as another's. let rename_file = RenameFileRequest { + durable: false, disk: "d".into(), src_volume: "sv".into(), src_path: "sp".into(), diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index a35797497..65c55a1b9 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -219,11 +219,14 @@ message RenameFileRequest { string src_path = 3; string dst_volume = 4; string dst_path = 5; + // Apply the destination's payload/metadata durability policy before success. + bool durable = 6; } message RenameFileResponse { bool success = 1; optional Error error = 2; + bool durability_applied = 3; } message WriteRequest { diff --git a/docs/README.md b/docs/README.md index 35d3324af..af6c50329 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,3 +24,6 @@ procedure is required. For storage dashboards, see [Storage metrics and observer selection](operations/storage-metrics.md): drive ownership, snapshot freshness, counter queries, and rolling upgrades. + +For optional shard commitments, see [Independent shard integrity rollout](operations/shard-integrity-rollout.md): +activation, legacy repair results, multipart mode changes, and rollback limits. diff --git a/docs/architecture/erasure-coding.md b/docs/architecture/erasure-coding.md index 921f845e7..abfd80a3f 100644 --- a/docs/architecture/erasure-coding.md +++ b/docs/architecture/erasure-coding.md @@ -156,6 +156,73 @@ Each shard file is self-verifying against silent disk corruption. --- +### 5.1 Independent shard commitments + +New PUTs and newly initiated multipart uploads retain `CSumAlgo = 1` and the +existing `[HighwayHash256][shard]` frames. Independent commitments are disabled +for new writes by default; both `RUSTFS_SHARD_INTEGRITY_WRITE` and +`RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED` must be enabled to start protecting new +writes. Existing protected objects and uploads retain their mode. An independent SHA-256 +commitment protects against replacing a complete frame with a same-length donor +frame whose self-contained checksum is valid (backlog#2497). + +The commitment describes an immutable part generation, with a random UUID, +part number, exact encoded length, erasure geometry, codec mode, stripe count, +and Merkle root. Each leaf commits to the ordered SHA-256 digests of **all** +encoded shards in one stripe. Payload digests include generation, part, stripe, +coding index, length, and geometry. The root also commits to the final part size. +A valid metadata quorum selects the expected root; neither RS consistency nor +an intact adjacent HighwayHash checksum establishes the root. + +The dual-prefix internal `shard-integrity-v1` metadata value encodes a canonical +Base64 table: a 32-byte header and 64 bytes per part, up to 10,000 sorted unique +parts. The same table is stored under both internal prefixes. Readers reject a +malformed, conflicting, or incomplete descriptor instead of treating it as +legacy metadata. Each external part has an immutable +`part.N.integrity.` sidecar, replicated on every participating +disk. Its 64-byte header is followed by stripe records containing all shard +digests and a Merkle path. An inline object stores the small proof under the +dual-prefix `shard-integrity-inline-v1` key. + +GET authenticates a stripe record against the selected root, then verifies each +source shard before decoding or emitting bytes. Reconstructed data is also +checked against its expected digest. Range reads fetch only the proof records +for touched stripes; deferred parity readers retain their exact stripe position. +A missing or corrupt index replica can use another authenticated replica. Deep +Heal verifies at the coordinator, including data returned by older disk servers, +and restores missing indexes only from proofs matching the existing root. It +never mints a new commitment from suspect stored bytes. Index-only repair leaves +payload and `xl.meta` bytes unchanged and requires acknowledged durable publish. + +Writes publish the payload and proof on the same write quorum. UploadPart uses +a fresh generation for each replacement, includes that generation and root in +part-metadata quorum selection, and publishes its index before the existing +part transaction switches data and metadata. Settlement removes only the +obsolete generation; rollback retains the old index. Interrupted preparation +can leave unreferenced files until the upload directory is reclaimed. Ordinary +metadata COPY preserves commitments. Physical rewrites inherit their source +mode: protected sources create new commitments after reading through their +verifier; legacy sources remain legacy. Rewriting existing bytes is not a +trusted migration of their historical identity. An upload's persisted marker, +not the current node's write switch, determines its UploadPart/Complete mode. + +Digest/index builders spill above a 1 MiB buffer limit, and request readers keep +a bounded stripe cache. For EC 12+4, a 5 GiB part with 1 MiB stripes has a +4,751,424-byte index on each disk (about 1.42% of logical data across 16 disks). +The maximum descriptor is 853,376 Base64 bytes per prefix, about 1.63 MiB for +both copies. These are format bounds, not measured throughput guarantees. + +Legacy objects retain their existing GET and traditional Heal behavior and +therefore their residual complete-donor substitution risk. Ordinary shard repair +and the existing explicit-version metadata recovery path remain available, but +do not create commitments or certify object identity. Actual drive repairs are +reported separately from strong integrity receipts. Normal +presence scans do not issue strong integrity receipts even for protected +objects. Only a completed exclusive Deep scan/repair with authenticated sources +can do so. See the [upgrade contract](minio-file-format-compat.md#independent-integrity-upgrade-contract) +for mixed-version and migration constraints and the +[rollout runbook](../operations/shard-integrity-rollout.md) for activation and rollback. + ## 6. On-disk format (`xl.meta`) This section is the load-bearing compatibility contract for stored metadata. It is byte-compatible with MinIO's `xl.meta`; interop proof, the fixture corpus, and the out-of-scope list are owned by [minio-file-format-compat.md](minio-file-format-compat.md) — cite it, do not re-derive interop claims. @@ -252,7 +319,7 @@ Version-aware heal ([set_disk/ops/heal.rs](../../crates/ecstore/src/set_disk/ops - **INVARIANT — reconstructability.** Heal refuses when `meta_to_heal_count > parity_blocks` (relaxed only if a quorum etag exists) or when any part loses more than `parity_blocks` shards. - **INVARIANT — geometry match.** `latest_meta.erasure.distribution.len()` must equal the online-disk, outdated-disk, and parts-metadata counts, else heal refuses ("backend disks manually modified"). A real object missing `data_dir` is `FileCorrupt`. -- **Data-safety guard (backlog#920).** If data shards survive on ≥ `data_blocks` disks, regenerate the missing `xl.meta` from a valid FileInfo and re-drive heal rather than dangling-delete; torn writes (< `data_blocks`) fall through to dangling-delete handling. +- **Data-safety guard (backlog#920).** Legacy explicit-version metadata recovery retains its validated geometry, consistent candidate identity, online-disk and available-data bounds. It does not establish independent payload identity. Protected recovery additionally requires a matching metadata quorum and authenticated data on ≥ `data_blocks` disks. Conflicting metadata and uncertain deletion conditions retain their existing refusal/grace behavior. - Healed shards are written to the outdated disks, each recording `erasure.index = slot + 1`, then `rename_data` to final. Heal admission / scanner budget is owned by [placement-repair-invariants.md](placement-repair-invariants.md). --- diff --git a/docs/architecture/minio-file-format-compat.md b/docs/architecture/minio-file-format-compat.md index 9b7675624..2daa1e50b 100644 --- a/docs/architecture/minio-file-format-compat.md +++ b/docs/architecture/minio-file-format-compat.md @@ -146,3 +146,55 @@ Not fixture-proven: transitioned `xl.meta`; CORS, public-access-block, and bucke - Any change to `crates/filemeta` or `crates/ecstore/src/bucket` metadata encoding is a storage-format change and follows the migration and readiness contracts in [README.md](README.md) and the ecstore layout boundary rules. - Do not bump a Version Anchor without a read path for the prior value; see [erasure-coding.md](erasure-coding.md) for the accept-older, reject-newer rule. - `.github/workflows/ci.yml`, `.github/workflows/cache-warm.yml`, and `ARCHITECTURE.md` cite the [rio-v2 variant lifecycle](#rio-v2-variant-lifecycle) heading; keep it when editing this file. + +## Independent integrity upgrade contract + +The optional shard-integrity extension preserves the release checksum algorithm +and payload framing. It does not bump XL container/header/metadata versions. +`ObjectPartInfo` extensions use named MessagePack fields; older eight-field +readers can ignore the new field, and new readers still decode old positional +arrays. This is a decoder compatibility claim, not certification of arbitrary +mixed-version write, repair, tiering, or downgrade workflows. + +| Workflow | Contract | +|---|---| +| New reader, old object | Existing reads and traditional shard/explicit-version metadata recovery remain available; independent donor-substitution protection is absent until a trusted rewrite. | +| Old reader, new object | Existing checksum/frame and metadata decoders remain usable; old readers do not enforce the independent proof. | +| New coordinator, old disk server | GET/Deep verification runs at the coordinator. A protected UploadPart requires a write quorum that publishes its index. An old RenameFile response cannot acknowledge durable index repair. | +| Old write/repair coordinator | Unsupported for protected data: it may omit, discard, or preserve stale integrity metadata. New protection is disabled by default so the coordinator rollout can precede activation. | +| Pre-upgrade in-progress multipart upload | Remains legacy when completed. Reinitiate and reupload from a trusted source to obtain protection. | +| New multipart upload completed by an old coordinator | Unsupported; a retained upload marker without a completed descriptor fails validation. | +| Write switch changed during an upload | The persisted upload mode wins: legacy uploads remain legacy and protected uploads must finish with all proofs. | +| Write switch disabled after activation | Existing protected reads, repair and rewrites still require protection. This does not make an old binary safe to reintroduce. | +| Automatic COPY, materialization or data movement | Preserve the source protection mode. Rewriting legacy bytes is not independent evidence of their original content. | +| Full downgrade after protected writes | No blanket guarantee; preserve a snapshot and validate read, write, repair, COPY, and multipart behavior before considering a downgrade. | + +Upgrade all readers, writers and background coordinators before explicitly +enabling both `RUSTFS_SHARD_INTEGRITY_WRITE` and +`RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED`. Both default to false. The confirmation +is an operator attestation, not automatic capability discovery or an old-node +fence. Validate actual mixed-version writes and restart recovery before claiming +a rolling-upgrade or pre-activation rollback guarantee. See the +[rollout runbook](../operations/shard-integrity-rollout.md). +There is no automatic migration of existing payloads and no need +to rewrite them merely to preserve reading. To protect legacy objects, compare +against a separately trusted source or end-to-end digest and rewrite/reupload +through upgraded coordinators. Recomputing a hash from existing suspect shards, +or agreeing RS parity alone, does not establish their original identity. + +Normal scans, tier metadata scans and unproven legacy scans cannot produce `VerifiedHealthy` or +`Repaired` integrity receipts. Legacy repair still runs and reports actual +before/after drive changes; an unknown strong result does not mean the repair +was never attempted. Authoritative historical-version absence/cleanup proofs +remain separate from live payload verification. Retain trusted backups for +legacy recovery. These commitments protect against misplaced or corrupted +shards under an authoritative metadata quorum; they are not signatures against +an attacker who can replace that quorum too. + +The public MinIO source inspected at +[`7aac2a2`](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/bitrot-streaming.go) +uses adjacent streaming HighwayHash checksums. Its +[XL metadata codec](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/xl-storage-format-v2.go) +does not establish the independent part-generation commitment described here. +This source comparison does not claim a live MinIO donor-shard reproduction or +behavior of proprietary MinIO editions. diff --git a/docs/operations/shard-integrity-rollout.md b/docs/operations/shard-integrity-rollout.md new file mode 100644 index 000000000..bf389fc11 --- /dev/null +++ b/docs/operations/shard-integrity-rollout.md @@ -0,0 +1,92 @@ +# Independent shard integrity rollout + +Independent shard commitments detect internally valid shards that belong to a +different part generation. The feature preserves the existing checksum frames +and adds optional metadata and proof indexes. It does not authenticate legacy +data retroactively. The [format contract](../architecture/erasure-coding.md#51-independent-shard-commitments) +defines the stored representation. + +## Configuration and object lifetime + +| Variable | Default | Effect | +|---|---|---| +| `RUSTFS_SHARD_INTEGRITY_WRITE` | `false` | Requests protection for new writes without an inherited object/upload mode. | +| `RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED` | `false` | Operator confirmation that every reader, writer and background coordinator supports the extension. | + +Both must be true before new writes opt in. Setting only the request switch +does not enable protection. These flags do not perform peer capability discovery +or stop an older binary from joining the cluster. + +A PUT fixes its mode before encoding. An MPU fixes its mode at initiation and +persists it with the upload. Completing an old upload after activation does not +promote it. Disabling the switches during a protected upload does not permit +parts or completion to drop the proof requirement. + +Reads always follow the object's metadata. Turning off creation does not disable +verification of existing protected objects. A corrupt, conflicting or incomplete +protection declaration is an error, not a legacy object. Metadata-only COPY +preserves the existing commitment. COPY and physical background rewrites inherit +the source mode; they do not turn an unverified legacy object into a historically +verified object simply by calculating a new hash. + +## Upgrade and activation + +1. Deploy a version that understands and maintains the extension, leaving both + switches false. Existing legacy reads, shard repair and explicit-version + metadata recovery continue under their existing bounds. +2. Validate old/new process combinations for PUT, multipart completion, COPY, + Heal and restart recovery in the actual deployment. Decoder fixture tests + alone do not certify these operations. +3. Finish upgrading every reader and write/repair/movement coordinator. Prevent + old services or rollback images from rejoining before enabling protection. +4. Qualify protected writes, range reads, repair, missing indexes and deployment + restart/failure behavior. Measure small-object throughput, range latency and + maximum multipart metadata overhead against the same release configuration. +5. Enable both switches consistently. Verify new objects with an exclusive Deep + scan before treating the enhanced mode as qualified for production traffic. + +Existing rollout or performance gaps remain gates for production activation; +this runbook is not a claim that a distributed upgrade has been exercised. + +## Legacy repair and interpretation + +Legacy objects remain exposed to substitution by a complete, internally valid +donor shard. Traditional reconstruction can restore missing or detectably +damaged shards, but cannot prove that all surviving bytes are the original +content. It therefore does not create an independent digest or a positive strong +integrity receipt. + +Use the heal item's before/after drives and actual repaired-drive count to +observe ordinary recovery. Strong outcome counters may remain unknown/skipped +because object identity is unproven; that is distinct from execution progress. +MRF likewise retains durable legacy repair obligations after physical recovery +when no independent verification receipt can discharge them. An idle attempt +queue does not imply that these persistent obligations have been cleared. +Partial-write MRF replay uses Deep verification so a protected object's repair +can discharge its obligation after verifying the payload. This adds full-object +read work to those background attempts, including healthy replay targets. +Normal presence scans likewise cannot certify protected payloads. Local Heal +of transitioned objects checks metadata without reading the tier payload, so it +does not issue a payload-integrity receipt even if a descriptor remains. An +authoritative historical-version cleanup or absence proof has its own identity +and commit checks and does not depend on a live payload digest. + +To protect a legacy version as a trusted migration, validate against an +independently trusted original or suitable end-to-end digest, then reupload. +An ETag is not universally a plaintext MD5. Rehashing existing bytes or checking +RS parity alone does not establish the original identity. Keep this distinction +when measuring migration coverage; an ordinary server-side rewrite is not proof +of historical correctness. + +## Rollback + +Before any protected objects or uploads exist, a rollback still needs actual +old-version write/repair/restart qualification. Metadata decoding compatibility +is insufficient. + +After activation, the supported rollback floor is a version that understands +and preserves the extension. Turning off creation leaves protected objects, +historical versions and in-progress protected uploads in place. It neither +converts them to legacy nor authorizes an arbitrary older reader/writer/repair +coordinator. Preserve recovery snapshots and use an explicitly validated +migration procedure if returning to an older format implementation is required. diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 4c58e8fdb..4910363e0 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -4126,6 +4126,41 @@ mod tests { } } + #[tokio::test] + async fn durable_rename_requires_its_own_authenticated_body_and_success() { + let service = make_server(); + let mut message = RenameFileRequest { + disk: "http://node-a:9000/data/rustfs0".to_owned(), + src_volume: ".rustfs.sys/tmp".to_owned(), + src_path: "integrity-stage/index".to_owned(), + dst_volume: "bucket".to_owned(), + dst_path: "object/index".to_owned(), + durable: false, + }; + let old_body = rustfs_protos::canonical_rename_file_request_body(&message).expect("ordinary rename body"); + message.durable = true; + let mut tampered = Request::new(message.clone()); + set_tonic_canonical_body_digest(&mut tampered, &old_body).expect("digest"); + mark_v2_authenticated(&mut tampered); + let error = service + .rename_file(tampered) + .await + .expect_err("durability flag must be authenticated"); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + + let body = rustfs_protos::canonical_rename_file_request_body(&message).expect("durable rename body"); + let mut request = Request::new(message); + set_tonic_canonical_body_digest(&mut request, &body).expect("digest"); + mark_v2_authenticated(&mut request); + let response = service + .rename_file(request) + .await + .expect("valid digest passes the gate") + .into_inner(); + assert!(!response.success, "unknown disk cannot apply the rename"); + assert!(!response.durability_applied, "failed publication cannot acknowledge durability"); + } + #[tokio::test] async fn disk_mutation_body_digest_gate_runs_before_disk_lookup() { let service = make_server(); @@ -4340,6 +4375,7 @@ mod tests { assert_gated!( rename_file, RenameFileRequest { + durable: false, disk: disk.clone(), src_volume: "src".into(), src_path: "sp".into(), @@ -5512,6 +5548,7 @@ mod tests { let service = create_test_node_service(); let request = Request::new(RenameFileRequest { + durable: false, disk: "invalid-disk-path".to_string(), src_volume: "src-volume".to_string(), src_path: "src-path".to_string(), diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index fb237b31c..3ece3c7bd 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -1379,22 +1379,29 @@ impl NodeService { )?; let request = request.into_inner(); if let Some(disk) = self.find_disk(&request.disk).await { - match disk - .rename_file(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path) - .await - { + let result = if request.durable { + disk.rename_file_durable(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path) + .await + } else { + disk.rename_file(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path) + .await + }; + match result { Ok(_) => Ok(Response::new(RenameFileResponse { success: true, + durability_applied: request.durable, error: None, })), Err(err) => Ok(Response::new(RenameFileResponse { success: false, + durability_applied: false, error: Some(err.into()), })), } } else { Ok(Response::new(RenameFileResponse { success: false, + durability_applied: false, error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 5c8a095ec..24a088c16 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -1395,6 +1395,15 @@ pub(crate) trait StorageDiskRpcExt { async fn read_file(&self, volume: &str, path: &str) -> DiskResult; async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult; async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()>; + async fn rename_file_durable( + &self, + _src_volume: &str, + _src_path: &str, + _dst_volume: &str, + _dst_path: &str, + ) -> DiskResult<()> { + Err(DiskError::MethodNotAllowed) + } async fn rename_part( &self, src_volume: &str, @@ -1550,6 +1559,10 @@ where ecstore_disk::DiskAPI::rename_file(self, src_volume, src_path, dst_volume, dst_path).await } + async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()> { + ecstore_disk::DiskAPI::rename_file_durable(self, src_volume, src_path, dst_volume, dst_path).await + } + async fn rename_part( &self, src_volume: &str,