fix(ecstore): gate shard integrity and preserve legacy recovery (#7741)

* fix(ecstore)!: bind bitrot shards to immutable part identities

Verify part, coding-index, and block identity across write, GET, and Heal
paths. Preserve identities across metadata-only copies and repair, include
them in multipart quorum selection, and require payload proof for receipts.

Keep legacy decoding with conservative parity and target-digest validation,
and document its unsupported cases and additional verification I/O.

BREAKING CHANGE: New bound-v1 shards require compatible readers throughout
the fleet. Legacy objects without sufficient integrity evidence return an
error; binary rollback after new writes requires verified data migration.

Refs: rustfs/backlog#2497

* fix(ecstore): preserve shard framing with independent integrity

Commit immutable part-generation Merkle roots and replicated proof indexes without changing existing checksum frames. Verify reads, reconstruction and Deep Heal against metadata quorum; keep legacy reads and explicitly defer unproven legacy data repair.

Preserve multipart rollback generations, require acknowledged durable index publication, and add decoder compatibility and donor-shard regression coverage.

* fix(heal): verify protected partial-write replay
This commit is contained in:
cxymds
2026-09-13 21:25:58 +08:00
committed by GitHub
parent d0c7aec0b4
commit 17ddecb075
53 changed files with 4281 additions and 160 deletions
Generated
+2
View File
@@ -9893,6 +9893,7 @@ name = "rustfs-filemeta"
version = "1.0.0-rc.5"
dependencies = [
"arc-swap",
"base64-simd",
"byteorder",
"bytes",
"crc-fast",
@@ -9907,6 +9908,7 @@ dependencies = [
"s3s",
"serde",
"serde_json",
"sha2 0.11.0",
"tempfile",
"thiserror 2.0.20",
"time",
+14
View File
@@ -116,6 +116,20 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
/// Default: bitrot verification is enabled on GetObject reads (do not skip).
pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
/// Create independent shard commitments for new writes after the fleet is upgraded.
/// Existing protected objects and multipart uploads retain their protection.
pub const ENV_SHARD_INTEGRITY_WRITE: &str = "RUSTFS_SHARD_INTEGRITY_WRITE";
pub const DEFAULT_SHARD_INTEGRITY_WRITE: bool = false;
/// Operator confirmation that every reader, writer and background coordinator
/// understands independent shard commitments. This is not capability discovery
/// or a fence against an old binary rejoining the fleet.
pub const ENV_SHARD_INTEGRITY_FLEET_CONFIRMED: &str = "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED";
pub const DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED: bool = false;
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_WRITE);
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED);
/// Request writing the complete remote-tier version state into object metadata.
///
/// This remains ineffective until
+3 -3
View File
@@ -501,9 +501,9 @@ pub mod object {
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len,
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCompletion,
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
};
pub use crate::store::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -5190,6 +5190,7 @@ pub async fn put_restore_opts(
// Restore writes stored (possibly encrypted) bytes, so the writer's
// computed MD5 is not the object's public plaintext ETag.
preserve_etag: oi.etag.clone(),
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
//expires: oi.expires,
..Default::default()
})
+44 -23
View File
@@ -844,6 +844,44 @@ fn spawn_control_channel_prewarm(addr: String) {
}
impl RemoteDisk {
async fn rename_file_with_durability(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
durable: bool,
) -> Result<()> {
self.execute_with_timeout(
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(RenameFileRequest {
durable,
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let canonical_body = rustfs_protos::canonical_rename_file_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "rename_file")?;
let response = client.rename_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
if durable && !response.durability_applied {
return Err(DiskError::MethodNotAllowed);
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
pub(crate) async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
@@ -3401,30 +3439,13 @@ impl DiskAPI for RemoteDisk {
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(RenameFileRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let canonical_body = rustfs_protos::canonical_rename_file_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "rename_file")?;
self.rename_file_with_durability(src_volume, src_path, dst_volume, dst_path, false)
.await
}
let response = client.rename_file(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
self.rename_file_with_durability(src_volume, src_path, dst_volume, dst_path, true)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
+1
View File
@@ -785,6 +785,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
let put_opts = ObjectOptions {
user_defined: dst_opts.user_defined.clone(),
shard_integrity_write_mode: Some(src_info.shard_integrity_write_mode()),
versioned: dst_opts.versioned,
version_id: dst_opts.version_id.clone(),
mod_time: dst_opts.mod_time,
+23
View File
@@ -296,6 +296,7 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz
preserve_etag: object_info.etag.clone(),
src_pool_idx,
data_movement: true,
shard_integrity_write_mode: Some(object_info.shard_integrity_write_mode()),
..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
}
}
@@ -476,6 +477,7 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize)
versioned: object_info.version_id.is_some(),
src_pool_idx,
data_movement: true,
shard_integrity_write_mode: Some(object_info.shard_integrity_write_mode()),
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
http_preconditions: Some(data_movement_target_precondition()),
mod_time: object_info.mod_time,
@@ -2109,6 +2111,27 @@ async fn migrate_object_inner(
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn data_movement_retains_source_shard_integrity_mode() {
use crate::object_api::ShardIntegrityWriteMode;
let mut source = ObjectInfo::default();
for mode in [ShardIntegrityWriteMode::Legacy, ShardIntegrityWriteMode::Protected] {
if mode == ShardIntegrityWriteMode::Protected {
// A declaration selects protected I/O; the source reader still
// rejects this deliberately incomplete descriptor before commit.
rustfs_utils::http::insert_str(
Arc::make_mut(&mut source.user_defined),
rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY,
"invalid".to_owned(),
);
}
let put = data_movement_put_object_opts(&source, 0);
let multipart = data_movement_new_multipart_opts(&source, 0);
assert_eq!(put.shard_integrity_write_mode, Some(mode));
assert_eq!(multipart.shard_integrity_write_mode, Some(mode));
}
}
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
use rustfs_rio::{Checksum, ChecksumType};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
+14
View File
@@ -2396,6 +2396,20 @@ impl DiskAPI for LocalDiskWrapper {
.await
}
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
self.track_disk_health_mutation(
"rename_file",
DiskMetricMutation::Write,
|| async {
self.disk
.rename_file_durable(src_volume, src_path, dst_volume, dst_path)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn prepare_part_transaction(
&self,
src_volume: &str,
+272 -6
View File
@@ -9354,6 +9354,41 @@ impl DiskAPI for LocalDisk {
}
let durability = effective_durability(dst_volume);
let part = ObjectPartInfo::unmarshal(&meta)?;
if let Some(integrity) = part.integrity {
integrity.validate()?;
if usize::try_from(integrity.number).map_err(|_| DiskError::FileCorrupt)? != part.number
|| usize::try_from(integrity.size).map_err(|_| DiskError::FileCorrupt)? != part.size
|| dst_file_path.file_name().and_then(|name| name.to_str()) != Some(format!("part.{}", part.number).as_str())
{
return Err(DiskError::FileCorrupt);
}
let proof_name = integrity.file_name();
let source = src_file_path.parent().ok_or(DiskError::FileCorrupt)?.join(&proof_name);
let destination = dst_file_path.parent().ok_or(DiskError::FileCorrupt)?.join(&proof_name);
check_path_length(source.to_string_lossy().as_ref())?;
check_path_length(destination.to_string_lossy().as_ref())?;
let stat = fs::symlink_metadata(&source).await.map_err(to_file_error)?;
if !stat.is_file() || stat.len() != u64::try_from(integrity.index_size()?).map_err(|_| DiskError::FileCorrupt)? {
return Err(DiskError::FileCorrupt);
}
if durability.syncs_data_shards() {
let source = source.clone();
tokio::task::spawn_blocking(move || os::sync_file(&source))
.await
.map_err(DiskError::from)?
.map_err(to_file_error)?;
}
// Publish the immutable generation before preparing the part switch.
// Rollback retains the old generation; an unreferenced new index is
// reclaimed with the upload directory if preparation is interrupted.
rename_all(&source, &destination, &dst_volume_dir, &self.publication_root).await?;
if durability.syncs_commit_metadata() {
os::fsync_dir(destination.parent().ok_or(DiskError::FileCorrupt)?)
.await
.map_err(to_file_error)?;
}
}
tokio::task::spawn_blocking(move || {
let source = std::fs::symlink_metadata(&src_file_path).map_err(to_file_error)?;
if !source.is_file() {
@@ -9462,6 +9497,41 @@ impl DiskAPI for LocalDisk {
let Some(parent) = transaction_path.parent() else {
return Err(DiskError::InvalidPath);
};
// Delete only the generation made obsolete by this settled switch.
// Check the published metadata before removing either proof, so a
// repeated or interrupted settlement cannot remove a live index.
let (retained_name, obsolete_name) = match action {
PartTransactionAction::Commit => (PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META),
PartTransactionAction::Rollback => (PART_TRANSACTION_OLD_META, PART_TRANSACTION_NEW_META),
};
if let (Ok(current), Ok(retained), Ok(obsolete)) = (
std::fs::read(&current_meta_path),
std::fs::read(transaction_path.join(retained_name)),
std::fs::read(transaction_path.join(obsolete_name)),
) && current == retained
&& let Ok(obsolete) = ObjectPartInfo::unmarshal(&obsolete)
&& let Some(integrity) = obsolete.integrity
&& integrity.validate().is_ok()
&& usize::try_from(integrity.number).ok() == Some(obsolete.number)
&& current_data_path.file_name().and_then(|name| name.to_str())
== Some(format!("part.{}", obsolete.number).as_str())
&& ObjectPartInfo::unmarshal(&retained)
.ok()
.and_then(|part| part.integrity)
.as_ref()
!= Some(&integrity)
{
let obsolete_path = parent.join(integrity.file_name());
match std::fs::remove_file(&obsolete_path) {
Ok(()) => {
if durability.syncs_commit_metadata() {
os::fsync_dir_std(parent).map_err(to_file_error)?;
}
}
Err(error) if error.kind() == ErrorKind::NotFound => {}
Err(error) => return Err(to_file_error(error).into()),
}
}
let cleanup_path = parent.join(format!(".part-txn-settled-{}", Uuid::new_v4()));
std::fs::rename(&transaction_path, &cleanup_path).map_err(to_file_error)?;
if durability.syncs_commit_metadata() {
@@ -9735,6 +9805,31 @@ impl DiskAPI for LocalDisk {
.await
}
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
let source = self.io_get_object_path(src_volume, src_path)?;
let destination = self.io_get_object_path(dst_volume, dst_path)?;
check_path_length(source.to_string_lossy().as_ref())?;
check_path_length(destination.to_string_lossy().as_ref())?;
let durability = effective_durability(dst_volume);
let stat = fs::symlink_metadata(&source).await.map_err(to_file_error)?;
if !stat.is_file() {
return Err(DiskError::FileAccessDenied);
}
if durability.syncs_data_shards() {
tokio::task::spawn_blocking(move || os::sync_file(&source))
.await
.map_err(DiskError::from)?
.map_err(to_file_error)?;
}
self.rename_file(src_volume, src_path, dst_volume, dst_path).await?;
if durability.syncs_commit_metadata() {
os::fsync_dir(destination.parent().ok_or(DiskError::InvalidPath)?)
.await
.map_err(to_file_error)?;
}
Ok(())
}
#[tracing::instrument(level = "trace", skip_all)]
// async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result<File> {
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
@@ -12088,7 +12183,15 @@ mod test {
ensure_test_volume(&disk, bucket).await;
let payload = Bytes::from_static(b"part payload");
let meta = Bytes::from_static(b"part metadata");
let meta = Bytes::from(
ObjectPartInfo {
number: 1,
size: payload.len(),
..Default::default()
}
.marshal_msg()
.expect("legacy part metadata"),
);
disk.write_all(tmp_volume, "upload/part.1", payload.clone())
.await
.expect("source part should be written");
@@ -12178,7 +12281,15 @@ mod test {
"regression path must cross the traditional Windows MAX_PATH boundary: {deepest_marker:?}"
);
let payload = Bytes::from_static(b"part payload");
let meta = Bytes::from_static(b"part metadata");
let meta = Bytes::from(
ObjectPartInfo {
number: 1,
size: payload.len(),
..Default::default()
}
.marshal_msg()
.expect("legacy part metadata"),
);
disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, payload.clone())
.await
.expect("source part should be written");
@@ -12207,7 +12318,15 @@ mod test {
);
let replacement_payload = Bytes::from_static(b"replacement part payload");
let replacement_meta = Bytes::from_static(b"replacement part metadata");
let replacement_meta = Bytes::from(
ObjectPartInfo {
number: 1,
size: replacement_payload.len(),
..Default::default()
}
.marshal_msg()
.expect("legacy replacement metadata"),
);
disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, replacement_payload.clone())
.await
.expect("replacement source part should be written");
@@ -12272,9 +12391,23 @@ mod test {
.await
.expect("old part metadata should be staged");
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", Bytes::from_static(b"new metadata"))
.await
.expect("part transaction should be prepared");
disk.prepare_part_transaction(
"tmp",
"upload/part.1",
"bucket",
"object/part.1",
Bytes::from(
ObjectPartInfo {
number: 1,
size: 8,
..Default::default()
}
.marshal_msg()
.expect("legacy part metadata"),
),
)
.await
.expect("part transaction should be prepared");
disk.rename_file("tmp", "upload/part.1", "bucket", "object/part.1")
.await
.expect("data publication should succeed");
@@ -12296,6 +12429,139 @@ mod test {
);
}
#[tokio::test]
async fn shard_integrity_part_transaction_keeps_only_the_settled_generation() {
use crate::io_support::shard_integrity::IntegrityBuilder;
use rustfs_filemeta::shard_integrity::IntegrityLayout;
for action in [PartTransactionAction::Commit, PartTransactionAction::Rollback] {
let dir = tempfile::tempdir().expect("fixture");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("path")).expect("endpoint");
let disk = LocalDisk::new(&endpoint, false).await.expect("disk");
ensure_test_volume(&disk, "tmp").await;
ensure_test_volume(&disk, "bucket").await;
let mut parts = Vec::new();
for (volume, directory, byte) in [("bucket", "object", b'a'), ("tmp", "upload", b'b')] {
let mut builder =
IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 1).expect("builder");
let shard = [byte; 4];
builder.push([shard.as_slice(); 4].into_iter()).await.expect("stripe");
let prepared = builder.finish(8).await.expect("proof");
disk.write_all(
volume,
&format!("{directory}/{}", prepared.part.file_name()),
prepared.inline_bytes().expect("index"),
)
.await
.expect("index file");
disk.write_all(volume, &format!("{directory}/part.1"), Bytes::copy_from_slice(&shard))
.await
.expect("data");
let part = ObjectPartInfo {
number: 1,
size: 8,
integrity: Some(prepared.part),
..Default::default()
};
let meta = Bytes::from(part.marshal_msg().expect("metadata"));
disk.write_all(volume, &format!("{directory}/part.1.meta"), meta.clone())
.await
.expect("part meta");
parts.push((part, meta));
}
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", parts[1].1.clone())
.await
.expect("prepare");
for (part, _) in &parts {
assert!(
disk.read_all("bucket", &format!("object/{}", part.integrity.as_ref().expect("proof").file_name()))
.await
.is_ok(),
"both generations survive preparation"
);
}
disk.rename_part("tmp", "upload/part.1", "bucket", "object/part.1", parts[1].1.clone())
.await
.expect("publish");
disk.settle_part_transaction("bucket", "object/part.1", action)
.await
.expect("settle");
let retained = usize::from(action == PartTransactionAction::Commit);
assert_eq!(
disk.read_all("bucket", "object/part.1.meta").await.expect("settled metadata"),
parts[retained].1
);
for (index, (part, _)) in parts.iter().enumerate() {
let exists = disk
.read_all("bucket", &format!("object/{}", part.integrity.as_ref().expect("proof").file_name()))
.await
.is_ok();
assert_eq!(exists, index == retained, "obsolete proof is reclaimed after settlement");
}
}
}
#[tokio::test]
async fn shard_integrity_settlement_never_deletes_another_parts_index() {
use crate::io_support::shard_integrity::IntegrityBuilder;
use rustfs_filemeta::shard_integrity::IntegrityLayout;
let dir = tempfile::tempdir().expect("fixture");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("path")).expect("endpoint");
let disk = LocalDisk::new(&endpoint, false).await.expect("disk");
ensure_test_volume(&disk, "tmp").await;
ensure_test_volume(&disk, "bucket").await;
let mut builder = IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 2).expect("builder");
builder.push([b"data".as_slice(); 4].into_iter()).await.expect("stripe");
let prepared = builder.finish(8).await.expect("index");
let other_index = format!("object/{}", prepared.part.file_name());
let index_bytes = prepared.inline_bytes().expect("index bytes");
disk.write_all("bucket", &other_index, index_bytes.clone())
.await
.expect("part 2 index");
let corrupt = ObjectPartInfo {
number: 1,
size: 8,
integrity: Some(prepared.part),
..Default::default()
};
disk.write_all("bucket", "object/part.1", Bytes::from_static(b"old data"))
.await
.expect("old data");
disk.write_all(
"bucket",
"object/part.1.meta",
Bytes::from(corrupt.marshal_msg().expect("misdirected metadata")),
)
.await
.expect("corrupt old metadata");
disk.write_all("tmp", "upload/part.1", Bytes::from_static(b"new data"))
.await
.expect("new data");
let replacement = Bytes::from(
ObjectPartInfo {
number: 1,
size: 8,
..Default::default()
}
.marshal_msg()
.expect("replacement"),
);
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", replacement.clone())
.await
.expect("prepare");
disk.rename_part("tmp", "upload/part.1", "bucket", "object/part.1", replacement)
.await
.expect("publish");
disk.settle_part_transaction("bucket", "object/part.1", PartTransactionAction::Commit)
.await
.expect("settle");
assert_eq!(
disk.read_all("bucket", &other_index)
.await
.expect("unrelated part index retained"),
index_bytes
);
}
struct BlockingScanWriter {
entered_tx: Option<tokio::sync::oneshot::Sender<()>>,
}
+7 -1
View File
@@ -304,7 +304,13 @@ impl LocalDisk {
let data_dir = fi.data_dir.unwrap_or_default();
fi.parts
.iter()
.map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number))
.flat_map(|part| {
std::iter::once(format!("{dst_path}/{data_dir}/part.{}", part.number)).chain(
part.integrity
.as_ref()
.map(|integrity| format!("{dst_path}/{data_dir}/{}", integrity.file_name())),
)
})
.collect()
};
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
+10
View File
@@ -577,6 +577,13 @@ impl DiskAPI for Disk {
}
}
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
match self {
Disk::Local(disk) => disk.rename_file_durable(src_volume, src_path, dst_volume, dst_path).await,
Disk::Remote(disk) => disk.rename_file_durable(src_volume, src_path, dst_volume, dst_path).await,
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
match self {
@@ -1175,6 +1182,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<FileWriter>;
// ReadFileStream
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
async fn rename_file_durable(&self, _src_volume: &str, _src_path: &str, _dst_volume: &str, _dst_path: &str) -> Result<()> {
Err(DiskError::MethodNotAllowed)
}
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()>;
async fn prepare_part_transaction(
&self,
@@ -102,6 +102,7 @@ pin_project! {
chunks: Vec<bytes::Bytes>,
skip_verify: bool,
last_verify_duration: Duration,
integrity: Option<crate::io_support::shard_integrity::ShardVerifier>,
}
}
@@ -119,6 +120,7 @@ where
chunks: Vec::new(),
skip_verify,
last_verify_duration: Duration::ZERO,
integrity: None,
}
}
@@ -126,6 +128,24 @@ where
self.last_verify_duration
}
pub(crate) fn integrity_proof(&self) -> Option<std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>> {
self.integrity
.as_ref()
.map(crate::io_support::shard_integrity::ShardVerifier::proof)
}
pub(crate) fn set_integrity(&mut self, verifier: crate::io_support::shard_integrity::ShardVerifier) -> std::io::Result<()> {
if self.hash_algo != HashAlgorithm::HighwayHash256S {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"unsupported protected shard framing",
));
}
self.integrity = Some(verifier);
self.skip_verify = false;
Ok(())
}
#[cfg(test)]
pub(crate) fn inner_ref(&self) -> &R {
&self.inner
@@ -152,6 +172,9 @@ where
let need = self.hash_algo.size() + want;
self.read_scratch_block(need, want).await?;
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
if let Some(integrity) = &mut self.integrity {
integrity.verify(data).await?;
}
out.copy_from_slice(data);
self.last_verify_duration = verify;
Ok(want)
@@ -263,6 +286,18 @@ where
///
/// On return `out.len()` has grown by exactly the returned count.
pub async fn read_appending(&mut self, out: &mut Vec<u8>, want: usize) -> std::io::Result<usize> {
let start = out.len();
let count = self.read_appending_frame(out, want).await?;
if let Some(integrity) = &mut self.integrity
&& let Err(error) = integrity.verify(&out[start..]).await
{
out.truncate(start);
return Err(error);
}
Ok(count)
}
async fn read_appending_frame(&mut self, out: &mut Vec<u8>, want: usize) -> std::io::Result<usize> {
use bytes::BufMut as _;
use tokio::io::AsyncReadExt as _;
@@ -768,6 +803,7 @@ impl AsyncWrite for CustomWriter {
pub struct BitrotWriterWrapper {
bitrot_writer: BitrotWriter<CustomWriter>,
writer_type: WriterType,
integrity: Option<crate::io_support::shard_integrity::ShardVerifier>,
}
/// Enum to track the type of writer we're using
@@ -801,14 +837,22 @@ impl BitrotWriterWrapper {
Self {
bitrot_writer: BitrotWriter::new(writer, shard_size, checksum_algo),
writer_type,
integrity: None,
}
}
/// Write data to the bitrot writer
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if let Some(integrity) = &mut self.integrity {
integrity.verify(buf).await?;
}
self.bitrot_writer.write(buf).await
}
pub(crate) fn set_integrity(&mut self, verifier: crate::io_support::shard_integrity::ShardVerifier) {
self.integrity = Some(verifier);
}
pub async fn shutdown(&mut self) -> std::io::Result<()> {
self.bitrot_writer.shutdown().await
}
@@ -591,6 +591,7 @@ pub(crate) struct ParallelReader<R> {
// never consumes the unopened reader reserved for a later stripe.
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
stripe_index: usize,
integrity: Option<std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
}
}
@@ -782,6 +783,7 @@ where
.map(|index| !demand_bound_lockstep || index < e.data_shards)
.collect();
ParallelReader {
integrity: readers.iter().flatten().find_map(BitrotReader::integrity_proof),
readers,
offset,
shard_size,
@@ -1998,11 +2000,28 @@ where
R: crate::erasure::coding::ShardSource,
{
async fn read_next_stripe(&mut self) -> Box<StripeReadState> {
let stripe = self.offset.checked_div(self.shard_size);
let mut state = self
.stripe_state
.take()
.unwrap_or_else(|| Box::new(StripeReadState::with_slot_count(self.readers.len(), self.data_shards)));
self.read_into_state(&mut state).await;
if state.can_decode()
&& let Some(proof) = &self.integrity
{
let result = match stripe {
Some(stripe) => proof.reconstruction_proof(stripe, state.shards_mut()).await,
None => Err(io::Error::new(ErrorKind::InvalidData, "invalid integrity stripe offset")),
};
match result {
Ok(proof) => state.integrity = proof,
Err(_) => {
let (shards, errors) = state.parts_mut();
shards.iter_mut().for_each(|shard| *shard = None);
errors.fill(Some(Error::FileCorrupt));
}
}
}
state
}
@@ -2336,6 +2355,8 @@ impl Erasure {
ret_err: &mut Option<std::io::Error>,
stage_metrics_enabled: bool,
require_surplus_source: bool,
integrity: Option<&std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
request_offset: usize,
) -> StripeFlow
where
W: AsyncWrite + Send + Sync + Unpin,
@@ -2373,6 +2394,20 @@ impl Erasure {
// missing data shard and an extra source shard was available, verify
// the reconstructed data against that source before streaming bytes.
let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let proof = if let Some(integrity) = integrity {
match integrity
.reconstruction_proof((request_offset + *written) / self.block_size, shards)
.await
{
Ok(proof) => proof,
Err(error) => {
*ret_err = Some(error);
return StripeFlow::Stop;
}
}
} else {
None
};
let decode_result = if require_surplus_source {
self.decode_data_with_reconstruction_verification_for_lockstep(shards)
} else {
@@ -2397,6 +2432,12 @@ impl Erasure {
}
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
if let Some(proof) = proof
&& let Err(error) = proof.verify(shards)
{
*ret_err = Some(error);
return StripeFlow::Stop;
}
let emit_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let n = match write_data_blocks(writer, shards, self.data_shards, block_offset, block_length).await {
Ok(n) => {
@@ -2510,6 +2551,8 @@ impl Erasure {
.with_deferred_parity_handles(deferred_handles)
.with_deferred_parity_reopeners(deferred_reopeners);
let integrity = reader.integrity.clone();
let start = offset / self.block_size;
let end = end_offset.saturating_sub(1) / self.block_size;
@@ -2615,6 +2658,8 @@ impl Erasure {
&mut ret_err,
stage_metrics_enabled,
require_surplus_source,
integrity.as_ref(),
offset,
);
tokio::pin!(read_fut);
tokio::pin!(emit_fut);
@@ -2663,6 +2708,8 @@ impl Erasure {
&mut ret_err,
stage_metrics_enabled,
reader.demand_bound_lockstep,
integrity.as_ref(),
offset,
)
.await
{
@@ -2704,6 +2751,8 @@ impl Erasure {
&mut ret_err,
stage_metrics_enabled,
reader.demand_bound_lockstep,
integrity.as_ref(),
offset,
)
.await
{
@@ -816,6 +816,7 @@ where
};
rustfs_io_metrics::record_get_object_reconstruct_outcome(metrics_path, engine.engine_name(), reconstruct_outcome);
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
state.verify_reconstructed_integrity()?;
if state.shards_mut().len() < engine.data_shards() {
return Err(io::Error::new(
+129 -14
View File
@@ -19,6 +19,7 @@ use crate::disk::error_reduce::{
use crate::erasure::coding::BitrotWriterWrapper;
use crate::erasure::coding::Erasure;
use crate::erasure::coding::erasure::EncodedBlock;
use crate::io_support::shard_integrity::{IntegrityBuilder, PreparedIntegrity};
use crate::runtime::sources as runtime_sources;
use bytes::{Bytes, BytesMut};
use futures::StreamExt;
@@ -43,6 +44,13 @@ const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 32;
const DEFAULT_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: usize = 4;
const DEFAULT_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST: bool = false;
pub(crate) enum IntegrityEncodeMode {
Inline(usize),
SingleBlock(usize),
Streaming,
Batched,
}
/// Cached value of `RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES` env var.
/// Read once at first use via `OnceLock` to avoid per-encode syscall.
static CACHED_MAX_INFLIGHT_BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
@@ -297,6 +305,7 @@ impl Default for WriteProgressPolicy {
pub(crate) struct MultiWriter<'a> {
writers: &'a mut [Option<BitrotWriterWrapper>],
integrity: Option<&'a mut IntegrityBuilder>,
write_quorum: usize,
errs: Vec<Option<Error>>,
policy: WriteProgressPolicy,
@@ -314,6 +323,7 @@ impl<'a> MultiWriter<'a> {
let length = writers.len();
MultiWriter {
writers,
integrity: None,
write_quorum,
errs: vec![None; length],
policy,
@@ -375,6 +385,10 @@ impl<'a> MultiWriter<'a> {
}
async fn write_block(&mut self, block: &EncodedBlock) -> std::io::Result<()> {
if let Some(integrity) = &mut self.integrity {
// Commit every coding index, including disks absent from the write.
integrity.push(block.shards()).await?;
}
self.write_shards(block.shards()).await
}
@@ -508,6 +522,63 @@ impl<'a> MultiWriter<'a> {
}
impl Erasure {
pub(crate) async fn encode_with_shard_integrity<R>(
self: Arc<Self>,
reader: R,
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
part_number: usize,
mode: IntegrityEncodeMode,
protected: bool,
) -> std::io::Result<(R, usize, Option<Vec<Bytes>>, Option<PreparedIntegrity>)>
where
R: AsyncRead + Send + Sync + Unpin + 'static,
{
let mut integrity = if protected {
let layout = rustfs_filemeta::shard_integrity::IntegrityLayout::new(
self.data_shards,
self.parity_shards,
self.block_size,
self.uses_legacy_codec(),
)
.map_err(std::io::Error::other)?;
Some(IntegrityBuilder::new(layout, part_number)?)
} else {
None
};
let (reader, size, inline) = match mode {
IntegrityEncodeMode::Inline(hint) => {
let (reader, size, shards) = self
.encode_inline_shards_with_integrity(reader, hint, integrity.as_mut())
.await?;
(reader, size, Some(shards))
}
IntegrityEncodeMode::SingleBlock(hint) => {
let (reader, size) = self
.encode_small_direct(reader, writers, quorum, true, hint, integrity.as_mut())
.await?;
(reader, size, None)
}
IntegrityEncodeMode::Streaming => {
let (reader, size) = self
.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest(), integrity.as_mut())
.await?;
(reader, size, None)
}
IntegrityEncodeMode::Batched => {
let (reader, size) = self
.encode_batched_with_integrity(reader, writers, quorum, integrity.as_mut())
.await?;
(reader, size, None)
}
};
let integrity = match integrity {
Some(integrity) => Some(integrity.finish(size).await?),
None => None,
};
Ok((reader, size, inline, integrity))
}
async fn encode_block(self: Arc<Self>, encode_buf: Vec<u8>, len: usize) -> std::io::Result<(EncodedBlock, Vec<u8>)> {
let encode_stage_start = stage_timer_if_enabled();
let encode_once = move || {
@@ -564,6 +635,7 @@ impl Erasure {
quorum: usize,
require_single_block: bool,
size_hint: usize,
integrity: Option<&mut IntegrityBuilder>,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin,
@@ -596,6 +668,7 @@ impl Erasure {
let block = self.encode_data_owned_block(buf)?;
let mut mw = MultiWriter::new(writers, quorum);
mw.integrity = integrity;
mw.write_block(&block).await?;
mw.shutdown().await?;
Ok((reader, total))
@@ -605,10 +678,23 @@ impl Erasure {
/// The returned bytes are the same `[hash][shard]` representation produced
/// by `BitrotWriter`, ready to be embedded in each disk's staged `xl.meta`.
#[hotpath::measure(impl_type = "Erasure")]
#[cfg(test)]
pub(crate) async fn encode_inline_shards_with_size_hint<R>(
self: Arc<Self>,
reader: R,
size_hint: usize,
) -> std::io::Result<(R, usize, Vec<Bytes>)>
where
R: AsyncRead + Send + Sync + Unpin,
{
self.encode_inline_shards_with_integrity(reader, size_hint, None).await
}
async fn encode_inline_shards_with_integrity<R>(
self: Arc<Self>,
mut reader: R,
size_hint: usize,
integrity: Option<&mut IntegrityBuilder>,
) -> std::io::Result<(R, usize, Vec<Bytes>)>
where
R: AsyncRead + Send + Sync + Unpin,
@@ -622,6 +708,9 @@ impl Erasure {
}
let block = self.encode_data_owned_block(buf)?;
if let Some(integrity) = integrity {
integrity.push(block.shards()).await?;
}
let mut inline_shards = Vec::with_capacity(block.shards().len());
for shard in block.shards() {
let hash = HashAlgorithm::HighwayHash256S.hash_encode(shard);
@@ -645,7 +734,7 @@ impl Erasure {
R: AsyncRead + Send + Sync + Unpin + 'static,
{
let use_bytesmut_ingest = use_bytesmut_ingest();
self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest)
self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest, None)
.await
}
@@ -659,6 +748,7 @@ impl Erasure {
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
use_bytesmut_ingest: bool,
integrity: Option<&mut IntegrityBuilder>,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin + 'static,
@@ -766,6 +856,7 @@ impl Erasure {
}));
let mut writers = MultiWriter::new(writers, quorum);
writers.integrity = integrity;
let mut write_err = None;
@@ -808,10 +899,23 @@ impl Erasure {
#[hotpath::measure(impl_type = "Erasure")]
pub async fn encode_batched<R>(
self: Arc<Self>,
reader: R,
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin + 'static,
{
self.encode_batched_with_integrity(reader, writers, quorum, None).await
}
async fn encode_batched_with_integrity<R>(
self: Arc<Self>,
mut reader: R,
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
integrity: Option<&mut IntegrityBuilder>,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin + 'static,
@@ -892,6 +996,7 @@ impl Erasure {
}));
let mut writers = MultiWriter::new(writers, quorum);
writers.integrity = integrity;
let mut write_err = None;
loop {
@@ -946,7 +1051,8 @@ impl Erasure {
R: AsyncRead + Send + Sync + Unpin,
{
let size_hint = self.block_size;
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
.await
}
/// Size-aware inline fast path. `size_hint` only controls the bounded initial
@@ -962,7 +1068,8 @@ impl Erasure {
where
R: AsyncRead + Send + Sync + Unpin,
{
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
.await
}
/// Fast path for single-block non-inline objects: avoids the producer/consumer
@@ -978,7 +1085,7 @@ impl Erasure {
R: AsyncRead + Send + Sync + Unpin,
{
let size_hint = self.block_size;
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
}
/// Size-aware single-block fast path. `size_hint` only controls the bounded
@@ -994,7 +1101,7 @@ impl Erasure {
where
R: AsyncRead + Send + Sync + Unpin,
{
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
}
}
@@ -1381,10 +1488,10 @@ mod tests {
let encode = match pipeline {
EncodePipeline::Vec => {
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await })
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await })
}
EncodePipeline::BytesMut => {
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await })
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await })
}
EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
};
@@ -1443,8 +1550,8 @@ mod tests {
))];
let result = match pipeline {
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await,
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await,
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
};
@@ -1525,7 +1632,11 @@ mod tests {
BLOCK_SIZE,
))];
let erasure_for_task = erasure.clone();
let encode = tokio::spawn(async move { erasure_for_task.encode_with_ingest_mode(reader, &mut writers, 1, false).await });
let encode = tokio::spawn(async move {
erasure_for_task
.encode_with_ingest_mode(reader, &mut writers, 1, false, None)
.await
});
tokio::time::timeout(Duration::from_secs(1), writer_entered)
.await
@@ -2081,7 +2192,7 @@ mod tests {
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
let (_reader, written) = erasure
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true)
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
.await
.expect("BytesMut ingest path should encode the streaming payload");
@@ -2110,7 +2221,7 @@ mod tests {
let reader = tokio::io::BufReader::new(Cursor::new(vec![0x5a; block_size * 2]));
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await;
let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await;
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
let (_reader, written) = result.expect("bytesmut streaming encode should complete");
@@ -2728,10 +2839,14 @@ mod tests {
let (_reader, total) = match pipeline {
EncodePipeline::Vec => {
erasure
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false)
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false, None)
.await
}
EncodePipeline::BytesMut => {
erasure
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
.await
}
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true).await,
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, DATA_SHARDS).await,
}
.expect("encode should succeed");
@@ -657,6 +657,10 @@ pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize {
}
impl Erasure {
pub(crate) fn uses_legacy_codec(&self) -> bool {
self.uses_legacy
}
/// Create a new Erasure instance
///
/// # Arguments
+10
View File
@@ -252,6 +252,7 @@ impl DeferredObjectReader {
DeferredReaderStripeHandle {
state: Arc::clone(&self.state),
stripe_stride,
advanced: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
}
}
}
@@ -274,9 +275,14 @@ impl DeferredObjectReader {
pub(crate) struct DeferredReaderStripeHandle {
state: Arc<Mutex<DeferredObjectReaderState>>,
stripe_stride: usize,
advanced: Arc<std::sync::atomic::AtomicUsize>,
}
impl DeferredReaderStripeHandle {
pub(crate) fn integrity_position(&self) -> Arc<std::sync::atomic::AtomicUsize> {
Arc::clone(&self.advanced)
}
/// Advance the pending source by `stripes` full stripes.
///
/// Returns `false` when the reader has already been opened (or failed):
@@ -297,8 +303,12 @@ impl DeferredReaderStripeHandle {
let Some(offset) = source.offset.checked_add(delta) else {
return false;
};
let Some(advanced) = self.advanced.load(std::sync::atomic::Ordering::Acquire).checked_add(stripes) else {
return false;
};
source.offset = offset;
source.length = source.length.saturating_sub(delta);
self.advanced.store(advanced, std::sync::atomic::Ordering::Release);
true
}
_ => false,
+1
View File
@@ -17,3 +17,4 @@
pub(crate) mod bitrot;
pub(crate) mod compress;
pub(crate) mod rio;
pub(crate) mod shard_integrity;
@@ -0,0 +1,861 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::{DiskAPI, DiskStore};
use bytes::Bytes;
use futures::{StreamExt, stream::FuturesUnordered};
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::shard_integrity::{
IntegrityLayout, MAX_INLINE_PROOF_BYTES, PartIntegrity, SUFFIX_INLINE_INTEGRITY, node_digest,
};
use rustfs_utils::http::{contains_key_str, get_consistent_str, insert_str};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::sync::Mutex;
const MEMORY_LIMIT: usize = 1024 * 1024;
const COPY_CHUNK_SIZE: usize = 64 * 1024;
fn corrupt() -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, "shard integrity verification failed")
}
/// Only encoded digest rows are retained. Payload bytes never enter the spool.
/// Large parts spill to an anonymous temporary file; cancellation drops it.
pub(crate) struct IntegrityBuilder {
part: PartIntegrity,
rows: Vec<u8>,
spill: Option<tokio::fs::File>,
}
impl IntegrityBuilder {
pub(crate) fn new(layout: IntegrityLayout, number: usize) -> io::Result<Self> {
Ok(Self {
part: PartIntegrity::new(layout, number).map_err(|_| corrupt())?,
rows: Vec::new(),
spill: None,
})
}
pub(crate) async fn push<'a>(&mut self, shards: impl ExactSizeIterator<Item = &'a [u8]>) -> io::Result<()> {
if shards.len() != self.part.layout.shards() {
return Err(corrupt());
}
let mut row = Vec::with_capacity(shards.len() * 32);
for (index, payload) in shards.enumerate() {
row.extend_from_slice(
&self
.part
.shard_digest(self.part.stripes, index, payload)
.map_err(|_| corrupt())?,
);
}
self.append(&row).await?;
self.part.stripes = self.part.stripes.checked_add(1).ok_or_else(corrupt)?;
Ok(())
}
async fn append(&mut self, bytes: &[u8]) -> io::Result<()> {
self.rows.extend_from_slice(bytes);
if self.rows.len() >= MEMORY_LIMIT {
if self.spill.is_none() {
let file = tokio::task::spawn_blocking(tempfile::tempfile)
.await
.map_err(io::Error::other)??;
self.spill = Some(tokio::fs::File::from_std(file));
}
if let Some(file) = &mut self.spill {
file.write_all(&self.rows).await?;
self.rows.clear();
}
}
Ok(())
}
async fn into_spool(self) -> io::Result<Spool> {
if let Some(mut file) = self.spill {
file.write_all(&self.rows).await?;
file.flush().await?;
Ok(Spool::File(file.into_std().await))
} else {
Ok(Spool::Memory(self.rows))
}
}
pub(crate) async fn finish(mut self, size: usize) -> io::Result<PreparedIntegrity> {
self.part.size = size.try_into().map_err(|_| corrupt())?;
self.part.validate().map_err(|_| corrupt())?;
let part = self.part.clone();
let rows = self.into_spool().await?;
if part.stripes < 16 {
build_index(part, rows)
} else {
tokio::task::spawn_blocking(move || build_index(part, rows))
.await
.map_err(io::Error::other)?
}
}
}
enum Spool {
Memory(Vec<u8>),
File(std::fs::File),
}
impl Spool {
fn append(&mut self, bytes: &[u8]) -> io::Result<()> {
if let Self::Memory(buffer) = self
&& buffer.len() + bytes.len() > MEMORY_LIMIT
{
let mut file = tempfile::tempfile()?;
file.write_all(buffer)?;
*self = Self::File(file);
}
match self {
Self::Memory(buffer) => buffer.extend_from_slice(bytes),
Self::File(file) => {
file.seek(SeekFrom::End(0))?;
file.write_all(bytes)?;
}
}
Ok(())
}
fn read_at(&mut self, offset: u64, out: &mut [u8]) -> io::Result<()> {
match self {
Self::Memory(buffer) => {
let offset = usize::try_from(offset).map_err(|_| corrupt())?;
let end = offset.checked_add(out.len()).ok_or_else(corrupt)?;
out.copy_from_slice(buffer.get(offset..end).ok_or_else(corrupt)?);
Ok(())
}
Self::File(file) => {
file.seek(SeekFrom::Start(offset))?;
file.read_exact(out)
}
}
}
}
fn build_index(mut part: PartIntegrity, mut rows: Spool) -> io::Result<PreparedIntegrity> {
let leaves = u64::from(part.stripes.max(1)).next_power_of_two();
let mut tree = Spool::Memory(Vec::new());
let row_size = part.layout.shards() * 32;
let mut row = vec![0; row_size];
for stripe in 0..leaves {
let stripe_number = u32::try_from(stripe).map_err(|_| corrupt())?;
let digest = if stripe < u64::from(part.stripes) {
rows.read_at(stripe * u64::try_from(row_size).map_err(|_| corrupt())?, &mut row)?;
part.leaf_digest(stripe_number, &row).map_err(|_| corrupt())?
} else {
part.padding_digest(stripe_number)
};
tree.append(&digest)?;
}
let mut levels = vec![0u64];
let mut count = leaves;
let mut level_offset = 0;
let mut next_offset = leaves * 32;
let mut pair = [0; 64];
while count > 1 {
for pair_number in 0..count / 2 {
tree.read_at(level_offset + pair_number * 64, &mut pair)?;
let left = pair[..32].try_into().map_err(|_| corrupt())?;
let right = pair[32..].try_into().map_err(|_| corrupt())?;
tree.append(&node_digest(left, right))?;
}
levels.push(next_offset);
level_offset = next_offset;
count /= 2;
next_offset += count * 32;
}
let mut root = [0; 32];
tree.read_at(level_offset, &mut root)?;
part.root = part.root_digest(&root);
let mut index = Spool::Memory(Vec::new());
index.append(&part.index_header())?;
let mut sibling = [0; 32];
for stripe in 0..part.stripes {
rows.read_at(u64::from(stripe) * u64::try_from(row_size).map_err(|_| corrupt())?, &mut row)?;
index.append(&row)?;
let mut position = u64::from(stripe);
for &offset in levels.iter().take(levels.len() - 1) {
tree.read_at(offset + (position ^ 1) * 32, &mut sibling)?;
index.append(&sibling)?;
position >>= 1;
}
}
Ok(PreparedIntegrity { part, index })
}
pub(crate) struct PreparedIntegrity {
pub(crate) part: PartIntegrity,
index: Spool,
}
impl PreparedIntegrity {
pub(crate) async fn copy_from(proof: &PartProofReader) -> io::Result<Self> {
let mut builder = IntegrityBuilder {
part: proof.part.clone(),
rows: proof.part.index_header().to_vec(),
spill: None,
};
for stripe in 0..proof.part.stripes {
builder.append(&proof.record(stripe).await?).await?;
}
let index = builder.into_spool().await?;
Ok(Self {
part: proof.part.clone(),
index,
})
}
pub(crate) fn inline_bytes(&self) -> io::Result<Bytes> {
match &self.index {
Spool::Memory(bytes) if bytes.len() <= MAX_INLINE_PROOF_BYTES => Ok(Bytes::copy_from_slice(bytes)),
_ => Err(corrupt()),
}
}
pub(crate) fn set_inline_metadata(&self, fi: &mut FileInfo) -> io::Result<()> {
insert_str(
&mut fi.metadata,
SUFFIX_INLINE_INTEGRITY,
base64_simd::STANDARD.encode_to_string(self.inline_bytes()?),
);
Ok(())
}
/// A disk survives only if BOTH payload and index writes succeed. The caller
/// supplies the surviving payload disks and rechecks the commit quorum.
pub(crate) async fn write(
self,
disks: &mut [Option<DiskStore>],
original_volume: &str,
volume: &str,
directory: &str,
) -> io::Result<PartIntegrity> {
let length = self.part.index_size().map_err(|_| corrupt())?;
let path = format!("{directory}/{}", self.part.file_name());
let mut writers = futures::future::join_all(disks.iter().map(|disk| async {
match disk {
Some(disk) => disk
.create_file(original_volume, volume, &path, i64::try_from(length).map_err(|_| corrupt())?)
.await
.map(Some)
.map_err(io::Error::other),
None => Ok(None),
}
}))
.await
.into_iter()
.map(Result::ok)
.map(Option::flatten)
.collect::<Vec<_>>();
let mut source: Box<dyn tokio::io::AsyncRead + Unpin + Send> = match self.index {
Spool::Memory(bytes) => Box::new(std::io::Cursor::new(bytes)),
Spool::File(mut file) => {
// The file has a single owner; duplicated descriptors share offsets.
file = tokio::task::spawn_blocking(move || {
file.seek(SeekFrom::Start(0))?;
io::Result::Ok(file)
})
.await
.map_err(io::Error::other)??;
Box::new(tokio::fs::File::from_std(file))
}
};
let mut buffer = vec![0; COPY_CHUNK_SIZE.min(length)];
loop {
let n = source.read(&mut buffer).await?;
if n == 0 {
break;
}
futures::future::join_all(writers.iter_mut().map(|writer| {
let bytes = &buffer[..n];
async move {
if let Some(inner) = writer
&& !matches!(
tokio::time::timeout(std::time::Duration::from_secs(60), inner.write_all(bytes)).await,
Ok(Ok(()))
)
{
*writer = None;
}
}
}))
.await;
}
futures::future::join_all(writers.iter_mut().zip(disks.iter_mut()).map(|(writer, disk)| async move {
let success = if let Some(writer) = writer {
matches!(
tokio::time::timeout(std::time::Duration::from_secs(60), writer.shutdown()).await,
Ok(Ok(()))
)
} else {
false
};
if !success {
*disk = None;
}
}))
.await;
Ok(self.part)
}
}
/// A shared, two-stripe cache coalesces the N readers' proof fetches. The only
/// async lock is held across a bounded read from at most N replicas; it never
/// nests another lock. A missing local index may use any authenticated replica.
pub(crate) struct PartProofReader {
pub(crate) part: PartIntegrity,
sources: Vec<(DiskStore, String)>,
volume: String,
inline: Option<Bytes>,
cached: Mutex<ProofCache>,
}
#[derive(Default)]
struct ProofCache {
records: std::collections::VecDeque<(u32, Bytes)>,
preferred: usize,
}
impl PartProofReader {
pub(crate) async fn reconstruction_proof(
&self,
stripe: usize,
shards: &[Option<Vec<u8>>],
) -> io::Result<Option<ReconstructionProof>> {
let mut missing = 0u16;
for (index, shard) in shards.iter().take(usize::from(self.part.layout.data)).enumerate() {
if shard.is_none() {
missing |= 1 << index;
}
}
if missing == 0 {
return Ok(None);
}
let stripe = u32::try_from(stripe).map_err(|_| corrupt())?;
Ok(Some(ReconstructionProof {
part: self.part.clone(),
stripe,
record: self.record(stripe).await?,
missing,
}))
}
pub(crate) fn new(
part: PartIntegrity,
files: &[FileInfo],
disks: &[Option<DiskStore>],
volume: &str,
object: &str,
) -> io::Result<Arc<Self>> {
part.validate().map_err(|_| corrupt())?;
let mut sources = Vec::with_capacity(disks.len());
let mut inline = None;
for (file, disk) in files.iter().zip(disks) {
let part_index = usize::try_from(part.number)
.ok()
.and_then(|number| file.parts.binary_search_by_key(&number, |p| p.number).ok());
if part_index.and_then(|index| file.parts[index].integrity.as_ref()) != Some(&part) {
continue;
}
if inline.is_none() && contains_key_str(&file.metadata, SUFFIX_INLINE_INTEGRITY) {
let candidate = (|| {
let value = get_consistent_str(&file.metadata, SUFFIX_INLINE_INTEGRITY)?;
if value.len() > MAX_INLINE_PROOF_BYTES.div_ceil(3) * 4 {
return None;
}
let decoded = base64_simd::STANDARD.decode_to_vec(value).ok()?;
if decoded.len() != part.index_size().ok()? || !decoded.starts_with(&part.index_header()) {
return None;
}
for stripe in 0..part.stripes {
let offset = part.record_offset(stripe).ok()?;
part.verify_record(stripe, decoded.get(offset..offset + part.record_size())?)
.ok()?;
}
Some(Bytes::from(decoded))
})();
if let Some(candidate) = candidate {
inline = Some(candidate);
}
}
if let (Some(disk), Some(directory)) = (disk, file.data_dir.filter(|id| !id.is_nil())) {
sources.push((disk.clone(), format!("{object}/{directory}/{}", part.file_name())));
}
}
Ok(Arc::new(Self {
part,
sources,
volume: volume.to_owned(),
inline,
cached: Mutex::new(ProofCache::default()),
}))
}
pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
let mut cached = self.cached.lock().await;
if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
return Ok(bytes.clone());
}
let size = self.part.record_size();
let record = if let Some(inline) = &self.inline {
inline.slice(offset..offset + size)
} else {
let read = |index: usize| async move {
let (disk, path) = &self.sources[index];
let result = tokio::time::timeout(std::time::Duration::from_secs(15), async {
let mut reader = disk
.read_file_stream(&self.volume, path, offset, size)
.await
.map_err(io::Error::other)?;
let mut bytes = vec![0; size];
reader.read_exact(&mut bytes).await?;
self.part.verify_record(stripe, &bytes).map_err(|_| corrupt())?;
io::Result::Ok(Bytes::from(bytes))
})
.await;
(index, result)
};
if self.sources.is_empty() {
return Err(corrupt());
}
let preferred = cached.preferred % self.sources.len();
let mut pending = FuturesUnordered::new();
pending.push(read(preferred));
let first = tokio::select! {
result = pending.next() => result,
() = tokio::time::sleep(std::time::Duration::from_millis(50)) => None,
};
let mut found = match first {
Some((index, Ok(Ok(bytes)))) => Some((index, bytes)),
_ => None,
};
if found.is_none() {
for index in 0..self.sources.len() {
if index != preferred {
pending.push(read(index));
}
}
while let Some((index, result)) = pending.next().await {
if let Ok(Ok(bytes)) = result {
found = Some((index, bytes));
break;
}
}
}
let (index, bytes) = found.ok_or_else(corrupt)?;
cached.preferred = index;
bytes
};
self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
if cached.records.len() == 2 {
cached.records.pop_front();
}
cached.records.push_back((stripe, record.clone()));
Ok(record)
}
pub(crate) async fn verify(&self, stripe: u32, index: usize, payload: &[u8]) -> io::Result<()> {
let record = self.record(stripe).await?;
// Authentication of the shared record is cached; payload digests are
// always recomputed, including skip_verify and repaired-shard paths.
let expected = record.get(index * 32..(index + 1) * 32).ok_or_else(corrupt)?;
if self
.part
.shard_digest(stripe, index, payload)
.map_err(|_| corrupt())?
.as_slice()
!= expected
{
return Err(corrupt());
}
Ok(())
}
}
#[derive(Clone)]
pub(crate) struct ShardVerifier {
proof: Arc<PartProofReader>,
coding_index: usize,
first_stripe: u32,
consumed: u32,
advanced: Option<Arc<AtomicUsize>>,
}
impl ShardVerifier {
pub(crate) fn proof(&self) -> Arc<PartProofReader> {
Arc::clone(&self.proof)
}
pub(crate) fn new(
proof: Arc<PartProofReader>,
coding_index: usize,
first_stripe: usize,
advanced: Option<Arc<AtomicUsize>>,
) -> io::Result<Self> {
if coding_index >= proof.part.layout.shards() {
return Err(corrupt());
}
Ok(Self {
proof,
coding_index,
first_stripe: first_stripe.try_into().map_err(|_| corrupt())?,
consumed: 0,
advanced,
})
}
pub(crate) async fn verify(&mut self, payload: &[u8]) -> io::Result<()> {
let advanced = self.advanced.as_ref().map_or(0, |value| value.load(Ordering::Acquire));
let stripe = self
.first_stripe
.checked_add(u32::try_from(advanced).map_err(|_| corrupt())?)
.and_then(|value| value.checked_add(self.consumed))
.ok_or_else(corrupt)?;
self.proof.verify(stripe, self.coding_index, payload).await?;
self.consumed = self.consumed.checked_add(1).ok_or_else(corrupt)?;
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ReconstructionProof {
part: PartIntegrity,
stripe: u32,
record: Bytes,
missing: u16,
}
impl ReconstructionProof {
pub(crate) fn verify(&self, shards: &[Option<Vec<u8>>]) -> io::Result<()> {
for index in 0..usize::from(self.part.layout.data) {
if self.missing & (1 << index) == 0 {
continue;
}
let payload = shards.get(index).and_then(Option::as_deref).ok_or_else(corrupt)?;
let actual = self.part.shard_digest(self.stripe, index, payload).map_err(|_| corrupt())?;
if self.record.get(index * 32..(index + 1) * 32) != Some(actual.as_slice()) {
return Err(corrupt());
}
}
Ok(())
}
}
/// Deep scans verify stripes across all disks in lockstep so a single bounded
/// proof fetch serves all N payload checks. VerifyFile from an older peer is
/// deliberately not treated as evidence of this independent commitment.
pub(crate) async fn verify_deep_parts(
files: &[FileInfo],
disks: &[Option<DiskStore>],
expected: &FileInfo,
volume: &str,
object: &str,
statuses: &mut std::collections::HashMap<usize, Vec<usize>>,
) -> crate::disk::error::Result<()> {
use crate::disk::error::DiskError;
use crate::disk::{CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, conv_part_err_to_int};
use crate::io_support::bitrot::create_bitrot_reader_from_bytes;
let erasure = crate::erasure::coding::Erasure::try_new_with_options(
expected.erasure.data_blocks,
expected.erasure.parity_blocks,
expected.erasure.block_size,
expected.uses_legacy_checksum,
)
.map_err(DiskError::from)?;
let read_timeout = crate::disk::disk_store::get_object_disk_read_timeout();
for (part_index, part) in expected.parts.iter().enumerate() {
let commitment = part.integrity.as_ref().ok_or(DiskError::FileCorrupt)?;
let proof = PartProofReader::new(commitment.clone(), files, disks, volume, object).map_err(DiskError::from)?;
if commitment.stripes == 0 && commitment.root != commitment.root_digest(&commitment.padding_digest(0)) {
return Err(DiskError::FileCorrupt);
}
let part_status = statuses.get_mut(&part_index).ok_or(DiskError::FileCorrupt)?;
let length = erasure.shard_file_offset(0, part.size, part.size);
let mut readers = Vec::with_capacity(disks.len());
for (index, disk) in disks.iter().enumerate() {
if part_status[index] != CHECK_PART_UNKNOWN {
readers.push(None);
continue;
}
let file = &files[index];
if file.erasure.get_checksum_info(part.number).algorithm != rustfs_utils::HashAlgorithm::HighwayHash256S {
return Err(DiskError::BitrotHashAlgoInvalid);
}
let path = format!("{object}/{}/part.{}", file.data_dir.unwrap_or_default(), part.number);
let result = create_bitrot_reader_from_bytes(
file.data.clone(),
disk.as_ref(),
volume,
&path,
0,
length,
erasure.shard_size(),
rustfs_utils::HashAlgorithm::HighwayHash256S,
false,
false,
)
.await;
match result {
Ok(Some(mut reader)) => {
let coding_index = expected
.erasure
.distribution
.get(index)
.and_then(|index| index.checked_sub(1))
.ok_or(DiskError::FileCorrupt)?;
reader
.set_integrity(ShardVerifier::new(Arc::clone(&proof), coding_index, 0, None).map_err(DiskError::from)?)
.map_err(DiskError::from)?;
readers.push(Some(reader));
part_status[index] = CHECK_PART_SUCCESS;
}
Ok(None) => {
readers.push(None);
part_status[index] = CHECK_PART_FILE_CORRUPT;
}
Err(error) => {
readers.push(None);
part_status[index] = conv_part_err_to_int(&Some(error));
}
}
}
let mut remaining = length;
let mut buffers = vec![vec![0; erasure.shard_size().min(length)]; disks.len()];
while remaining > 0 {
let want = remaining.min(erasure.shard_size());
let results = futures::future::join_all(readers.iter_mut().zip(&mut buffers).map(|(reader, buffer)| async move {
let Some(reader) = reader else { return Ok(()) };
let read = reader.read(&mut buffer[..want]);
if read_timeout.is_zero() {
read.await.map(|_| ())
} else {
tokio::time::timeout(read_timeout, read)
.await
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "integrity scan timed out"))?
.map(|_| ())
}
}))
.await;
for (index, result) in results.into_iter().enumerate() {
if result.is_err() {
part_status[index] = CHECK_PART_FILE_CORRUPT;
readers[index] = None;
}
}
remaining -= want;
}
}
Ok(())
}
async fn verify_index_file(disk: &DiskStore, volume: &str, path: &str, part: &PartIntegrity) -> io::Result<()> {
let size = part.index_size().map_err(|_| corrupt())?;
let source = disk.read_file_stream(volume, path, 0, size).await.map_err(io::Error::other)?;
let mut source = tokio::io::BufReader::with_capacity(COPY_CHUNK_SIZE, source);
let mut header = [0; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE];
source.read_exact(&mut header).await?;
if header != part.index_header() {
return Err(corrupt());
}
let mut record = vec![0; part.record_size()];
for stripe in 0..part.stripes {
source.read_exact(&mut record).await?;
part.verify_record(stripe, &record).map_err(|_| corrupt())?;
}
Ok(())
}
/// Restore only untrusted/missing proof replicas, under the existing root and
/// the caller's exclusive object lock. This does not rewrite payload or xl.meta.
pub(crate) async fn restore_proof_replicas(
expected: &FileInfo,
disks: &[Option<DiskStore>],
volume: &str,
object: &str,
) -> crate::disk::error::Result<usize> {
use crate::disk::DeleteOptions;
use crate::disk::RUSTFS_META_TMP_BUCKET;
use crate::disk::error::DiskError;
if contains_key_str(&expected.metadata, SUFFIX_INLINE_INTEGRITY) || expected.deleted || expected.is_remote() {
return Ok(0);
}
let directory = expected.data_dir.filter(|id| !id.is_nil()).ok_or(DiskError::FileCorrupt)?;
let mut repaired = 0;
for part in &expected.parts {
let part = part.integrity.as_ref().ok_or(DiskError::FileCorrupt)?;
let path = format!("{object}/{directory}/{}", part.file_name());
let checks = futures::future::join_all(disks.iter().map(|disk| async {
let Some(disk) = disk else { return true };
matches!(
tokio::time::timeout(std::time::Duration::from_secs(60), verify_index_file(disk, volume, &path, part)).await,
Ok(Ok(()))
)
}))
.await;
if checks.iter().all(|valid| *valid) {
continue;
}
let proof = PartProofReader {
part: part.clone(),
sources: disks.iter().flatten().map(|disk| (disk.clone(), path.clone())).collect(),
volume: volume.to_owned(),
inline: None,
cached: Mutex::new(ProofCache::default()),
};
let prepared = PreparedIntegrity::copy_from(&proof).await.map_err(DiskError::from)?;
let temporary = format!("integrity-{}", uuid::Uuid::new_v4());
let mut targets: Vec<_> = disks
.iter()
.zip(&checks)
.map(|(disk, valid)| if *valid { None } else { disk.clone() })
.collect();
let result: crate::disk::error::Result<()> = async {
prepared
.write(&mut targets, volume, RUSTFS_META_TMP_BUCKET, &temporary)
.await
.map_err(DiskError::from)?;
let source_path = format!("{temporary}/{}", part.file_name());
for (index, valid) in checks.iter().enumerate() {
if *valid {
continue;
}
let disk = targets[index].as_ref().ok_or(DiskError::ErasureWriteQuorum)?;
disk.rename_file_durable(RUSTFS_META_TMP_BUCKET, &source_path, volume, &path)
.await?;
repaired += 1;
}
Ok(())
}
.await;
for disk in disks.iter().flatten() {
let _ = disk
.delete(
RUSTFS_META_TMP_BUCKET,
&temporary,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await;
}
result?;
}
Ok(repaired)
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn integrity_index_rejects_donor_payload_record_position_and_generation() {
let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
builder.part.generation = uuid::Uuid::parse_str("00112233-4455-4677-8899-aabbccddeeff").expect("fixed generation");
let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
builder.push(shards.into_iter()).await.expect("first stripe");
builder.push(shards.into_iter()).await.expect("second stripe");
let prepared = builder.finish(16).await.expect("index");
let bytes = prepared.inline_bytes().expect("inline index");
let part = prepared.part;
// Frozen independently with Python hashlib from the v1 byte layout.
let expected_root = [
33, 254, 159, 226, 214, 235, 239, 175, 182, 132, 172, 81, 6, 234, 125, 187, 25, 98, 190, 6, 251, 241, 171, 33, 150,
188, 179, 250, 26, 60, 47, 198,
];
assert_eq!(part.root, expected_root);
let offset = part.record_offset(0).expect("offset");
let record = &bytes[offset..offset + part.record_size()];
part.verify_shard(0, 0, shards[0], record).expect("target payload");
assert!(part.verify_shard(0, 0, shards[1], record).is_err());
assert!(part.verify_record(1, record).is_err());
let mut other = part.clone();
other.generation = uuid::Uuid::new_v4();
assert!(other.verify_record(0, record).is_err());
for position in [0, 32, record.len() - 1] {
let mut corrupt_record = record.to_vec();
corrupt_record[position] ^= 1;
assert!(part.verify_record(0, &corrupt_record).is_err());
}
let proof = Arc::new(PartProofReader {
part,
sources: vec![],
volume: "bucket".to_owned(),
inline: Some(bytes),
cached: Mutex::new(ProofCache::default()),
});
let mut verifier = ShardVerifier::new(proof, 0, 0, None).expect("verifier");
verifier.verify(shards[0]).await.expect("stripe 0");
verifier.verify(shards[0]).await.expect("stripe 1");
assert!(verifier.verify(shards[0]).await.is_err(), "no clean success past committed stripes");
}
#[tokio::test]
async fn integrity_reconstruction_rejects_wrong_decoder_output() {
let mut builder = IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 1).expect("builder");
builder
.push([b"abcd".as_slice(), b"efgh", b"ijkl", b"mnop"].into_iter())
.await
.expect("stripe");
let prepared = builder.finish(8).await.expect("index");
let index = prepared.inline_bytes().expect("bytes");
let proof = PartProofReader {
part: prepared.part,
inline: Some(index),
sources: Vec::new(),
volume: String::new(),
cached: Mutex::new(ProofCache::default()),
};
let mut shards = vec![None, Some(b"efgh".to_vec()), Some(b"ijkl".to_vec()), Some(b"mnop".to_vec())];
let verification = proof
.reconstruction_proof(0, &shards)
.await
.expect("proof")
.expect("missing data");
shards[0] = Some(b"donr".to_vec());
assert!(verification.verify(&shards).is_err());
shards[0] = Some(b"abcd".to_vec());
verification.verify(&shards).expect("verified reconstruction");
}
#[tokio::test]
async fn integrity_builder_spills_without_changing_proofs() {
let layout = IntegrityLayout::new(12, 4, 12, false).expect("layout");
let mut builder = IntegrityBuilder::new(layout, 7).expect("builder");
let shards = [b"x".as_slice(); 16];
for _ in 0..2050 {
builder.push(shards.into_iter()).await.expect("stripe");
}
assert!(builder.spill.is_some(), "digest rows exceed memory bound");
let mut prepared = builder.finish(2050 * 12).await.expect("spilled tree");
assert!(matches!(prepared.index, Spool::File(_)), "materialized index is also bounded");
let mut record = vec![0; prepared.part.record_size()];
for stripe in [0, 1025, 2049] {
let offset = prepared.part.record_offset(stripe).expect("offset");
prepared
.index
.read_at(u64::try_from(offset).expect("offset fits"), &mut record)
.expect("proof");
prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
}
}
}
+50
View File
@@ -882,6 +882,15 @@ pub enum WriteCompletion {
TailDrained,
}
/// Storage-owned write mode inherited by physical rewrites. This selects the
/// destination format; the source reader must still validate every source byte.
#[doc(hidden)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ShardIntegrityWriteMode {
Legacy,
Protected,
}
#[derive(Default, Clone)]
pub struct ObjectOptions {
// Use the maximum parity (N/2), used when saving server configuration files
@@ -942,6 +951,10 @@ pub struct ObjectOptions {
pub data_movement: bool,
pub raw_data_movement_read: bool,
/// Internal, in-memory protection context. Never populated from S3 metadata.
/// None selects the rollout default only for a new write, not for a rewrite.
#[doc(hidden)]
pub shard_integrity_write_mode: Option<ShardIntegrityWriteMode>,
/// Durable reservation identity carried only by decommission writes. Other
/// data-movement users, including rebalance, leave it unset. Keep this
/// context boxed because `ObjectOptions` is passed by value through deep
@@ -1043,6 +1056,25 @@ pub enum ReplicationStatusWritebackMode {
}
impl ObjectOptions {
pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
match self.shard_integrity_write_mode {
Some(ShardIntegrityWriteMode::Protected) => true,
Some(ShardIntegrityWriteMode::Legacy) => false,
None if self.data_movement => false,
None => {
rustfs_utils::get_env_bool(rustfs_config::ENV_SHARD_INTEGRITY_WRITE, rustfs_config::DEFAULT_SHARD_INTEGRITY_WRITE)
&& rustfs_utils::get_env_bool(
rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED,
rustfs_config::DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED,
)
}
}
}
pub(crate) fn inherit_shard_integrity(&mut self, source: &ObjectInfo) {
self.shard_integrity_write_mode = Some(source.shard_integrity_write_mode());
}
pub(crate) fn with_capacity_expected_data_bytes(expected_data_bytes: Option<usize>) -> Self {
Self {
decommission_capacity: expected_data_bytes.map(|expected_data_bytes| {
@@ -1438,6 +1470,23 @@ impl Clone for ObjectInfo {
}
impl ObjectInfo {
pub(crate) fn shard_integrity_write_mode(&self) -> ShardIntegrityWriteMode {
// Any declaration requires protection. Malformed declarations remain
// errors in the source reader and must never select the legacy path.
if self.parts.iter().any(|part| part.integrity.is_some())
|| [
rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY,
rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY,
]
.iter()
.any(|suffix| rustfs_utils::http::contains_key_str(&self.user_defined, suffix))
{
ShardIntegrityWriteMode::Protected
} else {
ShardIntegrityWriteMode::Legacy
}
}
/// Capture the source mutation snapshot used by replication workers when
/// publishing terminal status. The semantic fingerprint is recomputed at
/// the storage CAS boundary, so an older writer that preserves an unknown
@@ -1840,6 +1889,7 @@ impl ObjectInfo {
checksums: part.checksums.clone(),
number: part.number,
error: part.error.clone(),
integrity: part.integrity.clone(),
})
.collect::<Vec<_>>();
@@ -924,7 +924,7 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
responses: &[Option<Vec<ObjectPartInfo>>],
read_quorum: usize,
) -> disk::error::Result<ObjectPartInfo> {
let mut part_quorum: HashMap<(&str, usize, usize, i64), (usize, &ObjectPartInfo)> = HashMap::new();
let mut part_quorum = HashMap::new();
let mut present_count = 0usize;
let mut missing_count = 0usize;
let mut transient_error_count = 0usize;
@@ -943,7 +943,7 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
if !parts[part_idx].etag.is_empty() {
present_count += 1;
let part = &parts[part_idx];
let key = (part.etag.as_str(), part.number, part.size, part.actual_size);
let key = (part.etag.as_str(), part.number, part.size, part.actual_size, part.integrity.as_ref());
let (count, _) = part_quorum.entry(key).or_insert((0, part));
*count += 1;
continue;
@@ -1420,6 +1420,40 @@ pub(in crate::set_disk) fn get_bitrot_reader_setup_strategy(
}
impl BitrotReaderSetup {
pub(in crate::set_disk) fn bind_integrity(
&mut self,
expected: Option<&rustfs_filemeta::shard_integrity::PartIntegrity>,
files: &[FileInfo],
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
first_stripe: usize,
) -> std::io::Result<()> {
use crate::io_support::shard_integrity::{PartProofReader, ShardVerifier};
let Some(expected) = expected else { return Ok(()) };
let proof = PartProofReader::new(expected.clone(), files, disks, bucket, object)?;
for (index, reader) in self.readers.iter_mut().enumerate() {
if let Some(reader) = reader {
let advanced = self.deferred_stripe_handles[index]
.as_ref()
.map(DeferredReaderStripeHandle::integrity_position);
reader.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, first_stripe, advanced)?)?;
}
if let Some(reopen) = self.deferred_reopeners[index].take() {
let proof = Arc::clone(&proof);
self.deferred_reopeners[index] = Some(Arc::new(move |stripe| {
let mut reader = reopen(stripe)?;
let first = first_stripe.checked_add(stripe)?;
reader
.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, first, None).ok()?)
.ok()?;
Some(reader)
}));
}
}
Ok(())
}
pub(in crate::set_disk) fn new(shards: usize) -> Self {
Self {
readers: (0..shards).map(|_| None).collect(),
@@ -5698,6 +5732,8 @@ impl SetDisks {
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
) -> disk::error::Result<Vec<Option<DiskStore>>> {
self.recover_part_transaction(dst_object, write_quorum).await?;
let part = ObjectPartInfo::unmarshal(&meta)?;
let integrity = part.integrity.map(Arc::new);
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
@@ -5711,12 +5747,39 @@ impl SetDisks {
let dst_bucket = dst_bucket.clone();
let dst_object = dst_object.clone();
let meta = meta.clone();
let integrity = integrity.clone();
async move {
let disk = disk?;
Some(
disk.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
.await,
)
let prepared = disk
.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
.await;
if let Err(error) = prepared {
return Some(Err(error));
}
if let Some(integrity) = integrity {
let Some((directory, _)) = dst_object.rsplit_once('/') else {
return Some(Err(DiskError::FileCorrupt));
};
let path = format!("{directory}/{}", integrity.file_name());
// Older peers may return Ok from PreparePart without moving
// the index. They must not enter the protected write quorum.
let result = async {
let mut reader = disk
.read_file_stream(&dst_bucket, &path, 0, rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE)
.await?;
let mut header = [0; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE];
tokio::io::AsyncReadExt::read_exact(&mut reader, &mut header)
.await
.map_err(DiskError::from)?;
if header != integrity.index_header() {
return Err(DiskError::FileCorrupt);
}
Ok(())
}
.await;
return Some(result);
}
Some(Ok(()))
}
});
let prepare_results = join_all(prepare_tasks).await;
+1
View File
@@ -488,6 +488,7 @@ impl SetDisks {
}
pub(crate) fn hydrate_selected_fileinfo_part_checksums(fi: &mut FileInfo) -> disk::error::Result<()> {
fi.hydrate_shard_integrity().map_err(DiskError::from)?;
fi.hydrate_data_movement_part_checksums().map_err(DiskError::from)?;
for part in &fi.parts {
let Some(checksums) = part.checksums.as_ref() else {
+19
View File
@@ -6426,6 +6426,24 @@ async fn disks_with_all_parts(
}
}
if scan_mode == HealScanMode::Deep
&& !latest_meta.deleted
&& !latest_meta.is_remote()
&& latest_meta.parts.iter().any(|part| part.integrity.is_some())
{
crate::io_support::shard_integrity::verify_deep_parts(
parts_metadata,
online_disks,
latest_meta,
bucket,
object,
&mut data_errs_by_part,
)
.await?;
populate_data_errs_by_disk(&mut data_errs_by_disk, &data_errs_by_part);
return Ok((data_errs_by_disk, data_errs_by_part));
}
// Check data for each disk
for (index, disk) in online_disks.iter().enumerate() {
if meta_errs[index].is_some() {
@@ -6858,6 +6876,7 @@ fn completed_multipart_object_part(part_num: usize, ext_part: &ObjectPartInfo) -
actual_size: ext_part.actual_size,
index: ext_part.index.clone(),
checksums: ext_part.checksums.clone(),
integrity: ext_part.integrity.clone(),
..Default::default()
}
}
+882 -2
View File
@@ -823,6 +823,13 @@ impl SetDisks {
match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) {
Ok(mut latest_meta) => {
Self::hydrate_selected_fileinfo_part_checksums(&mut latest_meta)?;
let protected =
!latest_meta.parts.is_empty() && latest_meta.parts.iter().all(|part| part.integrity.is_some());
result.integrity_verified = protected
&& !latest_meta.deleted
&& !latest_meta.is_remote()
&& opts.scan_mode == HealScanMode::Deep
&& !read_repair_uses_shared_lock;
trace!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
@@ -926,7 +933,31 @@ impl SetDisks {
});
}
if !latest_meta.deleted && !latest_meta.is_remote() && !protected {
result.detail =
"Legacy object uses standard repair; independent object identity remains unverified".to_owned();
}
if disks_to_heal_count == 0 {
if result.integrity_verified && !opts.dry_run {
match crate::io_support::shard_integrity::restore_proof_replicas(
&latest_meta,
&disks,
bucket,
object,
)
.await
{
Ok(repaired) if repaired > 0 => {
result.detail = format!("Restored {repaired} independent integrity indexes")
}
Ok(_) => {}
Err(error) => {
result.integrity_verified = false;
return Ok((result, Some(error)));
}
}
}
// The object is already healthy: no disk needs healing.
// This is the common case for the very objects PR #4356
// targets — a valid `xl.meta` plus a leaked pre-#3510
@@ -1215,6 +1246,15 @@ impl SetDisks {
let mut writer_failure_warned = false;
for (part_index, part) in latest_meta.parts.iter().enumerate() {
use crate::io_support::shard_integrity::{PartProofReader, PreparedIntegrity, ShardVerifier};
let proof = part
.integrity
.as_ref()
.map(|part| {
PartProofReader::new(part.clone(), &parts_metadata, &latest_disks, bucket, object)
})
.transpose()
.map_err(DiskError::from)?;
let till_offset = erasure.shard_file_offset(0, part.size, part.size);
let use_mmap_read = object_mmap_read_enabled();
@@ -1265,7 +1305,15 @@ impl SetDisks {
)
.await
{
Ok(Some(reader)) => {
Ok(Some(mut reader)) => {
if let Some(proof) = &proof {
reader
.set_integrity(
ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None)
.map_err(DiskError::from)?,
)
.map_err(DiskError::from)?;
}
readers.push(Some(reader));
}
Ok(None) => {
@@ -1324,6 +1372,13 @@ impl SetDisks {
continue;
}
};
let mut writer = writer;
if let Some(proof) = &proof {
writer.set_integrity(
ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None)
.map_err(DiskError::from)?,
);
}
writers.push(Some(writer));
} else {
writers.push(None);
@@ -1363,6 +1418,26 @@ impl SetDisks {
return Err(e);
}
// close_bitrot_writers(&mut writers).await?;
if !is_inline_buffer && let Some(proof) = &proof {
let before = out_dated_disks.iter().filter(|disk| disk.is_some()).count();
let proof_result = async {
PreparedIntegrity::copy_from(proof)
.await?
.write(
&mut out_dated_disks,
bucket,
RUSTFS_META_TMP_BUCKET,
&format!("{tmp_id}/{dst_data_dir}"),
)
.await
}
.await;
if let Err(error) = proof_result {
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
return Err(error.into());
}
disks_to_heal_count -= before - out_dated_disks.iter().filter(|disk| disk.is_some()).count();
}
for (index, disk_op) in out_dated_disks.iter_mut().enumerate() {
if disk_op.is_none() {
@@ -1385,6 +1460,13 @@ impl SetDisks {
part.index.clone(),
part.checksums.clone(),
);
if let Some(updated) = parts_metadata[index]
.parts
.iter_mut()
.find(|updated| updated.number == part.number)
{
updated.integrity.clone_from(&part.integrity);
}
if is_inline_buffer {
if let Some(writer) = writers[index].take() {
// if let Some(w) = writer.as_any().downcast_ref::<BitrotFileWriter>() {
@@ -1560,6 +1642,15 @@ impl SetDisks {
));
}
if result.integrity_verified
&& let Err(error) =
crate::io_support::shard_integrity::restore_proof_replicas(&latest_meta, &disks, bucket, object)
.await
{
result.integrity_verified = false;
return Ok((result, Some(error)));
}
// The object is healthy here; sweep any data dirs left behind
// by pre-#3510 unversioned overwrites, which the dangling paths
// above never touch (issues #3231, #3191). Best effort — a
@@ -1678,6 +1769,17 @@ impl SetDisks {
let Some(candidate) = candidates.first().copied() else {
return Ok(false);
};
let protected = rustfs_filemeta::shard_integrity::descriptor_from_metadata(&candidate.metadata)
.map_err(DiskError::from)?
.is_some()
|| candidate.parts.iter().any(|part| part.integrity.is_some());
// A minority metadata copy cannot establish an independent commitment.
// Legacy recovery retains its existing bounds without certifying identity.
if protected
&& (candidates.len() < candidate.erasure.data_blocks || candidate.parts.iter().any(|part| part.integrity.is_none()))
{
return Ok(false);
}
let identity = Self::file_info_quorum_hash(candidate);
if candidates
.iter()
@@ -1711,6 +1813,31 @@ impl SetDisks {
return Ok(false);
}
let mut verified_disks = vec![true; disks.len()];
if protected {
let mut proof_files = parts_metadata.to_vec();
for (index, file) in proof_files.iter_mut().enumerate() {
if matches!(
errs.get(index).and_then(Option::as_ref),
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
) {
*file = candidate.clone();
file.erasure.index = candidate.erasure.distribution[index];
}
}
let mut verified = (0..candidate.parts.len())
.map(|part| (part, vec![crate::disk::CHECK_PART_UNKNOWN; disks.len()]))
.collect();
crate::io_support::shard_integrity::verify_deep_parts(&proof_files, disks, candidate, bucket, object, &mut verified)
.await?;
for (index, valid) in verified_disks.iter_mut().enumerate() {
*valid = verified.values().all(|parts| parts.get(index) == Some(&CHECK_PART_SUCCESS));
}
if verified_disks.iter().filter(|valid| **valid).count() < candidate.erasure.data_blocks {
return Ok(false);
}
}
let mut wrote = 0usize;
for (index, disk) in disks.iter().enumerate() {
let Some(disk) = disk else {
@@ -1720,7 +1847,7 @@ impl SetDisks {
errs.get(index).and_then(Option::as_ref),
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
);
if !metadata_absent {
if !metadata_absent || !verified_disks[index] {
continue;
}
let Some(&shard_index) = candidate.erasure.distribution.get(index) else {
@@ -5439,3 +5566,756 @@ mod heal_result_report_tests {
}
}
}
#[cfg(test)]
mod shard_integrity_rollout_tests;
#[cfg(test)]
mod shard_identity_tests {
use crate::disk::{DiskAPI as _, ReadOptions};
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated;
use crate::storage_api_contracts::multipart::MultipartOperations as _;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use crate::storage_api_contracts::range::HTTPRangeSpec;
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
use tokio::io::AsyncReadExt;
#[tokio::test]
async fn shard_integrity_heal_rejects_complete_donor_shards_ec12_4() {
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(16, 0, 4).await;
let bucket = "donor-shard-identity";
for disk in &disks {
disk.make_volume(bucket).await.expect("create fixture bucket");
}
let expected = vec![0x5a; 5 * 1024 * 1024 + 123];
let donor_body = vec![0xa6; expected.len()];
let options = ObjectOptions {
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
no_lock: true,
versioned: true,
..Default::default()
};
for (case, (coding_indexes, replace_descriptor)) in [
(vec![1], false),
(vec![6], false),
(vec![12], false),
(vec![13], false),
(vec![16], false),
(vec![1, 2, 3, 4], false),
(vec![1, 2, 3, 4, 5], false),
(vec![1], true),
]
.into_iter()
.enumerate()
{
let target = format!("target-{case}");
let donor = format!("donor-{case}");
set.put_object(bucket, &target, &mut PutObjReader::from_vec(expected.clone()), &options)
.await
.expect("commit every target shard");
set.put_object(bucket, &donor, &mut PutObjReader::from_vec(donor_body.clone()), &options)
.await
.expect("commit every donor shard");
let mut target_meta = Vec::new();
let mut donor_meta = Vec::new();
for disk in &disks {
target_meta.push(
disk.read_version("", bucket, &target, "", &ReadOptions::default())
.await
.expect("target metadata"),
);
donor_meta.push(
disk.read_version("", bucket, &donor, "", &ReadOptions::default())
.await
.expect("donor metadata"),
);
}
assert_eq!((target_meta[0].erasure.data_blocks, target_meta[0].erasure.parity_blocks), (12, 4));
let mut before = Vec::new();
let mut paths = Vec::new();
let mut meta_before = Vec::new();
for (slot, meta) in target_meta.iter().enumerate() {
let object_path = dirs[slot].path().join(bucket).join(&target);
let path = object_path
.join(meta.data_dir.expect("external data dir").to_string())
.join("part.1");
before.push(tokio::fs::read(&path).await.expect("original shard"));
meta_before.push(tokio::fs::read(object_path.join("xl.meta")).await.expect("original xl.meta"));
paths.push(path);
}
for index in &coding_indexes {
let target_slot = target_meta
.iter()
.position(|m| m.erasure.index == *index)
.expect("target coding index");
let donor_slot = donor_meta
.iter()
.position(|m| m.erasure.index == *index)
.expect("donor coding index");
let path = dirs[donor_slot]
.path()
.join(bucket)
.join(&donor)
.join(donor_meta[donor_slot].data_dir.expect("donor data dir").to_string())
.join("part.1");
let replacement = tokio::fs::read(path).await.expect("complete valid donor");
assert_eq!(
replacement.len(),
before[target_slot].len(),
"the adversarial file must have the same physical length"
);
assert_ne!(replacement, before[target_slot]);
tokio::fs::write(&paths[target_slot], replacement)
.await
.expect("replace the complete shard without changing metadata");
if replace_descriptor {
let mut altered = target_meta[target_slot].clone();
let suffix = rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY;
let descriptor = rustfs_utils::http::get_consistent_str(&donor_meta[donor_slot].metadata, suffix)
.expect("donor descriptor");
rustfs_utils::http::insert_str(&mut altered.metadata, suffix, descriptor.to_owned());
disks[target_slot]
.write_metadata("", bucket, &target, altered)
.await
.expect("a minority descriptor must not authorize its own donor payload");
}
}
let version = target_meta[0].version_id.expect("versioned fixture").to_string();
if coding_indexes.len() <= 4 {
let mut reader = set
.get_object_reader(bucket, &target, None, Default::default(), &options)
.await
.expect("recoverable GET before explicit heal");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("verify and reconstruct every GET block");
assert_eq!(body, expected, "GET must not expose donor data before explicit heal");
}
let heal = set
.heal_object(
bucket,
&target,
&version,
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await;
if coding_indexes.len() <= 4 {
let (result, error) = heal.expect("recoverable donor mismatch");
assert!(error.is_none(), "{error:?}");
assert_eq!(
result.drives_healed(),
Some(coding_indexes.len()),
"every complete donor must be detected"
);
for (slot, path) in paths.iter().enumerate() {
assert_eq!(
tokio::fs::read(path).await.expect("post-heal shard"),
before[slot],
"reconstruct exact original bytes at slot {slot}"
);
}
for (range, expected_range) in [
(None, expected.as_slice()),
(
Some(HTTPRangeSpec {
start: 1024 * 1024 + 17,
end: 2 * 1024 * 1024 + 99,
is_suffix_length: false,
}),
&expected[1024 * 1024 + 17..2 * 1024 * 1024 + 100],
),
] {
let mut reader = set
.get_object_reader(
bucket,
&target,
range,
Default::default(),
&ObjectOptions {
version_id: Some(version.clone()),
..options.clone()
},
)
.await
.expect("read repaired version");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("complete verified GET body");
assert_eq!(body, expected_range, "full and range GET must match original bytes");
}
} else {
assert!(
heal.as_ref().map_or(true, |(_, error)| error.is_some()),
"eleven authoritative sources must not produce a successful heal"
);
let read = set
.get_object_reader(bucket, &target, None, Default::default(), &options)
.await;
if let Ok(mut reader) = read {
let mut body = Vec::new();
assert!(reader.stream.read_to_end(&mut body).await.is_err(), "quorum-minus-one must fail the body");
assert!(body.is_empty(), "unverified first-stripe bytes must not escape");
}
for (slot, meta) in target_meta.iter().enumerate() {
if !coding_indexes.contains(&meta.erasure.index) {
assert_eq!(tokio::fs::read(&paths[slot]).await.expect("retained correct shard"), before[slot]);
}
assert_eq!(
tokio::fs::read(dirs[slot].path().join(bucket).join(&target).join("xl.meta"))
.await
.expect("retained metadata"),
meta_before[slot]
);
}
}
}
}
#[tokio::test]
async fn shard_integrity_inline_donor_repair_and_late_external_get() {
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
let bucket = "bound-inline-and-late";
for disk in &disks {
disk.make_volume(bucket).await.expect("fixture bucket");
}
let options = ObjectOptions {
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
no_lock: true,
..Default::default()
};
let read_options = ReadOptions {
read_data: true,
..Default::default()
};
for (size, late) in [(4113usize, false), (3 * 1024 * 1024 + 123, true)] {
let target = format!("target-{size}");
let donor = format!("donor-{size}");
let expected = vec![0x4d; size];
set.put_object(bucket, &target, &mut PutObjReader::from_vec(expected.clone()), &options)
.await
.expect("target PUT");
set.put_object(bucket, &donor, &mut PutObjReader::from_vec(vec![0x9a; size]), &options)
.await
.expect("donor PUT");
let mut target_parts = Vec::new();
let mut donor_parts = Vec::new();
for disk in &disks {
target_parts.push(
disk.read_version("", bucket, &target, "", &read_options)
.await
.expect("target meta"),
);
donor_parts.push(
disk.read_version("", bucket, &donor, "", &read_options)
.await
.expect("donor meta"),
);
}
let slot = target_parts
.iter()
.position(|fi| fi.erasure.index == 1)
.expect("target data slot");
let donor_slot = donor_parts
.iter()
.position(|fi| fi.erasure.index == 1)
.expect("donor data slot");
if !late {
let original = target_parts[slot].data.clone().expect("inline target");
let donor_bytes = donor_parts[donor_slot].data.as_ref().expect("inline donor");
assert_eq!(donor_bytes.len(), original.len());
let path = dirs[slot].path().join(bucket).join(&target).join("xl.meta");
let mut raw = tokio::fs::read(&path).await.expect("physical inline metadata");
let offsets: Vec<_> = raw
.windows(original.len())
.enumerate()
.filter_map(|(offset, bytes)| (bytes == original.as_ref()).then_some(offset))
.collect();
assert_eq!(offsets.len(), 1, "unique inline payload range");
raw[offsets[0]..offsets[0] + original.len()].copy_from_slice(donor_bytes);
tokio::fs::write(path, raw)
.await
.expect("replace inline bytes with target identity retained");
let (result, error) = set
.heal_object(
bucket,
&target,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("inline heal");
assert!(error.is_none(), "{error:?}");
assert_eq!(result.drives_healed(), Some(1));
let repaired = disks[slot]
.read_version("", bucket, &target, "", &read_options)
.await
.expect("repaired inline");
assert_eq!(repaired.data, Some(original));
} else {
let target_path = dirs[slot]
.path()
.join(bucket)
.join(&target)
.join(target_parts[slot].data_dir.expect("target dir").to_string())
.join("part.1");
let donor_path = dirs[donor_slot]
.path()
.join(bucket)
.join(&donor)
.join(donor_parts[donor_slot].data_dir.expect("donor dir").to_string())
.join("part.1");
let mut bytes = tokio::fs::read(&target_path).await.expect("target bytes");
let donor_bytes = tokio::fs::read(donor_path).await.expect("donor bytes");
let frame = 32 + target_parts[slot].erasure.shard_size();
bytes[frame..2 * frame].copy_from_slice(&donor_bytes[frame..2 * frame]);
tokio::fs::write(target_path, bytes)
.await
.expect("replace a later complete frame");
}
let mut reader = set
.get_object_reader(bucket, &target, None, Default::default(), &options)
.await
.expect("GET after corruption");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("late corruption must reconstruct without leaking donor bytes");
assert_eq!(body, expected);
}
}
#[tokio::test]
async fn shard_integrity_legacy_requires_target_digest_not_only_parity_consistency() {
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
let bucket = "legacy-shard-authority";
for disk in &disks {
disk.make_volume(bucket).await.expect("fixture bucket");
}
let options = ObjectOptions {
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
no_lock: true,
..Default::default()
};
let expected = vec![0x4d; 1024 * 1024 + 123];
let mut parts = Vec::new();
let mut paths = Vec::new();
for (object, body) in [("target", expected.clone()), ("donor", vec![0x9a; expected.len()])] {
set.put_object(bucket, object, &mut PutObjReader::from_vec(body), &options)
.await
.expect("seed legacy fixture");
let mut object_parts = Vec::new();
let mut object_paths = Vec::new();
for (slot, disk) in disks.iter().enumerate() {
let mut fi = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("source meta");
let path = dirs[slot]
.path()
.join(bucket)
.join(object)
.join(fi.data_dir.expect("data dir").to_string())
.join("part.1");
let encoded = tokio::fs::read(&path).await.expect("encoded source");
let mut legacy = Vec::with_capacity(encoded.len());
for frame in encoded.chunks(32 + fi.erasure.shard_size()) {
let payload = &frame[32..];
legacy.extend_from_slice(rustfs_utils::HashAlgorithm::HighwayHash256S.hash_encode(payload).as_ref());
legacy.extend_from_slice(payload);
}
rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut fi.metadata);
for part in &mut fi.parts {
part.integrity = None;
}
tokio::fs::write(&path, legacy).await.expect("legacy framing");
disk.write_metadata("", bucket, object, fi.clone())
.await
.expect("legacy metadata");
object_parts.push(fi);
object_paths.push(path);
}
parts.push(object_parts);
paths.push(object_paths);
}
let mut reader = set
.get_object_reader(bucket, "target", None, Default::default(), &options)
.await
.expect("healthy legacy object remains readable");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("legacy body");
assert_eq!(body, expected);
for (slot, fi) in parts[0].iter().enumerate() {
let source = parts[1]
.iter()
.position(|donor| donor.erasure.index == fi.erasure.index)
.expect("coding index");
tokio::fs::copy(&paths[1][source], &paths[0][slot])
.await
.expect("install a complete parity-consistent foreign codeword");
}
// Compatibility keeps legacy GET semantics. A complete foreign codeword
// remains an explicit residual risk until a trusted source rewrites it.
let mut legacy = set
.get_object_reader(bucket, "target", None, Default::default(), &options)
.await
.expect("legacy GET remains available");
let mut foreign = Vec::new();
legacy
.stream
.read_to_end(&mut foreign)
.await
.expect("legacy donor is internally consistent");
assert_ne!(foreign, expected);
for scan_mode in [HealScanMode::Normal, HealScanMode::Deep] {
let heal = set
.heal_object(
bucket,
"target",
"",
&HealOpts {
no_lock: true,
scan_mode,
..Default::default()
},
)
.await;
let (result, error) = heal.expect("legacy scan");
assert!(error.is_none(), "legacy scan remains observable");
assert!(!result.integrity_verified, "a foreign codeword must not be certified");
assert_eq!(result.drives_healed(), Some(0), "legacy scan must not rewrite data");
}
}
#[tokio::test]
async fn shard_integrity_multipart_replacement_and_metadata_copy_preserve_identity() {
use crate::set_disk::{CompletePart, RUSTFS_META_MULTIPART_BUCKET, SetDisks};
use rustfs_filemeta::ObjectPartInfo;
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
let bucket = "bound-multipart-copy";
let object = "target";
for disk in &disks {
disk.make_volume(bucket).await.expect("fixture bucket");
}
let options = ObjectOptions {
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
no_lock: true,
versioned: true,
..Default::default()
};
let upload = set.new_multipart_upload(bucket, object, &options).await.expect("new upload");
let (upload_meta, _) = set
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
.await
.expect("upload metadata");
let upload_path = SetDisks::get_multipart_upload_dir(bucket, object, &upload.upload_id, false);
let part_path = dirs[0]
.path()
.join(RUSTFS_META_MULTIPART_BUCKET)
.join(upload_path)
.join(upload_meta.data_dir.expect("staging directory").to_string())
.join("part.2");
let first = vec![0x36; 5 * 1024 * 1024];
let second = vec![0x69; 1024 * 1024 + 123];
let p1 = set
.put_object_part(bucket, object, &upload.upload_id, 1, &mut PutObjReader::from_vec(first.clone()), &options)
.await
.expect("first part");
set.put_object_part(
bucket,
object,
&upload.upload_id,
2,
&mut PutObjReader::from_vec(vec![0xa5; second.len()]),
&options,
)
.await
.expect("original second part");
let old_frame = tokio::fs::read(&part_path).await.expect("old part shard");
let old_meta: ObjectPartInfo = rmp_serde::from_slice(
&tokio::fs::read(part_path.with_extension("2.meta"))
.await
.expect("old part descriptor"),
)
.expect("decode old part descriptor");
let p2 = set
.put_object_part(
bucket,
object,
&upload.upload_id,
2,
&mut PutObjReader::from_vec(second.clone()),
&options,
)
.await
.expect("replace second part");
let mut completed = set
.clone()
.complete_multipart_upload(
bucket,
object,
&upload.upload_id,
vec![
CompletePart {
part_num: p1.part_num,
etag: p1.etag,
..Default::default()
},
CompletePart {
part_num: p2.part_num,
etag: p2.etag,
..Default::default()
},
],
&options,
)
.await
.expect("complete replacement");
let original = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("completed descriptor");
assert_ne!(
original.parts[1].integrity.as_ref().map(|proof| proof.generation),
old_meta.integrity.as_ref().map(|proof| proof.generation)
);
assert_ne!(
original.parts[0].integrity.as_ref().map(|proof| proof.generation),
original.parts[1].integrity.as_ref().map(|proof| proof.generation)
);
let final_path = dirs[0]
.path()
.join(bucket)
.join(object)
.join(original.data_dir.expect("completed directory").to_string())
.join("part.2");
assert_eq!(old_frame.len(), tokio::fs::metadata(&final_path).await.unwrap().len() as usize);
tokio::fs::write(final_path, old_frame)
.await
.expect("replay replaced part shard");
let (result, error) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("heal replayed part");
assert!(error.is_none(), "{error:?}");
assert_eq!(result.drives_healed(), Some(1));
// REPLACE may omit every internal field; metadata-only COPY must keep
// the existing payload domain even when it creates another version.
completed.metadata_only = true;
completed.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-amz-meta-label".to_string(),
"replacement".to_string(),
)]));
set.copy_object(bucket, object, bucket, object, &mut completed, &options, &options)
.await
.expect("metadata replacement copy");
let copied = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("copied descriptor");
for part in [1, 2] {
assert_eq!(copied.parts[part - 1].integrity, original.parts[part - 1].integrity);
}
let expected = [first, second].concat();
for version in [None, original.version_id.map(|id| id.to_string())] {
for range in [
None,
Some(HTTPRangeSpec {
start: 5 * 1024 * 1024 - 17,
end: 5 * 1024 * 1024 + 99,
is_suffix_length: false,
}),
] {
let expected_bytes = if range.is_some() {
&expected[5 * 1024 * 1024 - 17..5 * 1024 * 1024 + 100]
} else {
&expected[..]
};
let mut reader = set
.get_object_reader(
bucket,
object,
range,
Default::default(),
&ObjectOptions {
version_id: version.clone(),
..options.clone()
},
)
.await
.expect("multipart GET");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("verified multipart body");
assert_eq!(actual, expected_bytes);
}
}
}
#[tokio::test]
async fn shard_integrity_index_repair_preserves_payload_and_metadata() {
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
let bucket = "integrity-index-repair";
let object = "target";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket");
}
let expected = vec![0x6a; 2 * 1024 * 1024 + 123];
let opts = ObjectOptions {
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
no_lock: true,
..Default::default()
};
set.put_object(bucket, object, &mut PutObjReader::from_vec(expected.clone()), &opts)
.await
.expect("PUT");
let mut snapshots = Vec::new();
for (dir, disk) in dirs.iter().zip(&disks) {
let info = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("metadata");
let root = dir.path().join(bucket).join(object);
let data = root.join(info.data_dir.expect("data dir").to_string());
let proof = data.join(info.parts[0].integrity.as_ref().expect("commitment").file_name());
snapshots.push((
data.join("part.1"),
tokio::fs::read(data.join("part.1")).await.expect("payload"),
root.join("xl.meta"),
tokio::fs::read(root.join("xl.meta")).await.expect("xl.meta"),
proof.clone(),
tokio::fs::read(proof).await.expect("index"),
));
}
tokio::fs::remove_file(&snapshots[0].4).await.expect("missing index");
let mut bad = snapshots[1].5.clone();
bad[70] ^= 1;
tokio::fs::write(&snapshots[1].4, bad).await.expect("corrupt index");
let mut reader = set
.get_object_reader(bucket, object, None, Default::default(), &opts)
.await
.expect("GET with proof fallback");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("verified fallback body");
assert_eq!(body, expected);
let (healed, error) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("index repair");
assert!(error.is_none(), "{error:?}");
assert!(healed.integrity_verified);
for ((path, payload, meta_path, meta, proof_path, proof), (disk, dir)) in snapshots.iter().zip(disks.iter().zip(&dirs)) {
assert_eq!(tokio::fs::read(path).await.expect("payload"), *payload);
assert_eq!(tokio::fs::read(meta_path).await.expect("metadata"), *meta);
assert_eq!(tokio::fs::read(proof_path).await.expect("restored proof"), *proof);
let relative = proof_path.strip_prefix(dir.path().join(bucket)).expect("relative proof path");
disk.delete(bucket, relative.to_str().expect("proof path"), crate::disk::DeleteOptions::default())
.await
.expect("lose all independent proofs");
}
// All missing proofs cannot be recreated by trusting the surviving
// payload's self-contained bitrot checksum.
if let Ok(mut reader) = set.get_object_reader(bucket, object, None, Default::default(), &opts).await {
let mut body = Vec::new();
assert!(reader.stream.read_to_end(&mut body).await.is_err());
}
if let Ok((result, error)) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
{
assert!(error.is_some() || !result.integrity_verified);
}
for (path, payload, meta_path, meta, proof_path, _) in snapshots {
assert_eq!(tokio::fs::read(path).await.expect("retained payload"), payload);
assert_eq!(tokio::fs::read(meta_path).await.expect("retained metadata"), meta);
assert!(!proof_path.exists(), "no proof minted from unverified payload");
}
}
#[tokio::test]
async fn shard_integrity_empty_and_no_parity_objects_remain_usable() {
for parity in [0, 2] {
let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, parity).await;
let bucket = "integrity-empty-no-parity";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket");
}
for size in [0, 1, 1024 * 1024 + 123] {
let object = format!("target-{size}");
let expected = vec![0x51; size];
let opts = ObjectOptions {
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
no_lock: true,
..Default::default()
};
set.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &opts)
.await
.expect("PUT");
let mut reader = set
.get_object_reader(bucket, &object, None, Default::default(), &opts)
.await
.expect("GET");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("body");
assert_eq!(body, expected);
let (result, error) = set
.heal_object(
bucket,
&object,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("Deep scan");
assert!(error.is_none(), "{parity} {size}: {error:?}");
assert!(result.integrity_verified);
}
}
}
}
@@ -0,0 +1,396 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::{DiskAPI as _, ReadOptions};
use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
use crate::set_disk::CompletePart;
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated;
use crate::storage_api_contracts::multipart::MultipartOperations as _;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
use tokio::io::AsyncReadExt;
fn rollout_vars(requested: bool, confirmed: bool) -> [(&'static str, Option<&'static str>); 2] {
[
(rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some(if requested { "true" } else { "false" })),
(
rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED,
Some(if confirmed { "true" } else { "false" }),
),
]
}
#[tokio::test]
async fn remote_metadata_heal_does_not_certify_unread_tier_payload() {
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
let bucket = "remote-integrity";
let object = "transitioned";
for disk in &disks {
disk.make_volume(bucket).await.expect("fixture bucket");
}
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]),
&ObjectOptions {
no_lock: true,
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
..Default::default()
},
)
.await
.expect("protected source");
for (disk, dir) in disks.iter().zip(&dirs) {
let mut info = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("source metadata");
assert!(info.parts[0].integrity.is_some());
info.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_owned();
info.transition_tier = "WARM".to_owned();
info.transitioned_objname = "remote/transitioned".to_owned();
let data_dir = info.data_dir.expect("external shard directory");
disk.write_metadata("", bucket, object, info)
.await
.expect("publish remote metadata");
tokio::fs::remove_dir_all(dir.path().join(bucket).join(object).join(data_dir.to_string()))
.await
.expect("tier transition releases local payload and proof indexes");
}
let (item, error) = set
.heal_object(
bucket,
object,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("remote metadata scan");
assert!(error.is_none(), "{error:?}");
assert_eq!(item.drives_healed(), Some(0));
assert!(!item.integrity_verified, "local metadata presence cannot certify unread remote bytes");
}
#[tokio::test]
#[serial_test::serial(shard_integrity_rollout)]
async fn rollout_requires_both_flags_for_new_puts() {
let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
let bucket = "integrity-rollout";
for disk in &disks {
disk.make_volume(bucket).await.expect("fixture bucket");
}
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
for (requested, confirmed, protected) in [
(false, false, false),
(true, false, false),
(false, true, false),
(true, true, true),
] {
for size in [4096, 1024 * 1024 + 19] {
let object = format!("put-{requested}-{confirmed}-{size}");
let expected = vec![0x53; size];
temp_env::async_with_vars(rollout_vars(requested, confirmed), async {
set.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &opts)
.await
.expect("write at rollout gate");
})
.await;
for disk in &disks {
let meta = disk
.read_version(
"",
bucket,
&object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("published metadata");
assert_eq!(meta.parts[0].integrity.is_some(), protected);
assert_eq!(
rustfs_utils::http::contains_key_str(
&meta.metadata,
rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY
),
protected,
"the public gate must control the committed extension"
);
assert_eq!(meta.erasure.get_checksum_info(1).algorithm, rustfs_utils::HashAlgorithm::HighwayHash256S);
}
let mut reader = temp_env::async_with_vars(
rollout_vars(false, false),
set.get_object_reader(bucket, &object, None, Default::default(), &opts),
)
.await
.expect("read after disabling new protection");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("complete exact body");
assert_eq!(actual, expected);
}
}
}
#[tokio::test]
async fn legacy_shard_repair_and_explicit_version_metadata_recovery_remain_available() {
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
let bucket = "legacy-recovery";
for disk in &disks {
disk.make_volume(bucket).await.expect("fixture bucket");
}
let opts = ObjectOptions {
no_lock: true,
versioned: true,
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
..Default::default()
};
let expected = vec![0x67; 1024 * 1024 + 37];
for case in ["missing-part", "bad-bitrot", "missing-version-metadata"] {
let object = case;
let info = set
.put_object(bucket, object, &mut PutObjReader::from_vec(expected.clone()), &opts)
.await
.expect("write legacy fixture");
let version = info.version_id.expect("versioned fixture").to_string();
let meta = disks[0]
.read_version("", bucket, object, &version, &ReadOptions::default())
.await
.expect("legacy metadata");
assert!(meta.parts.iter().all(|part| part.integrity.is_none()));
let data_dir = meta.data_dir.expect("external part directory").to_string();
let mut original = Vec::new();
for dir in &dirs {
original.push(
tokio::fs::read(dir.path().join(bucket).join(object).join(&data_dir).join("part.1"))
.await
.expect("original framed shard"),
);
}
match case {
"missing-part" => {
tokio::fs::remove_file(dirs[0].path().join(bucket).join(object).join(&data_dir).join("part.1"))
.await
.expect("remove one shard");
}
"bad-bitrot" => {
let mut damaged = original[0].clone();
damaged[32] ^= 1;
tokio::fs::write(dirs[0].path().join(bucket).join(object).join(&data_dir).join("part.1"), damaged)
.await
.expect("corrupt one stored block");
}
_ => {
for dir in dirs.iter().skip(1) {
tokio::fs::remove_file(dir.path().join(bucket).join(object).join("xl.meta"))
.await
.expect("retain one consistent version metadata copy");
}
}
}
let (result, error) = set
.heal_object(
bucket,
object,
&version,
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("legacy recovery must run");
assert!(error.is_none(), "{case}: {error:?}");
assert!(!result.integrity_verified, "traditional recovery cannot certify original identity");
if case != "missing-version-metadata" {
assert_eq!(result.drives_healed(), Some(1), "physical repair must remain observable");
}
for (index, disk) in disks.iter().enumerate() {
let restored = disk
.read_version("", bucket, object, &version, &ReadOptions::default())
.await
.expect("metadata must be restored for the explicit version");
assert_eq!(restored.version_id, info.version_id);
assert!(restored.parts.iter().all(|part| part.integrity.is_none()));
let bytes = tokio::fs::read(
dirs[index]
.path()
.join(bucket)
.join(object)
.join(restored.data_dir.expect("restored data directory").to_string())
.join("part.1"),
)
.await
.expect("restored payload");
assert_eq!(bytes, original[index], "recover the original framed bytes on every drive");
}
let mut reader = set
.get_object_reader(
bucket,
object,
None,
Default::default(),
&ObjectOptions {
version_id: Some(version),
..opts.clone()
},
)
.await
.expect("read recovered historical version");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("complete recovered body");
assert_eq!(actual, expected);
}
}
#[tokio::test]
#[serial_test::serial(shard_integrity_rollout)]
async fn multipart_mode_survives_rollout_switch_changes() {
let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
let bucket = "multipart-rollout";
for disk in &disks {
disk.make_volume(bucket).await.expect("fixture bucket");
}
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
for protected in [false, true] {
let object = format!("upload-{protected}");
let upload =
temp_env::async_with_vars(rollout_vars(protected, protected), set.new_multipart_upload(bucket, &object, &opts))
.await
.expect("initiate upload");
let expected = vec![0x72; 128 * 1024 + 7];
let complete = temp_env::async_with_vars(rollout_vars(!protected, !protected), async {
let part = set
.put_object_part(
bucket,
&object,
&upload.upload_id,
1,
&mut PutObjReader::from_vec(expected.clone()),
&opts,
)
.await
.expect("upload after switch change");
set.clone()
.complete_multipart_upload(
bucket,
&object,
&upload.upload_id,
vec![CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
&opts,
)
.await
.expect("complete using persisted upload mode")
})
.await;
assert_eq!(
complete.shard_integrity_write_mode(),
if protected {
ShardIntegrityWriteMode::Protected
} else {
ShardIntegrityWriteMode::Legacy
}
);
let meta = disks[0]
.read_version("", bucket, &object, "", &ReadOptions::default())
.await
.expect("complete metadata");
assert_eq!(meta.parts[0].integrity.is_some(), protected);
assert!(!rustfs_utils::http::contains_key_str(
&meta.metadata,
rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY
));
let mut reader = set
.get_object_reader(bucket, &object, None, Default::default(), &opts)
.await
.expect("completed upload");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("full multipart body");
assert_eq!(actual, expected);
}
}
#[tokio::test]
#[serial_test::serial(shard_integrity_rollout)]
async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
let bucket = "rewrite-rollout";
for disk in &disks {
disk.make_volume(bucket).await.expect("fixture bucket");
}
for protected in [false, true] {
let mode = if protected {
ShardIntegrityWriteMode::Protected
} else {
ShardIntegrityWriteMode::Legacy
};
let object = format!("source-{protected}");
let expected = vec![0x39; 1024 * 1024 + 11];
let seed_opts = ObjectOptions {
no_lock: true,
shard_integrity_write_mode: Some(mode),
..Default::default()
};
let mut source = set
.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &seed_opts)
.await
.expect("source fixture");
source.metadata_only = false;
source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
let copied = temp_env::async_with_vars(
rollout_vars(!protected, !protected),
set.copy_object(bucket, &object, bucket, &object, &mut source, &opts, &opts),
)
.await
.expect("materialized self-copy");
assert_eq!(
copied.shard_integrity_write_mode(),
mode,
"a legacy rewrite is not a trusted migration and a protected rewrite cannot downgrade"
);
let meta = disks[0]
.read_version("", bucket, &object, "", &ReadOptions::default())
.await
.expect("rewritten metadata");
assert_eq!(meta.parts[0].integrity.is_some(), protected);
let mut reader = set
.get_object_reader(bucket, &object, None, Default::default(), &opts)
.await
.expect("read physical rewrite");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("complete rewritten body");
assert_eq!(actual, expected);
}
}
+66 -12
View File
@@ -1626,17 +1626,20 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
};
let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
let (reader, w_size) = match write_path {
SmallWritePath::SingleBlockNonInline => {
Arc::clone(&erasure)
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
.await?
}
SmallWritePath::PipelineBatchedLarge => {
Arc::clone(&erasure).encode_batched(stream, &mut writers, write_quorum).await?
}
SmallWritePath::Inline | SmallWritePath::Pipeline => Arc::clone(&erasure).encode(stream, &mut writers, write_quorum).await?,
let upload_suffix = rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY;
let protected_upload = rustfs_utils::http::get_consistent_str(&fi.metadata, upload_suffix) == Some("1");
if rustfs_utils::http::contains_key_str(&fi.metadata, upload_suffix) && !protected_upload {
return Err(DiskError::FileCorrupt.into());
}
use crate::erasure::coding::encode::IntegrityEncodeMode;
let mode = match write_path {
SmallWritePath::SingleBlockNonInline => IntegrityEncodeMode::SingleBlock(small_size_hint),
SmallWritePath::PipelineBatchedLarge => IntegrityEncodeMode::Batched,
SmallWritePath::Inline | SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming,
};
let (reader, w_size, _, integrity) = Arc::clone(&erasure)
.encode_with_shard_integrity(stream, &mut writers, write_quorum, part_id, mode, protected_upload)
.await?;
if let Some(stage_start) = encode_stage_start {
rustfs_io_metrics::record_put_object_stage_duration(
@@ -1674,6 +1677,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
)))?;
}
let part_integrity = if let Some(integrity) = integrity {
Some(integrity.write(&mut shuffle_disks, bucket, RUSTFS_META_TMP_BUCKET, &tmp_part).await?)
} else { None };
if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
return Err(Error::ErasureWriteQuorum);
}
let index_op = data
.stream
.try_get_index()
@@ -1706,6 +1716,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
actual_size,
index: index_op,
checksums: if checksums.is_empty() { None } else { Some(checksums) },
integrity: part_integrity,
..Default::default()
};
@@ -2065,6 +2076,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
#[tracing::instrument(skip(self))]
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
crate::hp_guard!("SetDisks::new_multipart_upload");
let protect_upload = opts.shard_integrity_write_enabled();
let storage_class_config = self.storage_class_config_snapshot();
let mut _object_lock_guard = None;
@@ -2086,6 +2098,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let disks = disks.clone();
let mut user_defined = opts.user_defined.clone();
rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut user_defined);
if protect_upload {
rustfs_utils::http::insert_str(
&mut user_defined,
rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY,
"1".to_owned(),
);
}
rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
if !opts.data_movement {
rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
@@ -2579,6 +2599,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
part.index.clone(),
part.checksums.clone(),
);
let inserted = fi
.parts
.iter_mut()
.find(|entry| entry.number == part.number)
.ok_or(Error::FileCorrupt)?;
inserted.integrity.clone_from(&part.integrity);
}
let (shuffle_disks, mut parts_metadatas) = Self::shuffle_disks_and_parts_metadata_by_index(&disks, &files_metas, &fi);
@@ -2949,6 +2975,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
}
}
if rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY)
&& (rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY)
!= Some("1")
|| fi.parts.iter().any(|part| part.integrity.is_none()))
{
return Err(Error::PartMissingOrCorrupt);
}
rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY);
fi.persist_shard_integrity()?;
for meta in parts_metadatas.iter_mut() {
if meta.has_valid_erasure_geometry() {
meta.size = fi.size;
@@ -5355,7 +5391,16 @@ mod tests {
upload_path,
upload_meta.data_dir.expect("multipart upload should have a data directory")
);
let retry_meta = Bytes::from_static(b"interrupted retry metadata");
let retry_meta = Bytes::from(
ObjectPartInfo {
number: 1,
size: 23,
etag: "interrupted-retry".to_owned(),
..Default::default()
}
.marshal_msg()
.expect("valid legacy retry metadata"),
);
for (index, disk) in disk_stores.iter().enumerate().take(3) {
let retry_path = format!("{}/part.1", Uuid::new_v4());
@@ -5423,7 +5468,16 @@ mod tests {
&src_path,
RUSTFS_META_MULTIPART_BUCKET,
&dst_path,
Bytes::from_static(b"retry metadata"),
Bytes::from(
ObjectPartInfo {
number: 1,
size: 9,
etag: "retry".to_owned(),
..Default::default()
}
.marshal_msg()
.expect("valid legacy part metadata"),
),
3,
None,
)
+58 -43
View File
@@ -3417,6 +3417,7 @@ impl SetDisks {
opts: &ObjectOptions,
mut publication_fence: Option<RemoteTuplePublicationFence>,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
let protect_write = opts.shard_integrity_write_enabled();
if publication_fence.is_none()
&& opts.data_movement
&& rustfs_utils::http::metadata_compat::contains_key_str(
@@ -3461,6 +3462,7 @@ impl SetDisks {
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
let mut user_defined = opts.user_defined.clone();
rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut user_defined);
if let Some(eval_metadata) = &opts.eval_metadata {
merge_evaluated_metadata(&mut user_defined, eval_metadata)?;
}
@@ -3556,7 +3558,9 @@ impl SetDisks {
let put_object_size = known_put_object_storage_size(data.size());
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned)
&& put_object_size >= 0
&& usize::try_from(put_object_size).is_ok_and(|size| size <= erasure.block_size);
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
let shard_file_size = shard_file_size_raw;
@@ -3677,48 +3681,16 @@ impl SetDisks {
};
let encode_stage_start = collect_stage_timing.then(Instant::now);
let mut inline_shards = None;
let (reader, w_size) = match write_path {
SmallWritePath::Inline => match Arc::clone(&erasure)
.encode_inline_shards_with_size_hint(stream, small_size_hint)
.await
{
Ok((r, w, shards)) => {
inline_shards = Some(shards);
(r, w)
}
Err(e) => {
error!("encode_inline_small err {:?}", e);
return Err(e.into());
}
},
SmallWritePath::SingleBlockNonInline => match Arc::clone(&erasure)
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
.await
{
Ok((r, w)) => (r, w),
Err(e) => {
error!("encode_single_block_non_inline err {:?}", e);
return Err(e.into());
}
},
SmallWritePath::PipelineBatchedLarge => {
match Arc::clone(&erasure).encode_batched(stream, &mut writers, write_quorum).await {
Ok((r, w)) => (r, w),
Err(e) => {
error!("encode_batched err {:?}", e);
return Err(e.into());
}
}
}
SmallWritePath::Pipeline => match Arc::clone(&erasure).encode(stream, &mut writers, write_quorum).await {
Ok((r, w)) => (r, w),
Err(e) => {
error!("encode err {:?}", e);
return Err(e.into());
}
},
use crate::erasure::coding::encode::IntegrityEncodeMode;
let mode = match write_path {
SmallWritePath::Inline => IntegrityEncodeMode::Inline(small_size_hint),
SmallWritePath::SingleBlockNonInline => IntegrityEncodeMode::SingleBlock(small_size_hint),
SmallWritePath::PipelineBatchedLarge => IntegrityEncodeMode::Batched,
SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming,
};
let (reader, w_size, inline_shards, integrity) = Arc::clone(&erasure)
.encode_with_shard_integrity(stream, &mut writers, write_quorum, 1, mode, protect_write)
.await?;
let encode_elapsed = encode_stage_start.map(|stage_start| stage_start.elapsed());
let encode_ms = encode_elapsed.map(|elapsed| elapsed.as_millis() as u64).unwrap_or_default();
if let Some(encode_elapsed) = encode_elapsed {
@@ -3820,6 +3792,35 @@ impl SetDisks {
)));
}
let part_integrity = if let Some(integrity) = integrity {
Some(if is_inline_buffer {
integrity.set_inline_metadata(&mut fi)?;
integrity.part
} else {
integrity
.write(
&mut shuffle_disks,
bucket,
RUSTFS_META_TMP_BUCKET,
&format!("{tmp_dir}/{}", fi.data_dir.ok_or(Error::FileCorrupt)?),
)
.await?
})
} else {
None
};
if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
return Err(Error::ErasureWriteQuorum);
}
if let Some(inline_proof) =
rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY)
{
insert_str(
&mut user_defined,
rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY,
inline_proof.to_owned(),
);
}
fi.metadata = user_defined;
if fi.version_id.is_none_or(|id| id.is_nil()) && !opts.data_movement && expected_restore_operation_id.is_none() {
// Every disk must publish the same cleanup owner alongside a
@@ -3832,6 +3833,8 @@ impl SetDisks {
fi.size = w_size as i64;
fi.versioned = opts.versioned || opts.version_suspended;
fi.add_object_part(1, etag, w_size, mod_time, actual_size, index_op, None);
fi.parts[0].integrity = part_integrity;
fi.persist_shard_integrity()?;
if opts.data_movement {
fi.set_data_moved();
}
@@ -7446,7 +7449,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// Self-copy with a data reader: write tier data back locally (de-tiering).
// Handles `mc cp --storage-class STANDARD obj obj` on a transitioned object.
if let Some(mut put_reader) = src_info.put_object_reader.take() {
return self.put_object(dst_bucket, dst_object, &mut put_reader, dst_opts).await;
let mut put_opts = dst_opts.clone();
put_opts.inherit_shard_integrity(src_info);
return self.put_object(dst_bucket, dst_object, &mut put_reader, &put_opts).await;
}
// Same-key tiered copy without a pre-fetched reader: fall through to the metadata
// path so the caller gets a disk/quorum error rather than NotImplemented.
@@ -7613,6 +7618,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
None
};
let mut replacement_metadata = (*src_info.user_defined).clone();
rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut replacement_metadata);
for suffix in [
rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY,
rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY,
] {
if rustfs_utils::http::contains_key_str(&fi.metadata, suffix) {
let value = rustfs_utils::http::get_consistent_str(&fi.metadata, suffix).ok_or(Error::FileCorrupt)?;
rustfs_utils::http::insert_str(&mut replacement_metadata, suffix, value.to_owned());
}
}
if let Some(part_checksums) = preserved_part_checksums {
rustfs_utils::http::insert_str(&mut replacement_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums);
}
+28 -2
View File
@@ -725,6 +725,15 @@ impl SetDisks {
skip_verify_bitrot,
)
.await?;
if let Some(expected) = part.integrity.as_ref() {
use crate::io_support::shard_integrity::{PartProofReader, ShardVerifier};
let proof = PartProofReader::new(expected.clone(), files, disks, bucket, object)?;
for (index, reader) in readers.iter_mut().enumerate() {
if let Some(reader) = reader {
reader.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, 0, None)?)?;
}
}
}
let reader_setup_elapsed = reader_setup_stage_start.elapsed();
rustfs_io_metrics::record_get_object_shard_reader_setup_duration(reader_setup_elapsed.as_secs_f64());
rustfs_io_metrics::record_get_object_stage_duration_by_size(
@@ -777,6 +786,7 @@ impl SetDisks {
erasure.data_shards,
)
.await;
reader_setup.bind_integrity(part.integrity.as_ref(), &files, &disks, bucket, object, 0)?;
let reader_setup_elapsed = reader_setup_stage_start.elapsed();
rustfs_io_metrics::record_get_object_shard_reader_setup_duration(reader_setup_elapsed.as_secs_f64());
rustfs_io_metrics::record_get_object_stage_duration_by_size(
@@ -978,6 +988,7 @@ impl SetDisks {
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(&disks));
let sync_spec = PartReaderSetupSpec {
integrity: fi.parts[current_part].integrity.clone(),
part_number,
read_offset,
read_length,
@@ -1032,6 +1043,7 @@ impl SetDisks {
let next_size = fi.parts[next_part].size;
let next_length = next_size.min(remaining_after_current);
let spec = PartReaderSetupSpec {
integrity: fi.parts[next_part].integrity.clone(),
part_number: next_number,
read_offset: 0,
read_length: erasure.shard_file_offset(0, next_length, next_size),
@@ -1657,7 +1669,7 @@ impl SetDisks {
});
let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(disks));
let reader_setup = create_bitrot_readers_until_quorum_with_preference(
let mut reader_setup = create_bitrot_readers_until_quorum_with_preference(
files,
disks,
bucket,
@@ -1681,6 +1693,12 @@ impl SetDisks {
}),
)
.await;
let expected = fi
.parts
.iter()
.find(|part| part.number == part_number)
.and_then(|part| part.integrity.as_ref());
reader_setup.bind_integrity(expected, files, disks, bucket, object, part_offset / erasure.block_size)?;
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_SETUP, reader_setup_stage_start);
let available_shards = reader_setup.available_shards();
@@ -1773,6 +1791,7 @@ impl SetDisks {
/// Per-part parameters for a multipart bitrot reader setup.
struct PartReaderSetupSpec {
integrity: Option<rustfs_filemeta::shard_integrity::PartIntegrity>,
part_number: usize,
read_offset: usize,
read_length: usize,
@@ -1914,7 +1933,7 @@ async fn setup_multipart_part_readers(
metrics_size_bucket: &'static str,
) -> (BitrotReaderSetup, Duration) {
let started = Instant::now();
let setup = create_bitrot_readers_until_quorum_with_preference(
let mut setup = create_bitrot_readers_until_quorum_with_preference(
files,
disks,
bucket,
@@ -1938,6 +1957,13 @@ async fn setup_multipart_part_readers(
}),
)
.await;
if setup
.bind_integrity(spec.integrity.as_ref(), files, disks, bucket, object, spec.read_offset / shard_size)
.is_err()
{
setup = BitrotReaderSetup::new(disks.len());
setup.errors.fill(Some(DiskError::FileCorrupt));
}
(setup, started.elapsed())
}
@@ -56,6 +56,7 @@ pub(crate) struct StripeReadState {
shards: ShardBuffers,
errors: ShardErrors,
read_quorum: usize,
pub(crate) integrity: Option<crate::io_support::shard_integrity::ReconstructionProof>,
}
impl StripeReadState {
@@ -70,6 +71,7 @@ impl StripeReadState {
shards,
errors,
read_quorum,
integrity: None,
}
}
@@ -78,6 +80,7 @@ impl StripeReadState {
shards: SmallVec::new(),
errors: SmallVec::new(),
read_quorum,
integrity: None,
};
state.reset(slot_count, read_quorum);
state
@@ -89,6 +92,7 @@ impl StripeReadState {
self.errors.clear();
self.errors.resize_with(slot_count, || None);
self.read_quorum = read_quorum;
self.integrity = None;
}
pub(crate) fn available_shards(&self) -> usize {
@@ -124,6 +128,13 @@ impl StripeReadState {
&mut self.shards
}
pub(crate) fn verify_reconstructed_integrity(&self) -> std::io::Result<()> {
if let Some(proof) = &self.integrity {
proof.verify(&self.shards)?;
}
Ok(())
}
pub(crate) fn into_parts(self) -> (ShardBuffers, ShardErrors) {
(self.shards, self.errors)
}
+4
View File
@@ -4336,6 +4336,7 @@ impl ECStore {
let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]);
let mut dst_opts = dst_opts.clone();
dst_opts.inherit_shard_integrity(src_info);
if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() {
dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?);
}
@@ -4455,6 +4456,7 @@ impl ECStore {
let mut put_opts = ObjectOptions {
user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
versioned: dst_opts.versioned,
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
version_id: dst_opts.version_id.clone(),
no_lock: dst_opts.no_lock,
mod_time: dst_opts.mod_time,
@@ -4493,6 +4495,7 @@ impl ECStore {
let mut put_opts = ObjectOptions {
user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
versioned: dst_opts.versioned,
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
version_id: dst_opts.version_id.clone(),
no_lock: dst_opts.no_lock,
mod_time: dst_opts.mod_time,
@@ -4541,6 +4544,7 @@ impl ECStore {
let mut put_opts = ObjectOptions {
user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
versioned: dst_opts.versioned,
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
version_id: dst_opts.version_id.clone(),
no_lock: dst_opts.no_lock,
mod_time: dst_opts.mod_time,
+2
View File
@@ -51,6 +51,8 @@ thiserror.workspace = true
s3s = { workspace = true, features = ["minio"] }
regex.workspace = true
arc-swap.workspace = true
sha2.workspace = true
base64-simd.workspace = true
[dev-dependencies]
criterion = { workspace = true, features = ["html_reports"] }
+32 -3
View File
@@ -35,7 +35,7 @@ use uuid::Uuid;
pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
const MAX_ERASURE_SHARDS: usize = 16;
pub(crate) const MAX_ERASURE_SHARDS: usize = 16;
const MAX_FILEINFO_PARTS: usize = 10_000;
const MAX_FILEINFO_CHECKSUMS: usize = 10_000;
const FILEINFO_PART_BITMAP_WORD_BITS: usize = std::mem::size_of::<u64>() * 8;
@@ -57,7 +57,7 @@ const ERR_RESTORE_HDR_MALFORMED: &str = "x-amz-restore header malformed";
const RFC1123: &[FormatItem<'_>] =
format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT");
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
#[derive(Deserialize, Debug, PartialEq, Clone, Default)]
pub struct ObjectPartInfo {
pub etag: String,
pub number: usize,
@@ -69,6 +69,28 @@ pub struct ObjectPartInfo {
// Checksums holds checksums of the part
pub checksums: Option<HashMap<String, String>>,
pub error: Option<String>,
#[serde(default)]
pub integrity: Option<crate::shard_integrity::PartIntegrity>,
}
impl Serialize for ObjectPartInfo {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
// Named fields let release readers ignore optional extensions. Appending
// a ninth element to the old positional array makes those readers fail.
let mut map = serializer.serialize_map(Some(8 + usize::from(self.integrity.is_some())))?;
map.serialize_entry("etag", &self.etag)?;
map.serialize_entry("number", &self.number)?;
map.serialize_entry("size", &self.size)?;
map.serialize_entry("actual_size", &self.actual_size)?;
map.serialize_entry("mod_time", &self.mod_time)?;
map.serialize_entry("index", &self.index)?;
map.serialize_entry("checksums", &self.checksums)?;
map.serialize_entry("error", &self.error)?;
if let Some(integrity) = &self.integrity {
map.serialize_entry("integrity", integrity)?;
}
map.end()
}
}
impl ObjectPartInfo {
@@ -639,7 +661,11 @@ impl<'de> Deserialize<'de> for FileInfo {
}
}
deserializer.deserialize_struct("FileInfo", FILE_INFO_FIELDS, FileInfoVisitor)
let mut file = deserializer.deserialize_struct("FileInfo", FILE_INFO_FIELDS, FileInfoVisitor)?;
if !file.parts.is_empty() {
file.hydrate_shard_integrity().map_err(de::Error::custom)?;
}
Ok(file)
}
}
@@ -1090,6 +1116,7 @@ impl FileInfo {
index,
checksums,
error: None,
integrity: None,
};
for p in self.parts.iter_mut() {
@@ -2172,6 +2199,7 @@ mod tests {
index,
checksums,
error,
integrity: None,
})
}
@@ -2379,6 +2407,7 @@ mod tests {
.collect(),
),
error: Some("part-error".to_string()),
integrity: None,
}],
erasure: ErasureInfo {
algorithm: "erasure-algorithm".to_string(),
+28
View File
@@ -2034,6 +2034,7 @@ impl From<MetaObjectV1Part> for ObjectPartInfo {
index: value.index,
checksums: value.checksums,
error: value.error,
integrity: None,
}
}
}
@@ -2711,6 +2712,33 @@ impl MetaObject {
if all_parts && include_part_checksums {
file_info.hydrate_data_movement_part_checksums()?;
}
if !self.part_numbers.is_empty()
&& rustfs_utils::http::contains_key_str(&file_info.metadata, crate::shard_integrity::SUFFIX_UPLOAD_INTEGRITY)
{
return Err(Error::FileCorrupt);
}
if let Some(commitments) = crate::shard_integrity::descriptor_from_metadata(&file_info.metadata)? {
let layout = crate::shard_integrity::IntegrityLayout::new(
self.erasure_m,
self.erasure_n,
self.erasure_block_size,
file_info.uses_legacy_checksum,
)?;
if commitments.len() != self.part_numbers.len() || commitments.len() != self.part_sizes.len() {
return Err(Error::FileCorrupt);
}
for (i, commitment) in commitments.into_iter().enumerate() {
if commitment.layout != layout
|| usize::try_from(commitment.number).map_err(|_| Error::FileCorrupt)? != self.part_numbers[i]
|| usize::try_from(commitment.size).map_err(|_| Error::FileCorrupt)? != self.part_sizes[i]
{
return Err(Error::FileCorrupt);
}
if all_parts {
file_info.parts[i].integrity = Some(commitment);
}
}
}
Ok(file_info)
}
+1
View File
@@ -28,3 +28,4 @@ pub use filemeta::*;
pub use filemeta_inline::*;
pub use metacache::*;
pub use replication::*;
pub mod shard_integrity;
+517
View File
@@ -0,0 +1,517 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Optional, quorum-owned SHA-256 commitments. The existing part payload and
//! HighwayHash framing are unchanged. A proof file is only an untrusted index;
//! authority comes from the root stored in the selected object's metadata.
use crate::{Error, FileInfo, Result};
use rustfs_utils::http::{contains_key_str, get_consistent_str, insert_str, remove_str};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use uuid::Uuid;
pub const SUFFIX_SHARD_INTEGRITY: &str = "shard-integrity-v1";
pub const SUFFIX_INLINE_INTEGRITY: &str = "shard-integrity-inline-v1";
pub const SUFFIX_UPLOAD_INTEGRITY: &str = "shard-integrity-upload-v1";
pub const MAX_PARTS: usize = 10_000;
pub const MAX_DESCRIPTOR_BYTES: usize = 1024 * 1024;
pub const MAX_INLINE_PROOF_BYTES: usize = 64 * 1024;
pub const INDEX_HEADER_SIZE: usize = 64;
const TABLE_HEADER_SIZE: usize = 32;
const PART_RECORD_SIZE: usize = 64;
const TABLE_MAGIC: &[u8; 8] = b"RFSI\x01\0\0\0";
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct IntegrityLayout {
pub data: u16,
pub parity: u16,
pub block_size: u32,
pub legacy: bool,
}
impl IntegrityLayout {
pub fn new(data: usize, parity: usize, block_size: usize, legacy: bool) -> Result<Self> {
let layout = Self {
data: data.try_into().map_err(|_| Error::FileCorrupt)?,
parity: parity.try_into().map_err(|_| Error::FileCorrupt)?,
block_size: block_size.try_into().map_err(|_| Error::FileCorrupt)?,
legacy,
};
layout.validate()?;
Ok(layout)
}
pub fn validate(&self) -> Result<()> {
if self.data == 0
|| self.data < self.parity
|| self.shards() > crate::fileinfo::MAX_ERASURE_SHARDS
|| self.block_size == 0
|| self.block_size > 64 * 1024 * 1024
{
return Err(Error::FileCorrupt);
}
Ok(())
}
pub fn shards(&self) -> usize {
usize::from(self.data) + usize::from(self.parity)
}
fn bytes(&self) -> [u8; 9] {
let mut out = [0; 9];
out[..2].copy_from_slice(&self.data.to_le_bytes());
out[2..4].copy_from_slice(&self.parity.to_le_bytes());
out[4..8].copy_from_slice(&self.block_size.to_le_bytes());
out[8] = u8::from(self.legacy);
out
}
}
/// One immutable UploadPart/PUT generation. A metadata-only copy retains this
/// identity; overwriting even an identical part creates a new generation.
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct PartIntegrity {
pub layout: IntegrityLayout,
pub number: u32,
pub generation: Uuid,
pub size: u64,
pub stripes: u32,
pub root: [u8; 32],
}
impl PartIntegrity {
pub fn new(layout: IntegrityLayout, number: usize) -> Result<Self> {
let part = Self {
layout,
number: number.try_into().map_err(|_| Error::FileCorrupt)?,
generation: Uuid::new_v4(),
size: 0,
stripes: 0,
root: [0; 32],
};
part.validate()?;
Ok(part)
}
pub fn validate(&self) -> Result<()> {
self.layout.validate()?;
if self.number == 0
|| usize::try_from(self.number).map_err(|_| Error::FileCorrupt)? > MAX_PARTS
|| self.generation.is_nil()
|| self.size > i64::MAX.unsigned_abs()
|| self.size.div_ceil(u64::from(self.layout.block_size)) != u64::from(self.stripes)
{
return Err(Error::FileCorrupt);
}
Ok(())
}
pub fn file_name(&self) -> String {
format!("part.{}.integrity.{}", self.number, self.generation)
}
pub fn height(&self) -> u32 {
u32::BITS - self.stripes.max(1).saturating_sub(1).leading_zeros()
}
pub fn record_size(&self) -> usize {
// At most 16 digests and 32 siblings, independent of untrusted sizes.
32 * (self.layout.shards() + usize::try_from(self.height()).unwrap_or(32))
}
pub fn index_size(&self) -> Result<usize> {
usize::try_from(self.stripes)
.ok()
.and_then(|n| n.checked_mul(self.record_size()))
.and_then(|n| n.checked_add(INDEX_HEADER_SIZE))
.ok_or(Error::FileCorrupt)
}
pub fn record_offset(&self, stripe: u32) -> Result<usize> {
if stripe >= self.stripes {
return Err(Error::FileCorrupt);
}
usize::try_from(stripe)
.ok()
.and_then(|n| n.checked_mul(self.record_size()))
.and_then(|n| n.checked_add(INDEX_HEADER_SIZE))
.ok_or(Error::FileCorrupt)
}
pub fn index_header(&self) -> [u8; INDEX_HEADER_SIZE] {
let mut out = [0; INDEX_HEADER_SIZE];
out[..8].copy_from_slice(b"RFSP\x01\0\0\0");
out[8..17].copy_from_slice(&self.layout.bytes());
out[20..24].copy_from_slice(&self.number.to_le_bytes());
out[24..40].copy_from_slice(self.generation.as_bytes());
out[40..48].copy_from_slice(&self.size.to_le_bytes());
out[48..52].copy_from_slice(&self.stripes.to_le_bytes());
out
}
fn hash_context(&self, domain: &[u8]) -> Sha256 {
let mut hash = Sha256::new();
hash.update(domain);
hash.update(self.layout.bytes());
hash.update(self.number.to_le_bytes());
hash.update(self.generation.as_bytes());
hash
}
pub fn shard_digest(&self, stripe: u32, coding_index: usize, payload: &[u8]) -> Result<[u8; 32]> {
if coding_index >= self.layout.shards() {
return Err(Error::FileCorrupt);
}
let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/payload\0");
hash.update(stripe.to_le_bytes());
hash.update(u16::try_from(coding_index).map_err(|_| Error::FileCorrupt)?.to_le_bytes());
hash.update(u64::try_from(payload.len()).map_err(|_| Error::FileCorrupt)?.to_le_bytes());
hash.update(payload);
Ok(hash.finalize().into())
}
pub fn leaf_digest(&self, stripe: u32, digests: &[u8]) -> Result<[u8; 32]> {
if digests.len() != self.layout.shards() * 32 {
return Err(Error::FileCorrupt);
}
let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/leaf\0");
hash.update(stripe.to_le_bytes());
hash.update(digests);
Ok(hash.finalize().into())
}
pub fn padding_digest(&self, stripe: u32) -> [u8; 32] {
let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/padding\0");
hash.update(stripe.to_le_bytes());
hash.finalize().into()
}
pub fn root_digest(&self, tree_root: &[u8; 32]) -> [u8; 32] {
let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/root\0");
hash.update(self.size.to_le_bytes());
hash.update(self.stripes.to_le_bytes());
hash.update(tree_root);
hash.finalize().into()
}
pub fn verify_record(&self, stripe: u32, record: &[u8]) -> Result<()> {
self.record_offset(stripe)?;
if record.len() != self.record_size() {
return Err(Error::FileCorrupt);
}
let (digests, siblings) = record.split_at(self.layout.shards() * 32);
let mut digest = self.leaf_digest(stripe, digests)?;
let mut position = stripe;
for sibling in siblings.as_chunks::<32>().0 {
digest = if position & 1 == 0 {
node_digest(&digest, sibling)
} else {
node_digest(sibling, &digest)
};
position >>= 1;
}
if self.root_digest(&digest) != self.root {
return Err(Error::FileCorrupt);
}
Ok(())
}
pub fn verify_shard(&self, stripe: u32, coding_index: usize, payload: &[u8], record: &[u8]) -> Result<()> {
self.verify_record(stripe, record)?;
let digest = self.shard_digest(stripe, coding_index, payload)?;
if record.get(coding_index * 32..(coding_index + 1) * 32) != Some(digest.as_slice()) {
return Err(Error::FileCorrupt);
}
Ok(())
}
}
pub fn node_digest(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] {
let mut hash = Sha256::new();
hash.update(b"rustfs/shard-integrity/v1/node\0");
hash.update(left);
hash.update(right);
hash.finalize().into()
}
pub fn encode_descriptor(parts: &[PartIntegrity]) -> Result<String> {
let Some(first) = parts.first() else { return Err(Error::FileCorrupt) };
if parts.len() > MAX_PARTS {
return Err(Error::FileCorrupt);
}
let mut out = vec![0; TABLE_HEADER_SIZE];
out[..8].copy_from_slice(TABLE_MAGIC);
out[8..17].copy_from_slice(&first.layout.bytes());
out[28..32].copy_from_slice(&u32::try_from(parts.len()).map_err(|_| Error::FileCorrupt)?.to_le_bytes());
let mut previous = 0;
for part in parts {
part.validate()?;
if part.layout != first.layout || part.number <= previous {
return Err(Error::FileCorrupt);
}
previous = part.number;
out.extend_from_slice(&part.number.to_le_bytes());
out.extend_from_slice(part.generation.as_bytes());
out.extend_from_slice(&part.size.to_le_bytes());
out.extend_from_slice(&part.stripes.to_le_bytes());
out.extend_from_slice(&part.root);
}
Ok(base64_simd::STANDARD.encode_to_string(out))
}
pub fn decode_descriptor(value: &str) -> Result<Vec<PartIntegrity>> {
if value.len() > MAX_DESCRIPTOR_BYTES {
return Err(Error::FileCorrupt);
}
let bytes = base64_simd::STANDARD.decode_to_vec(value).map_err(|_| Error::FileCorrupt)?;
// Reject non-canonical encodings as well as unsupported versions/reserved bits.
if bytes.len() < TABLE_HEADER_SIZE
|| &bytes[..8] != TABLE_MAGIC
|| bytes[16] > 1
|| bytes[17..28].iter().any(|v| *v != 0)
|| base64_simd::STANDARD.encode_to_string(&bytes) != value
{
return Err(Error::FileCorrupt);
}
let u32_at = |start| -> Result<u32> {
Ok(u32::from_le_bytes(
bytes
.get(start..start + 4)
.ok_or(Error::FileCorrupt)?
.try_into()
.map_err(|_| Error::FileCorrupt)?,
))
};
let count = usize::try_from(u32_at(28)?).map_err(|_| Error::FileCorrupt)?;
if count == 0 || count > MAX_PARTS || bytes.len() != TABLE_HEADER_SIZE + count * PART_RECORD_SIZE {
return Err(Error::FileCorrupt);
}
let layout = IntegrityLayout {
data: u16::from_le_bytes([bytes[8], bytes[9]]),
parity: u16::from_le_bytes([bytes[10], bytes[11]]),
block_size: u32_at(12)?,
legacy: bytes[16] == 1,
};
layout.validate()?;
let mut parts = Vec::with_capacity(count);
let mut previous = 0;
for record in bytes[TABLE_HEADER_SIZE..].as_chunks::<PART_RECORD_SIZE>().0 {
let part = PartIntegrity {
layout,
number: u32::from_le_bytes(record[..4].try_into().map_err(|_| Error::FileCorrupt)?),
generation: Uuid::from_slice(&record[4..20])
.ok()
.filter(|id| !id.is_nil())
.ok_or(Error::FileCorrupt)?,
size: u64::from_le_bytes(record[20..28].try_into().map_err(|_| Error::FileCorrupt)?),
stripes: u32::from_le_bytes(record[28..32].try_into().map_err(|_| Error::FileCorrupt)?),
root: record[32..64].try_into().map_err(|_| Error::FileCorrupt)?,
};
part.validate()?;
if part.number <= previous {
return Err(Error::FileCorrupt);
}
previous = part.number;
parts.push(part);
}
Ok(parts)
}
pub fn descriptor_from_metadata(metadata: &HashMap<String, String>) -> Result<Option<Vec<PartIntegrity>>> {
if !contains_key_str(metadata, SUFFIX_SHARD_INTEGRITY) {
// An orphaned inline proof must not silently turn a protected object into legacy.
if contains_key_str(metadata, SUFFIX_INLINE_INTEGRITY) {
return Err(Error::FileCorrupt);
}
return Ok(None);
}
decode_descriptor(get_consistent_str(metadata, SUFFIX_SHARD_INTEGRITY).ok_or(Error::FileCorrupt)?).map(Some)
}
pub fn clear_integrity_metadata(metadata: &mut HashMap<String, String>) {
for suffix in [SUFFIX_SHARD_INTEGRITY, SUFFIX_INLINE_INTEGRITY, SUFFIX_UPLOAD_INTEGRITY] {
remove_str(metadata, suffix);
}
}
impl FileInfo {
/// Decode once at the metadata boundary and retain each part's validated
/// commitment. Reading part n must not reparse a 10,000-part descriptor.
pub fn hydrate_shard_integrity(&mut self) -> Result<()> {
if !self.parts.is_empty() && contains_key_str(&self.metadata, SUFFIX_UPLOAD_INTEGRITY) {
return Err(Error::FileCorrupt);
}
let Some(parts) = descriptor_from_metadata(&self.metadata)? else {
if self.parts.iter().any(|part| part.integrity.is_some()) {
return Err(Error::FileCorrupt);
}
return Ok(());
};
let layout = IntegrityLayout::new(
self.erasure.data_blocks,
self.erasure.parity_blocks,
self.erasure.block_size,
self.uses_legacy_checksum,
)?;
if parts.len() != self.parts.len() {
return Err(Error::FileCorrupt);
}
for (expected, actual) in parts.into_iter().zip(&mut self.parts) {
if expected.layout != layout
|| usize::try_from(expected.number).map_err(|_| Error::FileCorrupt)? != actual.number
|| usize::try_from(expected.size).map_err(|_| Error::FileCorrupt)? != actual.size
|| actual.integrity.as_ref().is_some_and(|value| value != &expected)
{
return Err(Error::FileCorrupt);
}
actual.integrity = Some(expected);
}
Ok(())
}
pub fn persist_shard_integrity(&mut self) -> Result<()> {
let parts: Vec<_> = self.parts.iter().filter_map(|part| part.integrity.clone()).collect();
if parts.is_empty() {
clear_integrity_metadata(&mut self.metadata);
return Ok(());
}
if parts.len() != self.parts.len() {
return Err(Error::FileCorrupt);
}
insert_str(&mut self.metadata, SUFFIX_SHARD_INTEGRITY, encode_descriptor(&parts)?);
self.hydrate_shard_integrity()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ObjectPartInfo;
fn part(number: usize) -> PartIntegrity {
let mut part =
PartIntegrity::new(IntegrityLayout::new(12, 4, 1024 * 1024, false).expect("layout"), number).expect("part");
part.size = 1024 * 1024 + 123;
part.stripes = 2;
part.root = [7; 32];
part
}
#[test]
fn integrity_descriptor_is_bounded_canonical_and_complete() {
let parts: Vec<_> = (1..=MAX_PARTS).map(part).collect();
let encoded = encode_descriptor(&parts).expect("maximum multipart descriptor");
assert_eq!(encoded.len(), 853_376);
assert_eq!(decode_descriptor(&encoded).expect("decode maximum table"), parts);
assert!(encode_descriptor(&[part(2), part(1)]).is_err());
assert!(encode_descriptor(&[part(1), part(1)]).is_err());
assert!(decode_descriptor(&format!("{encoded} ")).is_err());
assert!(decode_descriptor(&"A".repeat(MAX_DESCRIPTOR_BYTES + 1)).is_err());
let mut bytes = base64_simd::STANDARD
.decode_to_vec(encode_descriptor(&[part(1)]).expect("encode"))
.expect("base64");
for offset in [4, 17, 28, 36] {
let mut invalid = bytes.clone();
if offset == 36 {
invalid[36..52].fill(0);
} else {
invalid[offset] = 255;
}
assert!(
decode_descriptor(&base64_simd::STANDARD.encode_to_string(invalid)).is_err(),
"offset {offset}"
);
}
bytes.push(0);
assert!(decode_descriptor(&base64_simd::STANDARD.encode_to_string(bytes)).is_err());
}
#[test]
fn integrity_metadata_distinguishes_absence_corruption_and_part_mismatch() {
let mut file = FileInfo::new("object", 12, 4);
file.add_object_part(1, String::new(), 1024 * 1024 + 123, None, 0, None, None);
assert!(descriptor_from_metadata(&file.metadata).expect("legacy absence").is_none());
file.parts[0].integrity = Some(part(1));
file.persist_shard_integrity().expect("persist");
let mut decoded = file.clone();
decoded.parts[0].integrity = None;
decoded.hydrate_shard_integrity().expect("hydrate old peer response");
assert_eq!(decoded.parts[0].integrity, file.parts[0].integrity);
decoded.parts[0].size += 1;
assert!(decoded.hydrate_shard_integrity().is_err());
file.metadata
.insert(format!("x-minio-internal-{SUFFIX_SHARD_INTEGRITY}"), "invalid".to_owned());
assert!(descriptor_from_metadata(&file.metadata).is_err());
}
#[test]
fn integrity_descriptor_matches_independent_v1_vector() {
// Header, UUID byte order, dimensions and root frozen independently
// with Python struct/hashlib, rather than a Rust round trip.
let encoded = "UkZTSQEAAAACAAIACAAAAAAAAAAAAAAAAAAAAAEAAAABAAAAABEiM0RVRneImaq7zN3u/xAAAAAAAAAAAgAAACH+n+LW6++vtoSsUQbqfbsZYr4G+/GrIZa8s/oaPC/G";
let parts = decode_descriptor(encoded).expect("known descriptor");
assert_eq!(parts.len(), 1);
let part = &parts[0];
assert_eq!(part.layout, IntegrityLayout::new(2, 2, 8, false).expect("layout"));
assert_eq!(part.generation.to_string(), "00112233-4455-4677-8899-aabbccddeeff");
assert_eq!((part.number, part.size, part.stripes), (1, 16, 2));
assert_eq!(
part.root,
[
33, 254, 159, 226, 214, 235, 239, 175, 182, 132, 172, 81, 6, 234, 125, 187, 25, 98, 190, 6, 251, 241, 171, 33,
150, 188, 179, 250, 26, 60, 47, 198
]
);
assert_eq!(encode_descriptor(&parts).expect("encode known descriptor"), encoded);
}
#[derive(Serialize, Deserialize)]
struct ReleasePart {
etag: String,
number: usize,
size: usize,
actual_size: i64,
mod_time: Option<time::OffsetDateTime>,
index: Option<bytes::Bytes>,
checksums: Option<HashMap<String, String>>,
error: Option<String>,
}
#[test]
fn integrity_part_extension_is_readable_by_release_decoder_and_reads_old_arrays() {
let old = ReleasePart {
etag: "same-etag".to_owned(),
number: 1,
size: 5,
actual_size: 5,
mod_time: None,
index: None,
checksums: None,
error: None,
};
let tuple = rmp_serde::to_vec(&old).expect("release positional writer");
assert_eq!(tuple[0], 0x98, "release wrote eight positional fields");
let mut new = ObjectPartInfo::unmarshal(&tuple).expect("read release part");
assert!(new.integrity.is_none());
new.integrity = Some(part(1));
let bytes = new.marshal_msg().expect("new writer");
assert_eq!(bytes[0], 0x89, "extension uses nine named fields");
let decoded: ReleasePart = rmp_serde::from_slice(&bytes).expect("release decoder ignores unknown named field");
assert_eq!(decoded.etag, old.etag);
assert_eq!(decoded.size, 5);
assert_eq!(ObjectPartInfo::unmarshal(&bytes).expect("new decoder").integrity, new.integrity);
}
}
+7 -1
View File
@@ -456,7 +456,7 @@ fn warn_mrf_journal_write(err: &super::DiskError) {
/// Translate an intent into the prioritized heal request the issue specifies:
/// decode failures go Urgent ECDecode, metadata corruption goes High
/// Metadata, partial writes go Normal object heal.
/// Metadata, partial writes go Normal-priority object heal with Deep verification.
pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
let bucket = intent.bucket.to_string();
let object = intent.object.to_string();
@@ -484,6 +484,11 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
),
};
let mut options = HealOptions::default();
if matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::PartialWrite) {
// Presence-only repair cannot discharge a protected object's durable
// obligation. Verify payloads even after new protection is disabled.
options.scan_mode = rustfs_heal_contracts::heal_channel::HealScanMode::Deep;
}
if !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption)
&& let Some(scope) = intent.scope
{
@@ -2716,5 +2721,6 @@ mod tests {
});
assert!(matches!(partial.heal_type, HealType::Object { .. }));
assert_eq!(partial.priority, HealPriority::Normal);
assert_eq!(partial.options.scan_mode, rustfs_heal_contracts::heal_channel::HealScanMode::Deep);
}
}
+1 -1
View File
@@ -1169,7 +1169,7 @@ impl HealStorageAPI for ECStoreHealStorage {
} else {
None
}
} else if error.is_none() && !opts.dry_run {
} else if error.is_none() && !opts.dry_run && item.integrity_verified {
let ok_drive_state = DriveState::Ok.to_string();
let all_after_drives_ok = item.after.drives.iter().all(|drive| drive.state == ok_drive_state);
match (
@@ -762,10 +762,12 @@ mod absence_receipt_regressions {
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.processed, 3);
assert_eq!(outcome.counters.healed, 1, "completed cleanup must be repaired: {outcome:?}");
assert_eq!(outcome.counters.unchanged, 2);
assert_eq!(outcome.counters.unknown, 0);
// Exact historical absence has its own proof. The two live legacy
// versions remain readable but carry no independent payload receipt.
assert_eq!(outcome.counters.unchanged, 0);
assert_eq!(outcome.counters.unknown, 2);
assert_eq!(outcome.counters.failed, 0);
assert_eq!(outcome.counters.skipped, 0);
assert_eq!(outcome.counters.skipped, 2);
assert_versions(&store, bucket, &old, &current).await;
}
+35 -14
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#![recursion_limit = "256"]
use http::HeaderMap;
use rustfs_heal::heal::{
outcome::{HealObjectDisposition, HealTraversalCoverage},
@@ -24,7 +26,8 @@ use tokio::io::AsyncReadExt as _;
mod storage_api;
use storage_api::integration::{
DiskAPI as _, DiskError, DiskOption, Endpoint, NamespaceLocking as _, ObjectIO as _, ReadOptions, new_disk,
DiskAPI as _, DiskError, DiskOption, Endpoint, NamespaceLocking as _, ObjectIO as _, ReadOptions, ShardIntegrityWriteMode,
new_disk,
};
fn deep_heal_task(storage: &Arc<ECStoreHealStorage>, bucket: &str, object: &str, dry_run: bool) -> HealTask {
@@ -75,6 +78,7 @@ async fn deep_heal_truncated_xlmeta_scenario() {
let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
let bucket = "truncated-xlmeta";
env.make_bucket(bucket, false).await;
env.make_bucket("truncated-xlmeta-versioned", true).await;
let payload = vec![0x7b; 4 * 1024 * 1024];
let mut endpoint = Endpoint::try_from(env.disk_paths[0].to_str().expect("UTF-8 disk path")).expect("target endpoint");
endpoint.set_pool_index(0);
@@ -88,13 +92,13 @@ async fn deep_heal_truncated_xlmeta_scenario() {
..Default::default()
};
for damage in ["length-prefix", "metadata-body", "crc-tail", "versioned"] {
for (damage, protected) in ["length-prefix", "metadata-body", "crc-tail", "versioned"]
.into_iter()
.flat_map(|damage| [false, true].map(|protected| (damage, protected)))
{
let versioned = damage == "versioned";
let bucket = if versioned { "truncated-xlmeta-versioned" } else { bucket };
if versioned {
env.make_bucket(bucket, true).await;
}
let object = format!("{damage}/object.bin");
let object = format!("{damage}-{protected}/object.bin");
let mut reader = PutObjReader::from_vec(payload.clone());
env.ecstore
.put_object(
@@ -103,6 +107,11 @@ async fn deep_heal_truncated_xlmeta_scenario() {
&mut reader,
&ObjectOptions {
versioned,
shard_integrity_write_mode: Some(if protected {
ShardIntegrityWriteMode::Protected
} else {
ShardIntegrityWriteMode::Legacy
}),
..Default::default()
},
)
@@ -168,15 +177,26 @@ async fn deep_heal_truncated_xlmeta_scenario() {
let outcome = task.get_outcome().await;
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
assert_eq!(outcome.counters.processed, 1);
assert_eq!(outcome.counters.healed, 1, "{damage}: {outcome:?}");
assert_eq!(outcome.counters.unknown, 0);
assert_eq!(outcome.counters.skipped, 0);
// Both modes restore metadata and shards. Only independently verified
// payloads authorize a strong repair receipt.
assert_eq!(outcome.counters.healed, u64::from(protected), "{damage}: {outcome:?}");
assert_eq!(outcome.counters.unknown, u64::from(!protected));
assert_eq!(outcome.counters.skipped, u64::from(!protected));
assert_eq!(outcome.counters.failed, 0);
assert_eq!(outcome.counters.attempt_failures, 0);
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Repaired);
assert_eq!(
outcome.objects[0].disposition,
if protected {
HealObjectDisposition::Repaired
} else {
HealObjectDisposition::Unknown
}
);
assert_eq!(read_error, DiskError::FileCorrupt);
let results = task.get_result_items().await;
assert_eq!(results.len(), 1);
assert_eq!(results[0].drives_healed(), Some(1));
assert_eq!(results[0].integrity_verified, protected);
assert_eq!(results[0].before.drives.len(), 16);
assert_eq!(results[0].after.drives.len(), 16);
assert_eq!(results[0].before.drives[0].state, DriveState::Corrupt.to_string());
@@ -217,8 +237,8 @@ async fn deep_heal_truncated_xlmeta_scenario() {
repeat.execute().await.expect("repeated heal completes");
let repeat_outcome = repeat.get_outcome().await;
assert_eq!(repeat_outcome.counters.healed, 0);
assert_eq!(repeat_outcome.counters.unchanged, 1);
assert_eq!(repeat_outcome.counters.unknown, 0);
assert_eq!(repeat_outcome.counters.unchanged, u64::from(protected));
assert_eq!(repeat_outcome.counters.unknown, u64::from(!protected));
assert_eq!(tokio::fs::read(&target_meta).await.expect("read repeated-heal target"), healed_bytes);
let mut reader = env
.ecstore
@@ -273,8 +293,9 @@ async fn deep_heal_truncated_xlmeta_scenario() {
let outcome = quorum_task.get_outcome().await;
if damaged == 4 {
result.expect("twelve authoritative members must repair four damaged copies");
assert_eq!(outcome.counters.healed, 1);
assert_eq!(outcome.counters.unknown, 0);
assert_eq!(outcome.counters.healed, u64::from(protected));
assert_eq!(outcome.counters.unknown, u64::from(!protected));
assert_eq!(quorum_task.get_result_items().await[0].drives_healed(), Some(4));
assert!(
quorum_task.get_result_items().await[0]
.after
+55 -20
View File
@@ -16,6 +16,8 @@
//! tests never start a scanner, auto-heal, or an explicit Admin Heal request.
//! Run with nextest: channel and storage singletons require separate processes.
#![recursion_limit = "256"]
use rustfs_heal::heal::{
manager::{HealConfig, HealManager},
mrf_queue::{self, snapshot::inspect_local_committed_snapshot},
@@ -178,6 +180,8 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() {
[
("RUSTFS_HEAL_MRF_ENABLE", Some("true")),
("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", Some("false")),
("RUSTFS_SHARD_INTEGRITY_WRITE", Some("false")),
("RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED", Some("false")),
],
async {
let root = tempfile::tempdir().expect("test directory should be created");
@@ -313,14 +317,21 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() {
);
assert!(
wait_until(|| async {
inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
.await
.expect("cleanup snapshot must validate")
.is_none()
let snapshot = manager.operations_snapshot().await;
snapshot.queue_length == 0 && snapshot.active_tasks == 0
})
.await,
"verified repair must release durable responsibility"
"legacy repair attempts must finish without inventing verification"
);
assert!(snapshot_contains("new.bin").await);
assert!(snapshot_contains("versioned.bin").await);
manager.stop().await.expect("test manager should stop");
let before_purge = inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
.await
.expect("legacy checkpoint must validate")
.expect("unverified legacy responsibility must remain")
.payload()
.to_vec();
// Deleting an existing marker by VersionId removes a version; it
// must not create a new partial-write repair responsibility.
for path in &env.disk_paths[12..] {
@@ -346,12 +357,14 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() {
)
.await
.expect("explicit marker purge should retain quorum");
assert!(
assert_eq!(
inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
.await
.expect("purge must not create a checkpoint")
.is_none(),
"a physical marker purge must not be admitted as a marker creation repair"
.expect("purge checkpoint must validate")
.expect("existing legacy checkpoint must remain")
.payload(),
before_purge,
"a physical marker purge must not add a marker creation repair"
);
for (path, disk) in env.disk_paths[12..].iter().zip(&all[12..]) {
tokio::fs::remove_file(path).await.expect("remove purge outage sentinel");
@@ -361,7 +374,6 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() {
disk.reset_health_for_store_init_retry();
}
*set.disks.write().await = all.iter().cloned().map(Some).collect();
manager.stop().await.expect("test manager should stop");
},
)
.await;
@@ -391,6 +403,15 @@ fn partial_write_crash_fixture() {
#[tokio::test]
async fn partial_write_sigkill_replay_rearms_and_repairs() {
partial_write_sigkill_replay_scenario(true).await;
}
#[tokio::test]
async fn legacy_sigkill_replay_repairs_without_releasing_unverified_responsibility() {
partial_write_sigkill_replay_scenario(false).await;
}
async fn partial_write_sigkill_replay_scenario(protected: bool) {
use std::process::{Command, Stdio};
let root = tempfile::tempdir().expect("crash test directory");
let log = std::fs::File::create(root.path().join("child.log")).expect("child log");
@@ -399,6 +420,8 @@ async fn partial_write_sigkill_replay_rearms_and_repairs() {
.env("RUSTFS_TEST_PARTIAL_WRITE_CRASH_ROOT", root.path())
.env("RUSTFS_HEAL_MRF_ENABLE", "true")
.env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false")
.env("RUSTFS_SHARD_INTEGRITY_WRITE", protected.to_string())
.env("RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED", protected.to_string())
.stdout(Stdio::from(log.try_clone().expect("clone child log")))
.stderr(Stdio::from(log))
.spawn()
@@ -432,15 +455,27 @@ async fn partial_write_sigkill_replay_rearms_and_repairs() {
"replayed responsibility must heal the returning member"
);
assert_payload(&env, "partial-crash", "crash.bin", None, b"durable partial write across SIGKILL").await;
assert!(
wait_until(|| async {
inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
.await
.expect("valid checkpoint")
.is_none()
})
.await,
"replayed responsibility must be released only after verified repair"
);
if protected {
assert!(
wait_until(|| async {
inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
.await
.expect("valid checkpoint")
.is_none()
})
.await,
"replayed responsibility must be released only after verified repair"
);
} else {
assert!(
wait_until(|| async {
let snapshot = manager.operations_snapshot().await;
snapshot.queue_length == 0 && snapshot.active_tasks == 0
})
.await,
"legacy replay attempts must finish"
);
assert!(snapshot_contains("crash.bin").await, "unverified legacy responsibility must remain");
}
manager.stop().await.expect("restarted manager should stop");
}
@@ -0,0 +1,210 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![recursion_limit = "256"]
use rustfs_heal::heal::{
outcome::HealObjectDisposition,
storage::{ECStoreHealStorage, HealStorageAPI},
};
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
use rustfs_test_utils::TestECStoreEnv;
use serial_test::serial;
use tokio::io::AsyncReadExt as _;
mod storage_api;
use storage_api::integration::{DiskAPI, ObjectIO, ObjectOptions, PutObjReader, ReadOptions, ShardIntegrityWriteMode};
#[tokio::test]
#[serial]
async fn legacy_repair_reports_execution_without_strong_receipt() {
let root = tempfile::tempdir().expect("legacy receipt fixture");
let env = TestECStoreEnv::builder()
.base_dir(root.path())
.prefix("legacy_receipt")
.build()
.await;
let bucket = "legacy-receipt";
let object = "missing-shard";
env.make_bucket(bucket, false).await;
let expected = vec![0x5c; 1024 * 1024 + 37];
env.ecstore
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(expected.clone()),
&ObjectOptions {
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
..Default::default()
},
)
.await
.expect("legacy object");
let set = env.ecstore.pools[0].get_disks(0);
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
let meta = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("metadata");
tokio::fs::remove_file(
env.disk_paths[0]
.join(bucket)
.join(object)
.join(meta.data_dir.expect("external directory").to_string())
.join("part.1"),
)
.await
.expect("remove one shard");
let result = ECStoreHealStorage::new(env.ecstore.clone())
.heal_object_with_receipt(
bucket,
object,
None,
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("legacy repair");
assert!(result.error.is_none(), "{:?}", result.error);
assert_eq!(result.item.drives_healed(), Some(1));
assert!(!result.item.integrity_verified);
assert!(result.receipt.is_none(), "physical repair does not prove original identity");
let mut reader = env
.ecstore
.get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
.await
.expect("read repaired object");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("exact recovered body");
assert_eq!(actual, expected);
}
#[tokio::test]
#[serial]
async fn receipt_requires_independent_deep_verification() {
let root = tempfile::tempdir().expect("receipt fixture");
let env = TestECStoreEnv::builder()
.base_dir(root.path())
.prefix("shard_identity_receipt")
.build()
.await;
let bucket = "shard-identity-receipt";
env.make_bucket(bucket, false).await;
let set = env.ecstore.pools[0].get_disks(0);
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
let storage = ECStoreHealStorage::new(env.ecstore.clone());
for corrupt_count in [1usize, 3] {
let object = format!("target-{corrupt_count}");
let donor = format!("donor-{corrupt_count}");
for (name, byte) in [(&object, 0x3c), (&donor, 0xa9)] {
env.ecstore
.put_object(
bucket,
name,
&mut PutObjReader::from_vec(vec![byte; 1024 * 1024 + 123]),
&ObjectOptions {
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
no_lock: true,
..Default::default()
},
)
.await
.expect("commit all fixture shards");
}
let mut target_meta = Vec::new();
let mut donor_meta = Vec::new();
for disk in &disks {
target_meta.push(
disk.read_version("", bucket, &object, "", &ReadOptions::default())
.await
.expect("target metadata"),
);
donor_meta.push(
disk.read_version("", bucket, &donor, "", &ReadOptions::default())
.await
.expect("donor metadata"),
);
}
let mut retained = Vec::new();
for (slot, target) in target_meta.iter().enumerate() {
let target_path = env.disk_paths[slot]
.join(bucket)
.join(&object)
.join(target.data_dir.expect("target directory").to_string())
.join("part.1");
let original = tokio::fs::read(&target_path).await.expect("original shard");
if target.erasure.index <= corrupt_count {
let donor_slot = donor_meta
.iter()
.position(|part| part.erasure.index == target.erasure.index)
.expect("same donor coding index");
let donor_path = env.disk_paths[donor_slot]
.join(bucket)
.join(&donor)
.join(donor_meta[donor_slot].data_dir.expect("donor directory").to_string())
.join("part.1");
let replacement = tokio::fs::read(donor_path).await.expect("complete donor shard");
assert_eq!(replacement.len(), original.len());
tokio::fs::write(&target_path, replacement)
.await
.expect("replace intact shard");
} else {
retained.push((target_path, original));
}
}
let normal = storage
.heal_object_with_receipt(
bucket,
&object,
None,
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Normal,
..Default::default()
},
)
.await
.expect("normal presence scan");
assert!(normal.receipt.is_none(), "a presence scan cannot certify payload integrity");
let result = storage
.heal_object_with_receipt(
bucket,
&object,
None,
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await;
if corrupt_count == 1 {
let result = result.expect("recoverable repair");
assert!(result.error.is_none());
assert_eq!(
result.receipt.expect("verified repair receipt").disposition,
HealObjectDisposition::Repaired,
"presence alone must not produce VerifiedHealthy"
);
} else if let Ok(result) = result {
assert!(result.error.is_some(), "below quorum cannot be a success");
assert!(result.receipt.is_none(), "no completion receipt without authoritative sources");
}
for (path, original) in retained {
assert_eq!(tokio::fs::read(path).await.expect("retained correct shard"), original);
}
}
}
+1 -1
View File
@@ -23,7 +23,7 @@ pub(crate) mod integration {
pub(crate) use rustfs_ecstore::api::disk::{
DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, RUSTFS_META_BUCKET, ReadOptions, new_disk,
};
pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
pub(crate) use rustfs_ecstore::api::storage::ECStore;
pub(crate) use rustfs_storage_api::BucketOperations;
pub(crate) use rustfs_storage_api::MakeBucketOptions;
+4
View File
@@ -36,6 +36,10 @@ const DRIVE_STATE_OK: &str = "ok";
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct HealResultItem {
/// Only the coordinator's authenticated full scan may set this. Transported
/// or old-peer results default to unproven and cannot advance heal receipts.
#[serde(skip)]
pub integrity_verified: bool,
#[serde(rename = "resultId")]
pub result_index: usize,
#[serde(rename = "type")]
@@ -288,6 +288,9 @@ pub struct RenameFileRequest {
pub dst_volume: ::prost::alloc::string::String,
#[prost(string, tag = "5")]
pub dst_path: ::prost::alloc::string::String,
/// Apply the destination's payload/metadata durability policy before success.
#[prost(bool, tag = "6")]
pub durable: bool,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RenameFileResponse {
@@ -295,6 +298,8 @@ pub struct RenameFileResponse {
pub success: bool,
#[prost(message, optional, tag = "2")]
pub error: ::core::option::Option<Error>,
#[prost(bool, tag = "3")]
pub durability_applied: bool,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct WriteRequest {
+25
View File
@@ -1139,6 +1139,11 @@ pub fn canonical_rename_file_request_body(
body.push_str(&request.src_path)?;
body.push_str(&request.dst_volume)?;
body.push_str(&request.dst_path)?;
// Preserve the release signature for ordinary renames. An older server
// computes a different digest for a durable request and rejects mutation.
if request.durable {
body.push_bool(true);
}
Ok(body.finish())
}
@@ -1450,6 +1455,7 @@ mod disk_mutation_canonical_tests {
assert_all_distinct(&bodies);
let rename_file = RenameFileRequest {
durable: false,
disk: "d".into(),
src_volume: "sv".into(),
src_path: "sp".into(),
@@ -1458,6 +1464,7 @@ mod disk_mutation_canonical_tests {
};
let mut bodies = vec![canonical_rename_file_request_body(&rename_file).unwrap()];
for mutate in [
|r: &mut RenameFileRequest| r.durable = true,
|r: &mut RenameFileRequest| r.disk = "d2".into(),
|r: &mut RenameFileRequest| r.src_volume = "sv2".into(),
|r: &mut RenameFileRequest| r.src_path = "sp2".into(),
@@ -1632,10 +1639,28 @@ mod disk_mutation_canonical_tests {
);
}
#[test]
fn durable_rename_extension_preserves_old_wire_defaults_and_signatures() {
use crate::proto_gen::node_service::RenameFileResponse;
use prost::Message;
// Release wire messages omitted request tag 6 and response tag 3.
let mut request =
RenameFileRequest::decode(b"\x0a\x01d\x12\x01s\x1a\x01p\x22\x01v\x2a\x01q".as_slice()).expect("release request");
assert!(!request.durable);
let old_body = b"rustfs-rename-file-request-v1\0\0\0\0\0\0\0\0\x01d\0\0\0\0\0\0\0\x01s\0\0\0\0\0\0\0\x01p\0\0\0\0\0\0\0\x01v\0\0\0\0\0\0\0\x01q";
assert_eq!(canonical_rename_file_request_body(&request).expect("body"), old_body);
request.durable = true;
assert_ne!(canonical_rename_file_request_body(&request).expect("durable body"), old_body);
let old_success = RenameFileResponse::decode(b"\x08\x01".as_slice()).expect("release response");
assert!(old_success.success);
assert!(!old_success.durability_applied, "old success cannot certify durable publication");
}
#[test]
fn disk_mutation_canonical_domains_are_distinct_per_message() {
// The same field values must never authenticate one RPC's request as another's.
let rename_file = RenameFileRequest {
durable: false,
disk: "d".into(),
src_volume: "sv".into(),
src_path: "sp".into(),
+3
View File
@@ -219,11 +219,14 @@ message RenameFileRequest {
string src_path = 3;
string dst_volume = 4;
string dst_path = 5;
// Apply the destination's payload/metadata durability policy before success.
bool durable = 6;
}
message RenameFileResponse {
bool success = 1;
optional Error error = 2;
bool durability_applied = 3;
}
message WriteRequest {
+3
View File
@@ -24,3 +24,6 @@ procedure is required.
For storage dashboards, see [Storage metrics and observer selection](operations/storage-metrics.md):
drive ownership, snapshot freshness, counter queries, and rolling upgrades.
For optional shard commitments, see [Independent shard integrity rollout](operations/shard-integrity-rollout.md):
activation, legacy repair results, multipart mode changes, and rollback limits.
+68 -1
View File
@@ -156,6 +156,73 @@ Each shard file is self-verifying against silent disk corruption.
---
### 5.1 Independent shard commitments
New PUTs and newly initiated multipart uploads retain `CSumAlgo = 1` and the
existing `[HighwayHash256][shard]` frames. Independent commitments are disabled
for new writes by default; both `RUSTFS_SHARD_INTEGRITY_WRITE` and
`RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED` must be enabled to start protecting new
writes. Existing protected objects and uploads retain their mode. An independent SHA-256
commitment protects against replacing a complete frame with a same-length donor
frame whose self-contained checksum is valid (backlog#2497).
The commitment describes an immutable part generation, with a random UUID,
part number, exact encoded length, erasure geometry, codec mode, stripe count,
and Merkle root. Each leaf commits to the ordered SHA-256 digests of **all**
encoded shards in one stripe. Payload digests include generation, part, stripe,
coding index, length, and geometry. The root also commits to the final part size.
A valid metadata quorum selects the expected root; neither RS consistency nor
an intact adjacent HighwayHash checksum establishes the root.
The dual-prefix internal `shard-integrity-v1` metadata value encodes a canonical
Base64 table: a 32-byte header and 64 bytes per part, up to 10,000 sorted unique
parts. The same table is stored under both internal prefixes. Readers reject a
malformed, conflicting, or incomplete descriptor instead of treating it as
legacy metadata. Each external part has an immutable
`part.N.integrity.<generation UUID>` sidecar, replicated on every participating
disk. Its 64-byte header is followed by stripe records containing all shard
digests and a Merkle path. An inline object stores the small proof under the
dual-prefix `shard-integrity-inline-v1` key.
GET authenticates a stripe record against the selected root, then verifies each
source shard before decoding or emitting bytes. Reconstructed data is also
checked against its expected digest. Range reads fetch only the proof records
for touched stripes; deferred parity readers retain their exact stripe position.
A missing or corrupt index replica can use another authenticated replica. Deep
Heal verifies at the coordinator, including data returned by older disk servers,
and restores missing indexes only from proofs matching the existing root. It
never mints a new commitment from suspect stored bytes. Index-only repair leaves
payload and `xl.meta` bytes unchanged and requires acknowledged durable publish.
Writes publish the payload and proof on the same write quorum. UploadPart uses
a fresh generation for each replacement, includes that generation and root in
part-metadata quorum selection, and publishes its index before the existing
part transaction switches data and metadata. Settlement removes only the
obsolete generation; rollback retains the old index. Interrupted preparation
can leave unreferenced files until the upload directory is reclaimed. Ordinary
metadata COPY preserves commitments. Physical rewrites inherit their source
mode: protected sources create new commitments after reading through their
verifier; legacy sources remain legacy. Rewriting existing bytes is not a
trusted migration of their historical identity. An upload's persisted marker,
not the current node's write switch, determines its UploadPart/Complete mode.
Digest/index builders spill above a 1 MiB buffer limit, and request readers keep
a bounded stripe cache. For EC 12+4, a 5 GiB part with 1 MiB stripes has a
4,751,424-byte index on each disk (about 1.42% of logical data across 16 disks).
The maximum descriptor is 853,376 Base64 bytes per prefix, about 1.63 MiB for
both copies. These are format bounds, not measured throughput guarantees.
Legacy objects retain their existing GET and traditional Heal behavior and
therefore their residual complete-donor substitution risk. Ordinary shard repair
and the existing explicit-version metadata recovery path remain available, but
do not create commitments or certify object identity. Actual drive repairs are
reported separately from strong integrity receipts. Normal
presence scans do not issue strong integrity receipts even for protected
objects. Only a completed exclusive Deep scan/repair with authenticated sources
can do so. See the [upgrade contract](minio-file-format-compat.md#independent-integrity-upgrade-contract)
for mixed-version and migration constraints and the
[rollout runbook](../operations/shard-integrity-rollout.md) for activation and rollback.
## 6. On-disk format (`xl.meta`)
This section is the load-bearing compatibility contract for stored metadata. It is byte-compatible with MinIO's `xl.meta`; interop proof, the fixture corpus, and the out-of-scope list are owned by [minio-file-format-compat.md](minio-file-format-compat.md) — cite it, do not re-derive interop claims.
@@ -252,7 +319,7 @@ Version-aware heal ([set_disk/ops/heal.rs](../../crates/ecstore/src/set_disk/ops
- **INVARIANT — reconstructability.** Heal refuses when `meta_to_heal_count > parity_blocks` (relaxed only if a quorum etag exists) or when any part loses more than `parity_blocks` shards.
- **INVARIANT — geometry match.** `latest_meta.erasure.distribution.len()` must equal the online-disk, outdated-disk, and parts-metadata counts, else heal refuses ("backend disks manually modified"). A real object missing `data_dir` is `FileCorrupt`.
- **Data-safety guard (backlog#920).** If data shards survive on ≥ `data_blocks` disks, regenerate the missing `xl.meta` from a valid FileInfo and re-drive heal rather than dangling-delete; torn writes (< `data_blocks`) fall through to dangling-delete handling.
- **Data-safety guard (backlog#920).** Legacy explicit-version metadata recovery retains its validated geometry, consistent candidate identity, online-disk and available-data bounds. It does not establish independent payload identity. Protected recovery additionally requires a matching metadata quorum and authenticated data on ≥ `data_blocks` disks. Conflicting metadata and uncertain deletion conditions retain their existing refusal/grace behavior.
- Healed shards are written to the outdated disks, each recording `erasure.index = slot + 1`, then `rename_data` to final. Heal admission / scanner budget is owned by [placement-repair-invariants.md](placement-repair-invariants.md).
---
@@ -146,3 +146,55 @@ Not fixture-proven: transitioned `xl.meta`; CORS, public-access-block, and bucke
- Any change to `crates/filemeta` or `crates/ecstore/src/bucket` metadata encoding is a storage-format change and follows the migration and readiness contracts in [README.md](README.md) and the ecstore layout boundary rules.
- Do not bump a Version Anchor without a read path for the prior value; see [erasure-coding.md](erasure-coding.md) for the accept-older, reject-newer rule.
- `.github/workflows/ci.yml`, `.github/workflows/cache-warm.yml`, and `ARCHITECTURE.md` cite the [rio-v2 variant lifecycle](#rio-v2-variant-lifecycle) heading; keep it when editing this file.
## Independent integrity upgrade contract
The optional shard-integrity extension preserves the release checksum algorithm
and payload framing. It does not bump XL container/header/metadata versions.
`ObjectPartInfo` extensions use named MessagePack fields; older eight-field
readers can ignore the new field, and new readers still decode old positional
arrays. This is a decoder compatibility claim, not certification of arbitrary
mixed-version write, repair, tiering, or downgrade workflows.
| Workflow | Contract |
|---|---|
| New reader, old object | Existing reads and traditional shard/explicit-version metadata recovery remain available; independent donor-substitution protection is absent until a trusted rewrite. |
| Old reader, new object | Existing checksum/frame and metadata decoders remain usable; old readers do not enforce the independent proof. |
| New coordinator, old disk server | GET/Deep verification runs at the coordinator. A protected UploadPart requires a write quorum that publishes its index. An old RenameFile response cannot acknowledge durable index repair. |
| Old write/repair coordinator | Unsupported for protected data: it may omit, discard, or preserve stale integrity metadata. New protection is disabled by default so the coordinator rollout can precede activation. |
| Pre-upgrade in-progress multipart upload | Remains legacy when completed. Reinitiate and reupload from a trusted source to obtain protection. |
| New multipart upload completed by an old coordinator | Unsupported; a retained upload marker without a completed descriptor fails validation. |
| Write switch changed during an upload | The persisted upload mode wins: legacy uploads remain legacy and protected uploads must finish with all proofs. |
| Write switch disabled after activation | Existing protected reads, repair and rewrites still require protection. This does not make an old binary safe to reintroduce. |
| Automatic COPY, materialization or data movement | Preserve the source protection mode. Rewriting legacy bytes is not independent evidence of their original content. |
| Full downgrade after protected writes | No blanket guarantee; preserve a snapshot and validate read, write, repair, COPY, and multipart behavior before considering a downgrade. |
Upgrade all readers, writers and background coordinators before explicitly
enabling both `RUSTFS_SHARD_INTEGRITY_WRITE` and
`RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED`. Both default to false. The confirmation
is an operator attestation, not automatic capability discovery or an old-node
fence. Validate actual mixed-version writes and restart recovery before claiming
a rolling-upgrade or pre-activation rollback guarantee. See the
[rollout runbook](../operations/shard-integrity-rollout.md).
There is no automatic migration of existing payloads and no need
to rewrite them merely to preserve reading. To protect legacy objects, compare
against a separately trusted source or end-to-end digest and rewrite/reupload
through upgraded coordinators. Recomputing a hash from existing suspect shards,
or agreeing RS parity alone, does not establish their original identity.
Normal scans, tier metadata scans and unproven legacy scans cannot produce `VerifiedHealthy` or
`Repaired` integrity receipts. Legacy repair still runs and reports actual
before/after drive changes; an unknown strong result does not mean the repair
was never attempted. Authoritative historical-version absence/cleanup proofs
remain separate from live payload verification. Retain trusted backups for
legacy recovery. These commitments protect against misplaced or corrupted
shards under an authoritative metadata quorum; they are not signatures against
an attacker who can replace that quorum too.
The public MinIO source inspected at
[`7aac2a2`](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/bitrot-streaming.go)
uses adjacent streaming HighwayHash checksums. Its
[XL metadata codec](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/xl-storage-format-v2.go)
does not establish the independent part-generation commitment described here.
This source comparison does not claim a live MinIO donor-shard reproduction or
behavior of proprietary MinIO editions.
@@ -0,0 +1,92 @@
# Independent shard integrity rollout
Independent shard commitments detect internally valid shards that belong to a
different part generation. The feature preserves the existing checksum frames
and adds optional metadata and proof indexes. It does not authenticate legacy
data retroactively. The [format contract](../architecture/erasure-coding.md#51-independent-shard-commitments)
defines the stored representation.
## Configuration and object lifetime
| Variable | Default | Effect |
|---|---|---|
| `RUSTFS_SHARD_INTEGRITY_WRITE` | `false` | Requests protection for new writes without an inherited object/upload mode. |
| `RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED` | `false` | Operator confirmation that every reader, writer and background coordinator supports the extension. |
Both must be true before new writes opt in. Setting only the request switch
does not enable protection. These flags do not perform peer capability discovery
or stop an older binary from joining the cluster.
A PUT fixes its mode before encoding. An MPU fixes its mode at initiation and
persists it with the upload. Completing an old upload after activation does not
promote it. Disabling the switches during a protected upload does not permit
parts or completion to drop the proof requirement.
Reads always follow the object's metadata. Turning off creation does not disable
verification of existing protected objects. A corrupt, conflicting or incomplete
protection declaration is an error, not a legacy object. Metadata-only COPY
preserves the existing commitment. COPY and physical background rewrites inherit
the source mode; they do not turn an unverified legacy object into a historically
verified object simply by calculating a new hash.
## Upgrade and activation
1. Deploy a version that understands and maintains the extension, leaving both
switches false. Existing legacy reads, shard repair and explicit-version
metadata recovery continue under their existing bounds.
2. Validate old/new process combinations for PUT, multipart completion, COPY,
Heal and restart recovery in the actual deployment. Decoder fixture tests
alone do not certify these operations.
3. Finish upgrading every reader and write/repair/movement coordinator. Prevent
old services or rollback images from rejoining before enabling protection.
4. Qualify protected writes, range reads, repair, missing indexes and deployment
restart/failure behavior. Measure small-object throughput, range latency and
maximum multipart metadata overhead against the same release configuration.
5. Enable both switches consistently. Verify new objects with an exclusive Deep
scan before treating the enhanced mode as qualified for production traffic.
Existing rollout or performance gaps remain gates for production activation;
this runbook is not a claim that a distributed upgrade has been exercised.
## Legacy repair and interpretation
Legacy objects remain exposed to substitution by a complete, internally valid
donor shard. Traditional reconstruction can restore missing or detectably
damaged shards, but cannot prove that all surviving bytes are the original
content. It therefore does not create an independent digest or a positive strong
integrity receipt.
Use the heal item's before/after drives and actual repaired-drive count to
observe ordinary recovery. Strong outcome counters may remain unknown/skipped
because object identity is unproven; that is distinct from execution progress.
MRF likewise retains durable legacy repair obligations after physical recovery
when no independent verification receipt can discharge them. An idle attempt
queue does not imply that these persistent obligations have been cleared.
Partial-write MRF replay uses Deep verification so a protected object's repair
can discharge its obligation after verifying the payload. This adds full-object
read work to those background attempts, including healthy replay targets.
Normal presence scans likewise cannot certify protected payloads. Local Heal
of transitioned objects checks metadata without reading the tier payload, so it
does not issue a payload-integrity receipt even if a descriptor remains. An
authoritative historical-version cleanup or absence proof has its own identity
and commit checks and does not depend on a live payload digest.
To protect a legacy version as a trusted migration, validate against an
independently trusted original or suitable end-to-end digest, then reupload.
An ETag is not universally a plaintext MD5. Rehashing existing bytes or checking
RS parity alone does not establish the original identity. Keep this distinction
when measuring migration coverage; an ordinary server-side rewrite is not proof
of historical correctness.
## Rollback
Before any protected objects or uploads exist, a rollback still needs actual
old-version write/repair/restart qualification. Metadata decoding compatibility
is insufficient.
After activation, the supported rollback floor is a version that understands
and preserves the extension. Turning off creation leaves protected objects,
historical versions and in-progress protected uploads in place. It neither
converts them to legacy nor authorizes an arbitrary older reader/writer/repair
coordinator. Preserve recovery snapshots and use an explicitly validated
migration procedure if returning to an older format implementation is required.
+37
View File
@@ -4126,6 +4126,41 @@ mod tests {
}
}
#[tokio::test]
async fn durable_rename_requires_its_own_authenticated_body_and_success() {
let service = make_server();
let mut message = RenameFileRequest {
disk: "http://node-a:9000/data/rustfs0".to_owned(),
src_volume: ".rustfs.sys/tmp".to_owned(),
src_path: "integrity-stage/index".to_owned(),
dst_volume: "bucket".to_owned(),
dst_path: "object/index".to_owned(),
durable: false,
};
let old_body = rustfs_protos::canonical_rename_file_request_body(&message).expect("ordinary rename body");
message.durable = true;
let mut tampered = Request::new(message.clone());
set_tonic_canonical_body_digest(&mut tampered, &old_body).expect("digest");
mark_v2_authenticated(&mut tampered);
let error = service
.rename_file(tampered)
.await
.expect_err("durability flag must be authenticated");
assert_eq!(error.code(), tonic::Code::PermissionDenied);
let body = rustfs_protos::canonical_rename_file_request_body(&message).expect("durable rename body");
let mut request = Request::new(message);
set_tonic_canonical_body_digest(&mut request, &body).expect("digest");
mark_v2_authenticated(&mut request);
let response = service
.rename_file(request)
.await
.expect("valid digest passes the gate")
.into_inner();
assert!(!response.success, "unknown disk cannot apply the rename");
assert!(!response.durability_applied, "failed publication cannot acknowledge durability");
}
#[tokio::test]
async fn disk_mutation_body_digest_gate_runs_before_disk_lookup() {
let service = make_server();
@@ -4340,6 +4375,7 @@ mod tests {
assert_gated!(
rename_file,
RenameFileRequest {
durable: false,
disk: disk.clone(),
src_volume: "src".into(),
src_path: "sp".into(),
@@ -5512,6 +5548,7 @@ mod tests {
let service = create_test_node_service();
let request = Request::new(RenameFileRequest {
durable: false,
disk: "invalid-disk-path".to_string(),
src_volume: "src-volume".to_string(),
src_path: "src-path".to_string(),
+11 -4
View File
@@ -1379,22 +1379,29 @@ impl NodeService {
)?;
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
match disk
.rename_file(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path)
.await
{
let result = if request.durable {
disk.rename_file_durable(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path)
.await
} else {
disk.rename_file(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path)
.await
};
match result {
Ok(_) => Ok(Response::new(RenameFileResponse {
success: true,
durability_applied: request.durable,
error: None,
})),
Err(err) => Ok(Response::new(RenameFileResponse {
success: false,
durability_applied: false,
error: Some(err.into()),
})),
}
} else {
Ok(Response::new(RenameFileResponse {
success: false,
durability_applied: false,
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
+13
View File
@@ -1395,6 +1395,15 @@ pub(crate) trait StorageDiskRpcExt {
async fn read_file(&self, volume: &str, path: &str) -> DiskResult<FileReader>;
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult<FileReader>;
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()>;
async fn rename_file_durable(
&self,
_src_volume: &str,
_src_path: &str,
_dst_volume: &str,
_dst_path: &str,
) -> DiskResult<()> {
Err(DiskError::MethodNotAllowed)
}
async fn rename_part(
&self,
src_volume: &str,
@@ -1550,6 +1559,10 @@ where
ecstore_disk::DiskAPI::rename_file(self, src_volume, src_path, dst_volume, dst_path).await
}
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()> {
ecstore_disk::DiskAPI::rename_file_durable(self, src_volume, src_path, dst_volume, dst_path).await
}
async fn rename_part(
&self,
src_volume: &str,