mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-21 18:13:34 +00:00
fix(ecstore): gate shard integrity and preserve legacy recovery (#7741)
* fix(ecstore)!: bind bitrot shards to immutable part identities Verify part, coding-index, and block identity across write, GET, and Heal paths. Preserve identities across metadata-only copies and repair, include them in multipart quorum selection, and require payload proof for receipts. Keep legacy decoding with conservative parity and target-digest validation, and document its unsupported cases and additional verification I/O. BREAKING CHANGE: New bound-v1 shards require compatible readers throughout the fleet. Legacy objects without sufficient integrity evidence return an error; binary rollback after new writes requires verified data migration. Refs: rustfs/backlog#2497 * fix(ecstore): preserve shard framing with independent integrity Commit immutable part-generation Merkle roots and replicated proof indexes without changing existing checksum frames. Verify reads, reconstruction and Deep Heal against metadata quorum; keep legacy reads and explicitly defer unproven legacy data repair. Preserve multipart rollback generations, require acknowledged durable index publication, and add decoder compatibility and donor-shard regression coverage. * fix(heal): verify protected partial-write replay
This commit is contained in:
Generated
+2
@@ -9893,6 +9893,7 @@ name = "rustfs-filemeta"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"base64-simd",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
@@ -9907,6 +9908,7 @@ dependencies = [
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"tempfile",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
|
||||
@@ -116,6 +116,20 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
|
||||
/// Default: bitrot verification is enabled on GetObject reads (do not skip).
|
||||
pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
|
||||
|
||||
/// Create independent shard commitments for new writes after the fleet is upgraded.
|
||||
/// Existing protected objects and multipart uploads retain their protection.
|
||||
pub const ENV_SHARD_INTEGRITY_WRITE: &str = "RUSTFS_SHARD_INTEGRITY_WRITE";
|
||||
pub const DEFAULT_SHARD_INTEGRITY_WRITE: bool = false;
|
||||
|
||||
/// Operator confirmation that every reader, writer and background coordinator
|
||||
/// understands independent shard commitments. This is not capability discovery
|
||||
/// or a fence against an old binary rejoining the fleet.
|
||||
pub const ENV_SHARD_INTEGRITY_FLEET_CONFIRMED: &str = "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_WRITE);
|
||||
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED);
|
||||
|
||||
/// Request writing the complete remote-tier version state into object metadata.
|
||||
///
|
||||
/// This remains ineffective until
|
||||
|
||||
@@ -501,9 +501,9 @@ pub mod object {
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
|
||||
ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCompletion,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
|
||||
@@ -5190,6 +5190,7 @@ pub async fn put_restore_opts(
|
||||
// Restore writes stored (possibly encrypted) bytes, so the writer's
|
||||
// computed MD5 is not the object's public plaintext ETag.
|
||||
preserve_etag: oi.etag.clone(),
|
||||
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
|
||||
//expires: oi.expires,
|
||||
..Default::default()
|
||||
})
|
||||
|
||||
@@ -844,6 +844,44 @@ fn spawn_control_channel_prewarm(addr: String) {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
async fn rename_file_with_durability(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
durable: bool,
|
||||
) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(RenameFileRequest {
|
||||
durable,
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_file_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_file")?;
|
||||
|
||||
let response = client.rename_file(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
if durable && !response.durability_applied {
|
||||
return Err(DiskError::MethodNotAllowed);
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
@@ -3401,30 +3439,13 @@ impl DiskAPI for RemoteDisk {
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(RenameFileRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_file_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_file")?;
|
||||
self.rename_file_with_durability(src_volume, src_path, dst_volume, dst_path, false)
|
||||
.await
|
||||
}
|
||||
|
||||
let response = client.rename_file(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
self.rename_file_with_durability(src_volume, src_path, dst_volume, dst_path, true)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
|
||||
@@ -785,6 +785,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
|
||||
let put_opts = ObjectOptions {
|
||||
user_defined: dst_opts.user_defined.clone(),
|
||||
shard_integrity_write_mode: Some(src_info.shard_integrity_write_mode()),
|
||||
versioned: dst_opts.versioned,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
mod_time: dst_opts.mod_time,
|
||||
|
||||
@@ -296,6 +296,7 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz
|
||||
preserve_etag: object_info.etag.clone(),
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
shard_integrity_write_mode: Some(object_info.shard_integrity_write_mode()),
|
||||
..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
|
||||
}
|
||||
}
|
||||
@@ -476,6 +477,7 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize)
|
||||
versioned: object_info.version_id.is_some(),
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
shard_integrity_write_mode: Some(object_info.shard_integrity_write_mode()),
|
||||
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
|
||||
http_preconditions: Some(data_movement_target_precondition()),
|
||||
mod_time: object_info.mod_time,
|
||||
@@ -2109,6 +2111,27 @@ async fn migrate_object_inner(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn data_movement_retains_source_shard_integrity_mode() {
|
||||
use crate::object_api::ShardIntegrityWriteMode;
|
||||
let mut source = ObjectInfo::default();
|
||||
for mode in [ShardIntegrityWriteMode::Legacy, ShardIntegrityWriteMode::Protected] {
|
||||
if mode == ShardIntegrityWriteMode::Protected {
|
||||
// A declaration selects protected I/O; the source reader still
|
||||
// rejects this deliberately incomplete descriptor before commit.
|
||||
rustfs_utils::http::insert_str(
|
||||
Arc::make_mut(&mut source.user_defined),
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY,
|
||||
"invalid".to_owned(),
|
||||
);
|
||||
}
|
||||
let put = data_movement_put_object_opts(&source, 0);
|
||||
let multipart = data_movement_new_multipart_opts(&source, 0);
|
||||
assert_eq!(put.shard_integrity_write_mode, Some(mode));
|
||||
assert_eq!(multipart.shard_integrity_write_mode, Some(mode));
|
||||
}
|
||||
}
|
||||
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
|
||||
use rustfs_rio::{Checksum, ChecksumType};
|
||||
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
|
||||
|
||||
@@ -2396,6 +2396,20 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"rename_file",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
self.disk
|
||||
.rename_file_durable(src_volume, src_path, dst_volume, dst_path)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
|
||||
@@ -9354,6 +9354,41 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
let durability = effective_durability(dst_volume);
|
||||
let part = ObjectPartInfo::unmarshal(&meta)?;
|
||||
if let Some(integrity) = part.integrity {
|
||||
integrity.validate()?;
|
||||
if usize::try_from(integrity.number).map_err(|_| DiskError::FileCorrupt)? != part.number
|
||||
|| usize::try_from(integrity.size).map_err(|_| DiskError::FileCorrupt)? != part.size
|
||||
|| dst_file_path.file_name().and_then(|name| name.to_str()) != Some(format!("part.{}", part.number).as_str())
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let proof_name = integrity.file_name();
|
||||
let source = src_file_path.parent().ok_or(DiskError::FileCorrupt)?.join(&proof_name);
|
||||
let destination = dst_file_path.parent().ok_or(DiskError::FileCorrupt)?.join(&proof_name);
|
||||
check_path_length(source.to_string_lossy().as_ref())?;
|
||||
check_path_length(destination.to_string_lossy().as_ref())?;
|
||||
let stat = fs::symlink_metadata(&source).await.map_err(to_file_error)?;
|
||||
if !stat.is_file() || stat.len() != u64::try_from(integrity.index_size()?).map_err(|_| DiskError::FileCorrupt)? {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
if durability.syncs_data_shards() {
|
||||
let source = source.clone();
|
||||
tokio::task::spawn_blocking(move || os::sync_file(&source))
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
// Publish the immutable generation before preparing the part switch.
|
||||
// Rollback retains the old generation; an unreferenced new index is
|
||||
// reclaimed with the upload directory if preparation is interrupted.
|
||||
rename_all(&source, &destination, &dst_volume_dir, &self.publication_root).await?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir(destination.parent().ok_or(DiskError::FileCorrupt)?)
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let source = std::fs::symlink_metadata(&src_file_path).map_err(to_file_error)?;
|
||||
if !source.is_file() {
|
||||
@@ -9462,6 +9497,41 @@ impl DiskAPI for LocalDisk {
|
||||
let Some(parent) = transaction_path.parent() else {
|
||||
return Err(DiskError::InvalidPath);
|
||||
};
|
||||
// Delete only the generation made obsolete by this settled switch.
|
||||
// Check the published metadata before removing either proof, so a
|
||||
// repeated or interrupted settlement cannot remove a live index.
|
||||
let (retained_name, obsolete_name) = match action {
|
||||
PartTransactionAction::Commit => (PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META),
|
||||
PartTransactionAction::Rollback => (PART_TRANSACTION_OLD_META, PART_TRANSACTION_NEW_META),
|
||||
};
|
||||
if let (Ok(current), Ok(retained), Ok(obsolete)) = (
|
||||
std::fs::read(¤t_meta_path),
|
||||
std::fs::read(transaction_path.join(retained_name)),
|
||||
std::fs::read(transaction_path.join(obsolete_name)),
|
||||
) && current == retained
|
||||
&& let Ok(obsolete) = ObjectPartInfo::unmarshal(&obsolete)
|
||||
&& let Some(integrity) = obsolete.integrity
|
||||
&& integrity.validate().is_ok()
|
||||
&& usize::try_from(integrity.number).ok() == Some(obsolete.number)
|
||||
&& current_data_path.file_name().and_then(|name| name.to_str())
|
||||
== Some(format!("part.{}", obsolete.number).as_str())
|
||||
&& ObjectPartInfo::unmarshal(&retained)
|
||||
.ok()
|
||||
.and_then(|part| part.integrity)
|
||||
.as_ref()
|
||||
!= Some(&integrity)
|
||||
{
|
||||
let obsolete_path = parent.join(integrity.file_name());
|
||||
match std::fs::remove_file(&obsolete_path) {
|
||||
Ok(()) => {
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir_std(parent).map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(to_file_error(error).into()),
|
||||
}
|
||||
}
|
||||
let cleanup_path = parent.join(format!(".part-txn-settled-{}", Uuid::new_v4()));
|
||||
std::fs::rename(&transaction_path, &cleanup_path).map_err(to_file_error)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
@@ -9735,6 +9805,31 @@ impl DiskAPI for LocalDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
let source = self.io_get_object_path(src_volume, src_path)?;
|
||||
let destination = self.io_get_object_path(dst_volume, dst_path)?;
|
||||
check_path_length(source.to_string_lossy().as_ref())?;
|
||||
check_path_length(destination.to_string_lossy().as_ref())?;
|
||||
let durability = effective_durability(dst_volume);
|
||||
let stat = fs::symlink_metadata(&source).await.map_err(to_file_error)?;
|
||||
if !stat.is_file() {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
if durability.syncs_data_shards() {
|
||||
tokio::task::spawn_blocking(move || os::sync_file(&source))
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
self.rename_file(src_volume, src_path, dst_volume, dst_path).await?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir(destination.parent().ok_or(DiskError::InvalidPath)?)
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
// async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result<File> {
|
||||
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
|
||||
@@ -12088,7 +12183,15 @@ mod test {
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
|
||||
let payload = Bytes::from_static(b"part payload");
|
||||
let meta = Bytes::from_static(b"part metadata");
|
||||
let meta = Bytes::from(
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: payload.len(),
|
||||
..Default::default()
|
||||
}
|
||||
.marshal_msg()
|
||||
.expect("legacy part metadata"),
|
||||
);
|
||||
disk.write_all(tmp_volume, "upload/part.1", payload.clone())
|
||||
.await
|
||||
.expect("source part should be written");
|
||||
@@ -12178,7 +12281,15 @@ mod test {
|
||||
"regression path must cross the traditional Windows MAX_PATH boundary: {deepest_marker:?}"
|
||||
);
|
||||
let payload = Bytes::from_static(b"part payload");
|
||||
let meta = Bytes::from_static(b"part metadata");
|
||||
let meta = Bytes::from(
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: payload.len(),
|
||||
..Default::default()
|
||||
}
|
||||
.marshal_msg()
|
||||
.expect("legacy part metadata"),
|
||||
);
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, payload.clone())
|
||||
.await
|
||||
.expect("source part should be written");
|
||||
@@ -12207,7 +12318,15 @@ mod test {
|
||||
);
|
||||
|
||||
let replacement_payload = Bytes::from_static(b"replacement part payload");
|
||||
let replacement_meta = Bytes::from_static(b"replacement part metadata");
|
||||
let replacement_meta = Bytes::from(
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: replacement_payload.len(),
|
||||
..Default::default()
|
||||
}
|
||||
.marshal_msg()
|
||||
.expect("legacy replacement metadata"),
|
||||
);
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, replacement_payload.clone())
|
||||
.await
|
||||
.expect("replacement source part should be written");
|
||||
@@ -12272,9 +12391,23 @@ mod test {
|
||||
.await
|
||||
.expect("old part metadata should be staged");
|
||||
|
||||
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", Bytes::from_static(b"new metadata"))
|
||||
.await
|
||||
.expect("part transaction should be prepared");
|
||||
disk.prepare_part_transaction(
|
||||
"tmp",
|
||||
"upload/part.1",
|
||||
"bucket",
|
||||
"object/part.1",
|
||||
Bytes::from(
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 8,
|
||||
..Default::default()
|
||||
}
|
||||
.marshal_msg()
|
||||
.expect("legacy part metadata"),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("part transaction should be prepared");
|
||||
disk.rename_file("tmp", "upload/part.1", "bucket", "object/part.1")
|
||||
.await
|
||||
.expect("data publication should succeed");
|
||||
@@ -12296,6 +12429,139 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shard_integrity_part_transaction_keeps_only_the_settled_generation() {
|
||||
use crate::io_support::shard_integrity::IntegrityBuilder;
|
||||
use rustfs_filemeta::shard_integrity::IntegrityLayout;
|
||||
for action in [PartTransactionAction::Commit, PartTransactionAction::Rollback] {
|
||||
let dir = tempfile::tempdir().expect("fixture");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("path")).expect("endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("disk");
|
||||
ensure_test_volume(&disk, "tmp").await;
|
||||
ensure_test_volume(&disk, "bucket").await;
|
||||
let mut parts = Vec::new();
|
||||
for (volume, directory, byte) in [("bucket", "object", b'a'), ("tmp", "upload", b'b')] {
|
||||
let mut builder =
|
||||
IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 1).expect("builder");
|
||||
let shard = [byte; 4];
|
||||
builder.push([shard.as_slice(); 4].into_iter()).await.expect("stripe");
|
||||
let prepared = builder.finish(8).await.expect("proof");
|
||||
disk.write_all(
|
||||
volume,
|
||||
&format!("{directory}/{}", prepared.part.file_name()),
|
||||
prepared.inline_bytes().expect("index"),
|
||||
)
|
||||
.await
|
||||
.expect("index file");
|
||||
disk.write_all(volume, &format!("{directory}/part.1"), Bytes::copy_from_slice(&shard))
|
||||
.await
|
||||
.expect("data");
|
||||
let part = ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 8,
|
||||
integrity: Some(prepared.part),
|
||||
..Default::default()
|
||||
};
|
||||
let meta = Bytes::from(part.marshal_msg().expect("metadata"));
|
||||
disk.write_all(volume, &format!("{directory}/part.1.meta"), meta.clone())
|
||||
.await
|
||||
.expect("part meta");
|
||||
parts.push((part, meta));
|
||||
}
|
||||
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", parts[1].1.clone())
|
||||
.await
|
||||
.expect("prepare");
|
||||
for (part, _) in &parts {
|
||||
assert!(
|
||||
disk.read_all("bucket", &format!("object/{}", part.integrity.as_ref().expect("proof").file_name()))
|
||||
.await
|
||||
.is_ok(),
|
||||
"both generations survive preparation"
|
||||
);
|
||||
}
|
||||
disk.rename_part("tmp", "upload/part.1", "bucket", "object/part.1", parts[1].1.clone())
|
||||
.await
|
||||
.expect("publish");
|
||||
disk.settle_part_transaction("bucket", "object/part.1", action)
|
||||
.await
|
||||
.expect("settle");
|
||||
let retained = usize::from(action == PartTransactionAction::Commit);
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", "object/part.1.meta").await.expect("settled metadata"),
|
||||
parts[retained].1
|
||||
);
|
||||
for (index, (part, _)) in parts.iter().enumerate() {
|
||||
let exists = disk
|
||||
.read_all("bucket", &format!("object/{}", part.integrity.as_ref().expect("proof").file_name()))
|
||||
.await
|
||||
.is_ok();
|
||||
assert_eq!(exists, index == retained, "obsolete proof is reclaimed after settlement");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shard_integrity_settlement_never_deletes_another_parts_index() {
|
||||
use crate::io_support::shard_integrity::IntegrityBuilder;
|
||||
use rustfs_filemeta::shard_integrity::IntegrityLayout;
|
||||
let dir = tempfile::tempdir().expect("fixture");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("path")).expect("endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("disk");
|
||||
ensure_test_volume(&disk, "tmp").await;
|
||||
ensure_test_volume(&disk, "bucket").await;
|
||||
let mut builder = IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 2).expect("builder");
|
||||
builder.push([b"data".as_slice(); 4].into_iter()).await.expect("stripe");
|
||||
let prepared = builder.finish(8).await.expect("index");
|
||||
let other_index = format!("object/{}", prepared.part.file_name());
|
||||
let index_bytes = prepared.inline_bytes().expect("index bytes");
|
||||
disk.write_all("bucket", &other_index, index_bytes.clone())
|
||||
.await
|
||||
.expect("part 2 index");
|
||||
let corrupt = ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 8,
|
||||
integrity: Some(prepared.part),
|
||||
..Default::default()
|
||||
};
|
||||
disk.write_all("bucket", "object/part.1", Bytes::from_static(b"old data"))
|
||||
.await
|
||||
.expect("old data");
|
||||
disk.write_all(
|
||||
"bucket",
|
||||
"object/part.1.meta",
|
||||
Bytes::from(corrupt.marshal_msg().expect("misdirected metadata")),
|
||||
)
|
||||
.await
|
||||
.expect("corrupt old metadata");
|
||||
disk.write_all("tmp", "upload/part.1", Bytes::from_static(b"new data"))
|
||||
.await
|
||||
.expect("new data");
|
||||
let replacement = Bytes::from(
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 8,
|
||||
..Default::default()
|
||||
}
|
||||
.marshal_msg()
|
||||
.expect("replacement"),
|
||||
);
|
||||
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", replacement.clone())
|
||||
.await
|
||||
.expect("prepare");
|
||||
disk.rename_part("tmp", "upload/part.1", "bucket", "object/part.1", replacement)
|
||||
.await
|
||||
.expect("publish");
|
||||
disk.settle_part_transaction("bucket", "object/part.1", PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle");
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", &other_index)
|
||||
.await
|
||||
.expect("unrelated part index retained"),
|
||||
index_bytes
|
||||
);
|
||||
}
|
||||
|
||||
struct BlockingScanWriter {
|
||||
entered_tx: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
@@ -304,7 +304,13 @@ impl LocalDisk {
|
||||
let data_dir = fi.data_dir.unwrap_or_default();
|
||||
fi.parts
|
||||
.iter()
|
||||
.map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number))
|
||||
.flat_map(|part| {
|
||||
std::iter::once(format!("{dst_path}/{data_dir}/part.{}", part.number)).chain(
|
||||
part.integrity
|
||||
.as_ref()
|
||||
.map(|integrity| format!("{dst_path}/{data_dir}/{}", integrity.file_name())),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
|
||||
|
||||
@@ -577,6 +577,13 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(disk) => disk.rename_file_durable(src_volume, src_path, dst_volume, dst_path).await,
|
||||
Disk::Remote(disk) => disk.rename_file_durable(src_volume, src_path, dst_volume, dst_path).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
|
||||
match self {
|
||||
@@ -1175,6 +1182,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<FileWriter>;
|
||||
// ReadFileStream
|
||||
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
|
||||
async fn rename_file_durable(&self, _src_volume: &str, _src_path: &str, _dst_volume: &str, _dst_path: &str) -> Result<()> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()>;
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
|
||||
@@ -102,6 +102,7 @@ pin_project! {
|
||||
chunks: Vec<bytes::Bytes>,
|
||||
skip_verify: bool,
|
||||
last_verify_duration: Duration,
|
||||
integrity: Option<crate::io_support::shard_integrity::ShardVerifier>,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,6 +120,7 @@ where
|
||||
chunks: Vec::new(),
|
||||
skip_verify,
|
||||
last_verify_duration: Duration::ZERO,
|
||||
integrity: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,6 +128,24 @@ where
|
||||
self.last_verify_duration
|
||||
}
|
||||
|
||||
pub(crate) fn integrity_proof(&self) -> Option<std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>> {
|
||||
self.integrity
|
||||
.as_ref()
|
||||
.map(crate::io_support::shard_integrity::ShardVerifier::proof)
|
||||
}
|
||||
|
||||
pub(crate) fn set_integrity(&mut self, verifier: crate::io_support::shard_integrity::ShardVerifier) -> std::io::Result<()> {
|
||||
if self.hash_algo != HashAlgorithm::HighwayHash256S {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"unsupported protected shard framing",
|
||||
));
|
||||
}
|
||||
self.integrity = Some(verifier);
|
||||
self.skip_verify = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn inner_ref(&self) -> &R {
|
||||
&self.inner
|
||||
@@ -152,6 +172,9 @@ where
|
||||
let need = self.hash_algo.size() + want;
|
||||
self.read_scratch_block(need, want).await?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
|
||||
if let Some(integrity) = &mut self.integrity {
|
||||
integrity.verify(data).await?;
|
||||
}
|
||||
out.copy_from_slice(data);
|
||||
self.last_verify_duration = verify;
|
||||
Ok(want)
|
||||
@@ -263,6 +286,18 @@ where
|
||||
///
|
||||
/// On return `out.len()` has grown by exactly the returned count.
|
||||
pub async fn read_appending(&mut self, out: &mut Vec<u8>, want: usize) -> std::io::Result<usize> {
|
||||
let start = out.len();
|
||||
let count = self.read_appending_frame(out, want).await?;
|
||||
if let Some(integrity) = &mut self.integrity
|
||||
&& let Err(error) = integrity.verify(&out[start..]).await
|
||||
{
|
||||
out.truncate(start);
|
||||
return Err(error);
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
async fn read_appending_frame(&mut self, out: &mut Vec<u8>, want: usize) -> std::io::Result<usize> {
|
||||
use bytes::BufMut as _;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
@@ -768,6 +803,7 @@ impl AsyncWrite for CustomWriter {
|
||||
pub struct BitrotWriterWrapper {
|
||||
bitrot_writer: BitrotWriter<CustomWriter>,
|
||||
writer_type: WriterType,
|
||||
integrity: Option<crate::io_support::shard_integrity::ShardVerifier>,
|
||||
}
|
||||
|
||||
/// Enum to track the type of writer we're using
|
||||
@@ -801,14 +837,22 @@ impl BitrotWriterWrapper {
|
||||
Self {
|
||||
bitrot_writer: BitrotWriter::new(writer, shard_size, checksum_algo),
|
||||
writer_type,
|
||||
integrity: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Write data to the bitrot writer
|
||||
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
if let Some(integrity) = &mut self.integrity {
|
||||
integrity.verify(buf).await?;
|
||||
}
|
||||
self.bitrot_writer.write(buf).await
|
||||
}
|
||||
|
||||
pub(crate) fn set_integrity(&mut self, verifier: crate::io_support::shard_integrity::ShardVerifier) {
|
||||
self.integrity = Some(verifier);
|
||||
}
|
||||
|
||||
pub async fn shutdown(&mut self) -> std::io::Result<()> {
|
||||
self.bitrot_writer.shutdown().await
|
||||
}
|
||||
|
||||
@@ -591,6 +591,7 @@ pub(crate) struct ParallelReader<R> {
|
||||
// never consumes the unopened reader reserved for a later stripe.
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
stripe_index: usize,
|
||||
integrity: Option<std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -782,6 +783,7 @@ where
|
||||
.map(|index| !demand_bound_lockstep || index < e.data_shards)
|
||||
.collect();
|
||||
ParallelReader {
|
||||
integrity: readers.iter().flatten().find_map(BitrotReader::integrity_proof),
|
||||
readers,
|
||||
offset,
|
||||
shard_size,
|
||||
@@ -1998,11 +2000,28 @@ where
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
async fn read_next_stripe(&mut self) -> Box<StripeReadState> {
|
||||
let stripe = self.offset.checked_div(self.shard_size);
|
||||
let mut state = self
|
||||
.stripe_state
|
||||
.take()
|
||||
.unwrap_or_else(|| Box::new(StripeReadState::with_slot_count(self.readers.len(), self.data_shards)));
|
||||
self.read_into_state(&mut state).await;
|
||||
if state.can_decode()
|
||||
&& let Some(proof) = &self.integrity
|
||||
{
|
||||
let result = match stripe {
|
||||
Some(stripe) => proof.reconstruction_proof(stripe, state.shards_mut()).await,
|
||||
None => Err(io::Error::new(ErrorKind::InvalidData, "invalid integrity stripe offset")),
|
||||
};
|
||||
match result {
|
||||
Ok(proof) => state.integrity = proof,
|
||||
Err(_) => {
|
||||
let (shards, errors) = state.parts_mut();
|
||||
shards.iter_mut().for_each(|shard| *shard = None);
|
||||
errors.fill(Some(Error::FileCorrupt));
|
||||
}
|
||||
}
|
||||
}
|
||||
state
|
||||
}
|
||||
|
||||
@@ -2336,6 +2355,8 @@ impl Erasure {
|
||||
ret_err: &mut Option<std::io::Error>,
|
||||
stage_metrics_enabled: bool,
|
||||
require_surplus_source: bool,
|
||||
integrity: Option<&std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
|
||||
request_offset: usize,
|
||||
) -> StripeFlow
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
@@ -2373,6 +2394,20 @@ impl Erasure {
|
||||
// missing data shard and an extra source shard was available, verify
|
||||
// the reconstructed data against that source before streaming bytes.
|
||||
let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let proof = if let Some(integrity) = integrity {
|
||||
match integrity
|
||||
.reconstruction_proof((request_offset + *written) / self.block_size, shards)
|
||||
.await
|
||||
{
|
||||
Ok(proof) => proof,
|
||||
Err(error) => {
|
||||
*ret_err = Some(error);
|
||||
return StripeFlow::Stop;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let decode_result = if require_surplus_source {
|
||||
self.decode_data_with_reconstruction_verification_for_lockstep(shards)
|
||||
} else {
|
||||
@@ -2397,6 +2432,12 @@ impl Erasure {
|
||||
}
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
|
||||
|
||||
if let Some(proof) = proof
|
||||
&& let Err(error) = proof.verify(shards)
|
||||
{
|
||||
*ret_err = Some(error);
|
||||
return StripeFlow::Stop;
|
||||
}
|
||||
let emit_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let n = match write_data_blocks(writer, shards, self.data_shards, block_offset, block_length).await {
|
||||
Ok(n) => {
|
||||
@@ -2510,6 +2551,8 @@ impl Erasure {
|
||||
.with_deferred_parity_handles(deferred_handles)
|
||||
.with_deferred_parity_reopeners(deferred_reopeners);
|
||||
|
||||
let integrity = reader.integrity.clone();
|
||||
|
||||
let start = offset / self.block_size;
|
||||
let end = end_offset.saturating_sub(1) / self.block_size;
|
||||
|
||||
@@ -2615,6 +2658,8 @@ impl Erasure {
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
require_surplus_source,
|
||||
integrity.as_ref(),
|
||||
offset,
|
||||
);
|
||||
tokio::pin!(read_fut);
|
||||
tokio::pin!(emit_fut);
|
||||
@@ -2663,6 +2708,8 @@ impl Erasure {
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
reader.demand_bound_lockstep,
|
||||
integrity.as_ref(),
|
||||
offset,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2704,6 +2751,8 @@ impl Erasure {
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
reader.demand_bound_lockstep,
|
||||
integrity.as_ref(),
|
||||
offset,
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -816,6 +816,7 @@ where
|
||||
};
|
||||
rustfs_io_metrics::record_get_object_reconstruct_outcome(metrics_path, engine.engine_name(), reconstruct_outcome);
|
||||
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
|
||||
state.verify_reconstructed_integrity()?;
|
||||
|
||||
if state.shards_mut().len() < engine.data_shards() {
|
||||
return Err(io::Error::new(
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::disk::error_reduce::{
|
||||
use crate::erasure::coding::BitrotWriterWrapper;
|
||||
use crate::erasure::coding::Erasure;
|
||||
use crate::erasure::coding::erasure::EncodedBlock;
|
||||
use crate::io_support::shard_integrity::{IntegrityBuilder, PreparedIntegrity};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures::StreamExt;
|
||||
@@ -43,6 +44,13 @@ const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 32;
|
||||
const DEFAULT_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: usize = 4;
|
||||
const DEFAULT_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST: bool = false;
|
||||
|
||||
pub(crate) enum IntegrityEncodeMode {
|
||||
Inline(usize),
|
||||
SingleBlock(usize),
|
||||
Streaming,
|
||||
Batched,
|
||||
}
|
||||
|
||||
/// Cached value of `RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES` env var.
|
||||
/// Read once at first use via `OnceLock` to avoid per-encode syscall.
|
||||
static CACHED_MAX_INFLIGHT_BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
|
||||
@@ -297,6 +305,7 @@ impl Default for WriteProgressPolicy {
|
||||
|
||||
pub(crate) struct MultiWriter<'a> {
|
||||
writers: &'a mut [Option<BitrotWriterWrapper>],
|
||||
integrity: Option<&'a mut IntegrityBuilder>,
|
||||
write_quorum: usize,
|
||||
errs: Vec<Option<Error>>,
|
||||
policy: WriteProgressPolicy,
|
||||
@@ -314,6 +323,7 @@ impl<'a> MultiWriter<'a> {
|
||||
let length = writers.len();
|
||||
MultiWriter {
|
||||
writers,
|
||||
integrity: None,
|
||||
write_quorum,
|
||||
errs: vec![None; length],
|
||||
policy,
|
||||
@@ -375,6 +385,10 @@ impl<'a> MultiWriter<'a> {
|
||||
}
|
||||
|
||||
async fn write_block(&mut self, block: &EncodedBlock) -> std::io::Result<()> {
|
||||
if let Some(integrity) = &mut self.integrity {
|
||||
// Commit every coding index, including disks absent from the write.
|
||||
integrity.push(block.shards()).await?;
|
||||
}
|
||||
self.write_shards(block.shards()).await
|
||||
}
|
||||
|
||||
@@ -508,6 +522,63 @@ impl<'a> MultiWriter<'a> {
|
||||
}
|
||||
|
||||
impl Erasure {
|
||||
pub(crate) async fn encode_with_shard_integrity<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
part_number: usize,
|
||||
mode: IntegrityEncodeMode,
|
||||
protected: bool,
|
||||
) -> std::io::Result<(R, usize, Option<Vec<Bytes>>, Option<PreparedIntegrity>)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
let mut integrity = if protected {
|
||||
let layout = rustfs_filemeta::shard_integrity::IntegrityLayout::new(
|
||||
self.data_shards,
|
||||
self.parity_shards,
|
||||
self.block_size,
|
||||
self.uses_legacy_codec(),
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
Some(IntegrityBuilder::new(layout, part_number)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (reader, size, inline) = match mode {
|
||||
IntegrityEncodeMode::Inline(hint) => {
|
||||
let (reader, size, shards) = self
|
||||
.encode_inline_shards_with_integrity(reader, hint, integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, Some(shards))
|
||||
}
|
||||
IntegrityEncodeMode::SingleBlock(hint) => {
|
||||
let (reader, size) = self
|
||||
.encode_small_direct(reader, writers, quorum, true, hint, integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, None)
|
||||
}
|
||||
IntegrityEncodeMode::Streaming => {
|
||||
let (reader, size) = self
|
||||
.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest(), integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, None)
|
||||
}
|
||||
IntegrityEncodeMode::Batched => {
|
||||
let (reader, size) = self
|
||||
.encode_batched_with_integrity(reader, writers, quorum, integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, None)
|
||||
}
|
||||
};
|
||||
let integrity = match integrity {
|
||||
Some(integrity) => Some(integrity.finish(size).await?),
|
||||
None => None,
|
||||
};
|
||||
Ok((reader, size, inline, integrity))
|
||||
}
|
||||
|
||||
async fn encode_block(self: Arc<Self>, encode_buf: Vec<u8>, len: usize) -> std::io::Result<(EncodedBlock, Vec<u8>)> {
|
||||
let encode_stage_start = stage_timer_if_enabled();
|
||||
let encode_once = move || {
|
||||
@@ -564,6 +635,7 @@ impl Erasure {
|
||||
quorum: usize,
|
||||
require_single_block: bool,
|
||||
size_hint: usize,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
@@ -596,6 +668,7 @@ impl Erasure {
|
||||
|
||||
let block = self.encode_data_owned_block(buf)?;
|
||||
let mut mw = MultiWriter::new(writers, quorum);
|
||||
mw.integrity = integrity;
|
||||
mw.write_block(&block).await?;
|
||||
mw.shutdown().await?;
|
||||
Ok((reader, total))
|
||||
@@ -605,10 +678,23 @@ impl Erasure {
|
||||
/// The returned bytes are the same `[hash][shard]` representation produced
|
||||
/// by `BitrotWriter`, ready to be embedded in each disk's staged `xl.meta`.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn encode_inline_shards_with_size_hint<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
size_hint: usize,
|
||||
) -> std::io::Result<(R, usize, Vec<Bytes>)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_inline_shards_with_integrity(reader, size_hint, None).await
|
||||
}
|
||||
|
||||
async fn encode_inline_shards_with_integrity<R>(
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
size_hint: usize,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
) -> std::io::Result<(R, usize, Vec<Bytes>)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
@@ -622,6 +708,9 @@ impl Erasure {
|
||||
}
|
||||
|
||||
let block = self.encode_data_owned_block(buf)?;
|
||||
if let Some(integrity) = integrity {
|
||||
integrity.push(block.shards()).await?;
|
||||
}
|
||||
let mut inline_shards = Vec::with_capacity(block.shards().len());
|
||||
for shard in block.shards() {
|
||||
let hash = HashAlgorithm::HighwayHash256S.hash_encode(shard);
|
||||
@@ -645,7 +734,7 @@ impl Erasure {
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
let use_bytesmut_ingest = use_bytesmut_ingest();
|
||||
self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest)
|
||||
self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -659,6 +748,7 @@ impl Erasure {
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
use_bytesmut_ingest: bool,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
@@ -766,6 +856,7 @@ impl Erasure {
|
||||
}));
|
||||
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
writers.integrity = integrity;
|
||||
|
||||
let mut write_err = None;
|
||||
|
||||
@@ -808,10 +899,23 @@ impl Erasure {
|
||||
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn encode_batched<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
self.encode_batched_with_integrity(reader, writers, quorum, None).await
|
||||
}
|
||||
|
||||
async fn encode_batched_with_integrity<R>(
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
@@ -892,6 +996,7 @@ impl Erasure {
|
||||
}));
|
||||
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
writers.integrity = integrity;
|
||||
let mut write_err = None;
|
||||
|
||||
loop {
|
||||
@@ -946,7 +1051,8 @@ impl Erasure {
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Size-aware inline fast path. `size_hint` only controls the bounded initial
|
||||
@@ -962,7 +1068,8 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Fast path for single-block non-inline objects: avoids the producer/consumer
|
||||
@@ -978,7 +1085,7 @@ impl Erasure {
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
|
||||
}
|
||||
|
||||
/// Size-aware single-block fast path. `size_hint` only controls the bounded
|
||||
@@ -994,7 +1101,7 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1381,10 +1488,10 @@ mod tests {
|
||||
|
||||
let encode = match pipeline {
|
||||
EncodePipeline::Vec => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await })
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await })
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await })
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await })
|
||||
}
|
||||
EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
|
||||
};
|
||||
@@ -1443,8 +1550,8 @@ mod tests {
|
||||
))];
|
||||
|
||||
let result = match pipeline {
|
||||
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await,
|
||||
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await,
|
||||
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
|
||||
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
|
||||
};
|
||||
|
||||
@@ -1525,7 +1632,11 @@ mod tests {
|
||||
BLOCK_SIZE,
|
||||
))];
|
||||
let erasure_for_task = erasure.clone();
|
||||
let encode = tokio::spawn(async move { erasure_for_task.encode_with_ingest_mode(reader, &mut writers, 1, false).await });
|
||||
let encode = tokio::spawn(async move {
|
||||
erasure_for_task
|
||||
.encode_with_ingest_mode(reader, &mut writers, 1, false, None)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), writer_entered)
|
||||
.await
|
||||
@@ -2081,7 +2192,7 @@ mod tests {
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
|
||||
let (_reader, written) = erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true)
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
|
||||
.await
|
||||
.expect("BytesMut ingest path should encode the streaming payload");
|
||||
|
||||
@@ -2110,7 +2221,7 @@ mod tests {
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(vec![0x5a; block_size * 2]));
|
||||
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await;
|
||||
let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await;
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
|
||||
let (_reader, written) = result.expect("bytesmut streaming encode should complete");
|
||||
@@ -2728,10 +2839,14 @@ mod tests {
|
||||
let (_reader, total) = match pipeline {
|
||||
EncodePipeline::Vec => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false)
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true).await,
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, DATA_SHARDS).await,
|
||||
}
|
||||
.expect("encode should succeed");
|
||||
|
||||
@@ -657,6 +657,10 @@ pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize {
|
||||
}
|
||||
|
||||
impl Erasure {
|
||||
pub(crate) fn uses_legacy_codec(&self) -> bool {
|
||||
self.uses_legacy
|
||||
}
|
||||
|
||||
/// Create a new Erasure instance
|
||||
///
|
||||
/// # Arguments
|
||||
|
||||
@@ -252,6 +252,7 @@ impl DeferredObjectReader {
|
||||
DeferredReaderStripeHandle {
|
||||
state: Arc::clone(&self.state),
|
||||
stripe_stride,
|
||||
advanced: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -274,9 +275,14 @@ impl DeferredObjectReader {
|
||||
pub(crate) struct DeferredReaderStripeHandle {
|
||||
state: Arc<Mutex<DeferredObjectReaderState>>,
|
||||
stripe_stride: usize,
|
||||
advanced: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
impl DeferredReaderStripeHandle {
|
||||
pub(crate) fn integrity_position(&self) -> Arc<std::sync::atomic::AtomicUsize> {
|
||||
Arc::clone(&self.advanced)
|
||||
}
|
||||
|
||||
/// Advance the pending source by `stripes` full stripes.
|
||||
///
|
||||
/// Returns `false` when the reader has already been opened (or failed):
|
||||
@@ -297,8 +303,12 @@ impl DeferredReaderStripeHandle {
|
||||
let Some(offset) = source.offset.checked_add(delta) else {
|
||||
return false;
|
||||
};
|
||||
let Some(advanced) = self.advanced.load(std::sync::atomic::Ordering::Acquire).checked_add(stripes) else {
|
||||
return false;
|
||||
};
|
||||
source.offset = offset;
|
||||
source.length = source.length.saturating_sub(delta);
|
||||
self.advanced.store(advanced, std::sync::atomic::Ordering::Release);
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
|
||||
@@ -17,3 +17,4 @@
|
||||
pub(crate) mod bitrot;
|
||||
pub(crate) mod compress;
|
||||
pub(crate) mod rio;
|
||||
pub(crate) mod shard_integrity;
|
||||
|
||||
@@ -0,0 +1,861 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::{DiskAPI, DiskStore};
|
||||
use bytes::Bytes;
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_filemeta::shard_integrity::{
|
||||
IntegrityLayout, MAX_INLINE_PROOF_BYTES, PartIntegrity, SUFFIX_INLINE_INTEGRITY, node_digest,
|
||||
};
|
||||
use rustfs_utils::http::{contains_key_str, get_consistent_str, insert_str};
|
||||
use std::io::{self, Read, Seek, SeekFrom, Write};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
const MEMORY_LIMIT: usize = 1024 * 1024;
|
||||
const COPY_CHUNK_SIZE: usize = 64 * 1024;
|
||||
|
||||
fn corrupt() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "shard integrity verification failed")
|
||||
}
|
||||
|
||||
/// Only encoded digest rows are retained. Payload bytes never enter the spool.
|
||||
/// Large parts spill to an anonymous temporary file; cancellation drops it.
|
||||
pub(crate) struct IntegrityBuilder {
|
||||
part: PartIntegrity,
|
||||
rows: Vec<u8>,
|
||||
spill: Option<tokio::fs::File>,
|
||||
}
|
||||
|
||||
impl IntegrityBuilder {
|
||||
pub(crate) fn new(layout: IntegrityLayout, number: usize) -> io::Result<Self> {
|
||||
Ok(Self {
|
||||
part: PartIntegrity::new(layout, number).map_err(|_| corrupt())?,
|
||||
rows: Vec::new(),
|
||||
spill: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn push<'a>(&mut self, shards: impl ExactSizeIterator<Item = &'a [u8]>) -> io::Result<()> {
|
||||
if shards.len() != self.part.layout.shards() {
|
||||
return Err(corrupt());
|
||||
}
|
||||
let mut row = Vec::with_capacity(shards.len() * 32);
|
||||
for (index, payload) in shards.enumerate() {
|
||||
row.extend_from_slice(
|
||||
&self
|
||||
.part
|
||||
.shard_digest(self.part.stripes, index, payload)
|
||||
.map_err(|_| corrupt())?,
|
||||
);
|
||||
}
|
||||
self.append(&row).await?;
|
||||
self.part.stripes = self.part.stripes.checked_add(1).ok_or_else(corrupt)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append(&mut self, bytes: &[u8]) -> io::Result<()> {
|
||||
self.rows.extend_from_slice(bytes);
|
||||
if self.rows.len() >= MEMORY_LIMIT {
|
||||
if self.spill.is_none() {
|
||||
let file = tokio::task::spawn_blocking(tempfile::tempfile)
|
||||
.await
|
||||
.map_err(io::Error::other)??;
|
||||
self.spill = Some(tokio::fs::File::from_std(file));
|
||||
}
|
||||
if let Some(file) = &mut self.spill {
|
||||
file.write_all(&self.rows).await?;
|
||||
self.rows.clear();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn into_spool(self) -> io::Result<Spool> {
|
||||
if let Some(mut file) = self.spill {
|
||||
file.write_all(&self.rows).await?;
|
||||
file.flush().await?;
|
||||
Ok(Spool::File(file.into_std().await))
|
||||
} else {
|
||||
Ok(Spool::Memory(self.rows))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn finish(mut self, size: usize) -> io::Result<PreparedIntegrity> {
|
||||
self.part.size = size.try_into().map_err(|_| corrupt())?;
|
||||
self.part.validate().map_err(|_| corrupt())?;
|
||||
let part = self.part.clone();
|
||||
let rows = self.into_spool().await?;
|
||||
if part.stripes < 16 {
|
||||
build_index(part, rows)
|
||||
} else {
|
||||
tokio::task::spawn_blocking(move || build_index(part, rows))
|
||||
.await
|
||||
.map_err(io::Error::other)?
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum Spool {
|
||||
Memory(Vec<u8>),
|
||||
File(std::fs::File),
|
||||
}
|
||||
|
||||
impl Spool {
|
||||
fn append(&mut self, bytes: &[u8]) -> io::Result<()> {
|
||||
if let Self::Memory(buffer) = self
|
||||
&& buffer.len() + bytes.len() > MEMORY_LIMIT
|
||||
{
|
||||
let mut file = tempfile::tempfile()?;
|
||||
file.write_all(buffer)?;
|
||||
*self = Self::File(file);
|
||||
}
|
||||
match self {
|
||||
Self::Memory(buffer) => buffer.extend_from_slice(bytes),
|
||||
Self::File(file) => {
|
||||
file.seek(SeekFrom::End(0))?;
|
||||
file.write_all(bytes)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_at(&mut self, offset: u64, out: &mut [u8]) -> io::Result<()> {
|
||||
match self {
|
||||
Self::Memory(buffer) => {
|
||||
let offset = usize::try_from(offset).map_err(|_| corrupt())?;
|
||||
let end = offset.checked_add(out.len()).ok_or_else(corrupt)?;
|
||||
out.copy_from_slice(buffer.get(offset..end).ok_or_else(corrupt)?);
|
||||
Ok(())
|
||||
}
|
||||
Self::File(file) => {
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
file.read_exact(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_index(mut part: PartIntegrity, mut rows: Spool) -> io::Result<PreparedIntegrity> {
|
||||
let leaves = u64::from(part.stripes.max(1)).next_power_of_two();
|
||||
let mut tree = Spool::Memory(Vec::new());
|
||||
let row_size = part.layout.shards() * 32;
|
||||
let mut row = vec![0; row_size];
|
||||
for stripe in 0..leaves {
|
||||
let stripe_number = u32::try_from(stripe).map_err(|_| corrupt())?;
|
||||
let digest = if stripe < u64::from(part.stripes) {
|
||||
rows.read_at(stripe * u64::try_from(row_size).map_err(|_| corrupt())?, &mut row)?;
|
||||
part.leaf_digest(stripe_number, &row).map_err(|_| corrupt())?
|
||||
} else {
|
||||
part.padding_digest(stripe_number)
|
||||
};
|
||||
tree.append(&digest)?;
|
||||
}
|
||||
let mut levels = vec![0u64];
|
||||
let mut count = leaves;
|
||||
let mut level_offset = 0;
|
||||
let mut next_offset = leaves * 32;
|
||||
let mut pair = [0; 64];
|
||||
while count > 1 {
|
||||
for pair_number in 0..count / 2 {
|
||||
tree.read_at(level_offset + pair_number * 64, &mut pair)?;
|
||||
let left = pair[..32].try_into().map_err(|_| corrupt())?;
|
||||
let right = pair[32..].try_into().map_err(|_| corrupt())?;
|
||||
tree.append(&node_digest(left, right))?;
|
||||
}
|
||||
levels.push(next_offset);
|
||||
level_offset = next_offset;
|
||||
count /= 2;
|
||||
next_offset += count * 32;
|
||||
}
|
||||
let mut root = [0; 32];
|
||||
tree.read_at(level_offset, &mut root)?;
|
||||
part.root = part.root_digest(&root);
|
||||
let mut index = Spool::Memory(Vec::new());
|
||||
index.append(&part.index_header())?;
|
||||
let mut sibling = [0; 32];
|
||||
for stripe in 0..part.stripes {
|
||||
rows.read_at(u64::from(stripe) * u64::try_from(row_size).map_err(|_| corrupt())?, &mut row)?;
|
||||
index.append(&row)?;
|
||||
let mut position = u64::from(stripe);
|
||||
for &offset in levels.iter().take(levels.len() - 1) {
|
||||
tree.read_at(offset + (position ^ 1) * 32, &mut sibling)?;
|
||||
index.append(&sibling)?;
|
||||
position >>= 1;
|
||||
}
|
||||
}
|
||||
Ok(PreparedIntegrity { part, index })
|
||||
}
|
||||
|
||||
pub(crate) struct PreparedIntegrity {
|
||||
pub(crate) part: PartIntegrity,
|
||||
index: Spool,
|
||||
}
|
||||
|
||||
impl PreparedIntegrity {
|
||||
pub(crate) async fn copy_from(proof: &PartProofReader) -> io::Result<Self> {
|
||||
let mut builder = IntegrityBuilder {
|
||||
part: proof.part.clone(),
|
||||
rows: proof.part.index_header().to_vec(),
|
||||
spill: None,
|
||||
};
|
||||
for stripe in 0..proof.part.stripes {
|
||||
builder.append(&proof.record(stripe).await?).await?;
|
||||
}
|
||||
let index = builder.into_spool().await?;
|
||||
Ok(Self {
|
||||
part: proof.part.clone(),
|
||||
index,
|
||||
})
|
||||
}
|
||||
pub(crate) fn inline_bytes(&self) -> io::Result<Bytes> {
|
||||
match &self.index {
|
||||
Spool::Memory(bytes) if bytes.len() <= MAX_INLINE_PROOF_BYTES => Ok(Bytes::copy_from_slice(bytes)),
|
||||
_ => Err(corrupt()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn set_inline_metadata(&self, fi: &mut FileInfo) -> io::Result<()> {
|
||||
insert_str(
|
||||
&mut fi.metadata,
|
||||
SUFFIX_INLINE_INTEGRITY,
|
||||
base64_simd::STANDARD.encode_to_string(self.inline_bytes()?),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A disk survives only if BOTH payload and index writes succeed. The caller
|
||||
/// supplies the surviving payload disks and rechecks the commit quorum.
|
||||
pub(crate) async fn write(
|
||||
self,
|
||||
disks: &mut [Option<DiskStore>],
|
||||
original_volume: &str,
|
||||
volume: &str,
|
||||
directory: &str,
|
||||
) -> io::Result<PartIntegrity> {
|
||||
let length = self.part.index_size().map_err(|_| corrupt())?;
|
||||
let path = format!("{directory}/{}", self.part.file_name());
|
||||
let mut writers = futures::future::join_all(disks.iter().map(|disk| async {
|
||||
match disk {
|
||||
Some(disk) => disk
|
||||
.create_file(original_volume, volume, &path, i64::try_from(length).map_err(|_| corrupt())?)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(io::Error::other),
|
||||
None => Ok(None),
|
||||
}
|
||||
}))
|
||||
.await
|
||||
.into_iter()
|
||||
.map(Result::ok)
|
||||
.map(Option::flatten)
|
||||
.collect::<Vec<_>>();
|
||||
let mut source: Box<dyn tokio::io::AsyncRead + Unpin + Send> = match self.index {
|
||||
Spool::Memory(bytes) => Box::new(std::io::Cursor::new(bytes)),
|
||||
Spool::File(mut file) => {
|
||||
// The file has a single owner; duplicated descriptors share offsets.
|
||||
file = tokio::task::spawn_blocking(move || {
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
io::Result::Ok(file)
|
||||
})
|
||||
.await
|
||||
.map_err(io::Error::other)??;
|
||||
Box::new(tokio::fs::File::from_std(file))
|
||||
}
|
||||
};
|
||||
let mut buffer = vec![0; COPY_CHUNK_SIZE.min(length)];
|
||||
loop {
|
||||
let n = source.read(&mut buffer).await?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
futures::future::join_all(writers.iter_mut().map(|writer| {
|
||||
let bytes = &buffer[..n];
|
||||
async move {
|
||||
if let Some(inner) = writer
|
||||
&& !matches!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), inner.write_all(bytes)).await,
|
||||
Ok(Ok(()))
|
||||
)
|
||||
{
|
||||
*writer = None;
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
}
|
||||
futures::future::join_all(writers.iter_mut().zip(disks.iter_mut()).map(|(writer, disk)| async move {
|
||||
let success = if let Some(writer) = writer {
|
||||
matches!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), writer.shutdown()).await,
|
||||
Ok(Ok(()))
|
||||
)
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if !success {
|
||||
*disk = None;
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
Ok(self.part)
|
||||
}
|
||||
}
|
||||
|
||||
/// A shared, two-stripe cache coalesces the N readers' proof fetches. The only
|
||||
/// async lock is held across a bounded read from at most N replicas; it never
|
||||
/// nests another lock. A missing local index may use any authenticated replica.
|
||||
pub(crate) struct PartProofReader {
|
||||
pub(crate) part: PartIntegrity,
|
||||
sources: Vec<(DiskStore, String)>,
|
||||
volume: String,
|
||||
inline: Option<Bytes>,
|
||||
cached: Mutex<ProofCache>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ProofCache {
|
||||
records: std::collections::VecDeque<(u32, Bytes)>,
|
||||
preferred: usize,
|
||||
}
|
||||
|
||||
impl PartProofReader {
|
||||
pub(crate) async fn reconstruction_proof(
|
||||
&self,
|
||||
stripe: usize,
|
||||
shards: &[Option<Vec<u8>>],
|
||||
) -> io::Result<Option<ReconstructionProof>> {
|
||||
let mut missing = 0u16;
|
||||
for (index, shard) in shards.iter().take(usize::from(self.part.layout.data)).enumerate() {
|
||||
if shard.is_none() {
|
||||
missing |= 1 << index;
|
||||
}
|
||||
}
|
||||
if missing == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let stripe = u32::try_from(stripe).map_err(|_| corrupt())?;
|
||||
Ok(Some(ReconstructionProof {
|
||||
part: self.part.clone(),
|
||||
stripe,
|
||||
record: self.record(stripe).await?,
|
||||
missing,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn new(
|
||||
part: PartIntegrity,
|
||||
files: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
volume: &str,
|
||||
object: &str,
|
||||
) -> io::Result<Arc<Self>> {
|
||||
part.validate().map_err(|_| corrupt())?;
|
||||
let mut sources = Vec::with_capacity(disks.len());
|
||||
let mut inline = None;
|
||||
for (file, disk) in files.iter().zip(disks) {
|
||||
let part_index = usize::try_from(part.number)
|
||||
.ok()
|
||||
.and_then(|number| file.parts.binary_search_by_key(&number, |p| p.number).ok());
|
||||
if part_index.and_then(|index| file.parts[index].integrity.as_ref()) != Some(&part) {
|
||||
continue;
|
||||
}
|
||||
if inline.is_none() && contains_key_str(&file.metadata, SUFFIX_INLINE_INTEGRITY) {
|
||||
let candidate = (|| {
|
||||
let value = get_consistent_str(&file.metadata, SUFFIX_INLINE_INTEGRITY)?;
|
||||
if value.len() > MAX_INLINE_PROOF_BYTES.div_ceil(3) * 4 {
|
||||
return None;
|
||||
}
|
||||
let decoded = base64_simd::STANDARD.decode_to_vec(value).ok()?;
|
||||
if decoded.len() != part.index_size().ok()? || !decoded.starts_with(&part.index_header()) {
|
||||
return None;
|
||||
}
|
||||
for stripe in 0..part.stripes {
|
||||
let offset = part.record_offset(stripe).ok()?;
|
||||
part.verify_record(stripe, decoded.get(offset..offset + part.record_size())?)
|
||||
.ok()?;
|
||||
}
|
||||
Some(Bytes::from(decoded))
|
||||
})();
|
||||
if let Some(candidate) = candidate {
|
||||
inline = Some(candidate);
|
||||
}
|
||||
}
|
||||
if let (Some(disk), Some(directory)) = (disk, file.data_dir.filter(|id| !id.is_nil())) {
|
||||
sources.push((disk.clone(), format!("{object}/{directory}/{}", part.file_name())));
|
||||
}
|
||||
}
|
||||
Ok(Arc::new(Self {
|
||||
part,
|
||||
sources,
|
||||
volume: volume.to_owned(),
|
||||
inline,
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
|
||||
let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
|
||||
let mut cached = self.cached.lock().await;
|
||||
if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
|
||||
return Ok(bytes.clone());
|
||||
}
|
||||
let size = self.part.record_size();
|
||||
let record = if let Some(inline) = &self.inline {
|
||||
inline.slice(offset..offset + size)
|
||||
} else {
|
||||
let read = |index: usize| async move {
|
||||
let (disk, path) = &self.sources[index];
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(15), async {
|
||||
let mut reader = disk
|
||||
.read_file_stream(&self.volume, path, offset, size)
|
||||
.await
|
||||
.map_err(io::Error::other)?;
|
||||
let mut bytes = vec![0; size];
|
||||
reader.read_exact(&mut bytes).await?;
|
||||
self.part.verify_record(stripe, &bytes).map_err(|_| corrupt())?;
|
||||
io::Result::Ok(Bytes::from(bytes))
|
||||
})
|
||||
.await;
|
||||
(index, result)
|
||||
};
|
||||
if self.sources.is_empty() {
|
||||
return Err(corrupt());
|
||||
}
|
||||
let preferred = cached.preferred % self.sources.len();
|
||||
let mut pending = FuturesUnordered::new();
|
||||
pending.push(read(preferred));
|
||||
let first = tokio::select! {
|
||||
result = pending.next() => result,
|
||||
() = tokio::time::sleep(std::time::Duration::from_millis(50)) => None,
|
||||
};
|
||||
let mut found = match first {
|
||||
Some((index, Ok(Ok(bytes)))) => Some((index, bytes)),
|
||||
_ => None,
|
||||
};
|
||||
if found.is_none() {
|
||||
for index in 0..self.sources.len() {
|
||||
if index != preferred {
|
||||
pending.push(read(index));
|
||||
}
|
||||
}
|
||||
while let Some((index, result)) = pending.next().await {
|
||||
if let Ok(Ok(bytes)) = result {
|
||||
found = Some((index, bytes));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let (index, bytes) = found.ok_or_else(corrupt)?;
|
||||
cached.preferred = index;
|
||||
bytes
|
||||
};
|
||||
self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
|
||||
if cached.records.len() == 2 {
|
||||
cached.records.pop_front();
|
||||
}
|
||||
cached.records.push_back((stripe, record.clone()));
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
pub(crate) async fn verify(&self, stripe: u32, index: usize, payload: &[u8]) -> io::Result<()> {
|
||||
let record = self.record(stripe).await?;
|
||||
// Authentication of the shared record is cached; payload digests are
|
||||
// always recomputed, including skip_verify and repaired-shard paths.
|
||||
let expected = record.get(index * 32..(index + 1) * 32).ok_or_else(corrupt)?;
|
||||
if self
|
||||
.part
|
||||
.shard_digest(stripe, index, payload)
|
||||
.map_err(|_| corrupt())?
|
||||
.as_slice()
|
||||
!= expected
|
||||
{
|
||||
return Err(corrupt());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ShardVerifier {
|
||||
proof: Arc<PartProofReader>,
|
||||
coding_index: usize,
|
||||
first_stripe: u32,
|
||||
consumed: u32,
|
||||
advanced: Option<Arc<AtomicUsize>>,
|
||||
}
|
||||
|
||||
impl ShardVerifier {
|
||||
pub(crate) fn proof(&self) -> Arc<PartProofReader> {
|
||||
Arc::clone(&self.proof)
|
||||
}
|
||||
pub(crate) fn new(
|
||||
proof: Arc<PartProofReader>,
|
||||
coding_index: usize,
|
||||
first_stripe: usize,
|
||||
advanced: Option<Arc<AtomicUsize>>,
|
||||
) -> io::Result<Self> {
|
||||
if coding_index >= proof.part.layout.shards() {
|
||||
return Err(corrupt());
|
||||
}
|
||||
Ok(Self {
|
||||
proof,
|
||||
coding_index,
|
||||
first_stripe: first_stripe.try_into().map_err(|_| corrupt())?,
|
||||
consumed: 0,
|
||||
advanced,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn verify(&mut self, payload: &[u8]) -> io::Result<()> {
|
||||
let advanced = self.advanced.as_ref().map_or(0, |value| value.load(Ordering::Acquire));
|
||||
let stripe = self
|
||||
.first_stripe
|
||||
.checked_add(u32::try_from(advanced).map_err(|_| corrupt())?)
|
||||
.and_then(|value| value.checked_add(self.consumed))
|
||||
.ok_or_else(corrupt)?;
|
||||
self.proof.verify(stripe, self.coding_index, payload).await?;
|
||||
self.consumed = self.consumed.checked_add(1).ok_or_else(corrupt)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct ReconstructionProof {
|
||||
part: PartIntegrity,
|
||||
stripe: u32,
|
||||
record: Bytes,
|
||||
missing: u16,
|
||||
}
|
||||
|
||||
impl ReconstructionProof {
|
||||
pub(crate) fn verify(&self, shards: &[Option<Vec<u8>>]) -> io::Result<()> {
|
||||
for index in 0..usize::from(self.part.layout.data) {
|
||||
if self.missing & (1 << index) == 0 {
|
||||
continue;
|
||||
}
|
||||
let payload = shards.get(index).and_then(Option::as_deref).ok_or_else(corrupt)?;
|
||||
let actual = self.part.shard_digest(self.stripe, index, payload).map_err(|_| corrupt())?;
|
||||
if self.record.get(index * 32..(index + 1) * 32) != Some(actual.as_slice()) {
|
||||
return Err(corrupt());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Deep scans verify stripes across all disks in lockstep so a single bounded
|
||||
/// proof fetch serves all N payload checks. VerifyFile from an older peer is
|
||||
/// deliberately not treated as evidence of this independent commitment.
|
||||
pub(crate) async fn verify_deep_parts(
|
||||
files: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
expected: &FileInfo,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
statuses: &mut std::collections::HashMap<usize, Vec<usize>>,
|
||||
) -> crate::disk::error::Result<()> {
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, conv_part_err_to_int};
|
||||
use crate::io_support::bitrot::create_bitrot_reader_from_bytes;
|
||||
let erasure = crate::erasure::coding::Erasure::try_new_with_options(
|
||||
expected.erasure.data_blocks,
|
||||
expected.erasure.parity_blocks,
|
||||
expected.erasure.block_size,
|
||||
expected.uses_legacy_checksum,
|
||||
)
|
||||
.map_err(DiskError::from)?;
|
||||
let read_timeout = crate::disk::disk_store::get_object_disk_read_timeout();
|
||||
for (part_index, part) in expected.parts.iter().enumerate() {
|
||||
let commitment = part.integrity.as_ref().ok_or(DiskError::FileCorrupt)?;
|
||||
let proof = PartProofReader::new(commitment.clone(), files, disks, volume, object).map_err(DiskError::from)?;
|
||||
if commitment.stripes == 0 && commitment.root != commitment.root_digest(&commitment.padding_digest(0)) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let part_status = statuses.get_mut(&part_index).ok_or(DiskError::FileCorrupt)?;
|
||||
let length = erasure.shard_file_offset(0, part.size, part.size);
|
||||
let mut readers = Vec::with_capacity(disks.len());
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
if part_status[index] != CHECK_PART_UNKNOWN {
|
||||
readers.push(None);
|
||||
continue;
|
||||
}
|
||||
let file = &files[index];
|
||||
if file.erasure.get_checksum_info(part.number).algorithm != rustfs_utils::HashAlgorithm::HighwayHash256S {
|
||||
return Err(DiskError::BitrotHashAlgoInvalid);
|
||||
}
|
||||
let path = format!("{object}/{}/part.{}", file.data_dir.unwrap_or_default(), part.number);
|
||||
let result = create_bitrot_reader_from_bytes(
|
||||
file.data.clone(),
|
||||
disk.as_ref(),
|
||||
volume,
|
||||
&path,
|
||||
0,
|
||||
length,
|
||||
erasure.shard_size(),
|
||||
rustfs_utils::HashAlgorithm::HighwayHash256S,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(Some(mut reader)) => {
|
||||
let coding_index = expected
|
||||
.erasure
|
||||
.distribution
|
||||
.get(index)
|
||||
.and_then(|index| index.checked_sub(1))
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
reader
|
||||
.set_integrity(ShardVerifier::new(Arc::clone(&proof), coding_index, 0, None).map_err(DiskError::from)?)
|
||||
.map_err(DiskError::from)?;
|
||||
readers.push(Some(reader));
|
||||
part_status[index] = CHECK_PART_SUCCESS;
|
||||
}
|
||||
Ok(None) => {
|
||||
readers.push(None);
|
||||
part_status[index] = CHECK_PART_FILE_CORRUPT;
|
||||
}
|
||||
Err(error) => {
|
||||
readers.push(None);
|
||||
part_status[index] = conv_part_err_to_int(&Some(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut remaining = length;
|
||||
let mut buffers = vec![vec![0; erasure.shard_size().min(length)]; disks.len()];
|
||||
while remaining > 0 {
|
||||
let want = remaining.min(erasure.shard_size());
|
||||
let results = futures::future::join_all(readers.iter_mut().zip(&mut buffers).map(|(reader, buffer)| async move {
|
||||
let Some(reader) = reader else { return Ok(()) };
|
||||
let read = reader.read(&mut buffer[..want]);
|
||||
if read_timeout.is_zero() {
|
||||
read.await.map(|_| ())
|
||||
} else {
|
||||
tokio::time::timeout(read_timeout, read)
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "integrity scan timed out"))?
|
||||
.map(|_| ())
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
for (index, result) in results.into_iter().enumerate() {
|
||||
if result.is_err() {
|
||||
part_status[index] = CHECK_PART_FILE_CORRUPT;
|
||||
readers[index] = None;
|
||||
}
|
||||
}
|
||||
remaining -= want;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn verify_index_file(disk: &DiskStore, volume: &str, path: &str, part: &PartIntegrity) -> io::Result<()> {
|
||||
let size = part.index_size().map_err(|_| corrupt())?;
|
||||
let source = disk.read_file_stream(volume, path, 0, size).await.map_err(io::Error::other)?;
|
||||
let mut source = tokio::io::BufReader::with_capacity(COPY_CHUNK_SIZE, source);
|
||||
let mut header = [0; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE];
|
||||
source.read_exact(&mut header).await?;
|
||||
if header != part.index_header() {
|
||||
return Err(corrupt());
|
||||
}
|
||||
let mut record = vec![0; part.record_size()];
|
||||
for stripe in 0..part.stripes {
|
||||
source.read_exact(&mut record).await?;
|
||||
part.verify_record(stripe, &record).map_err(|_| corrupt())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Restore only untrusted/missing proof replicas, under the existing root and
|
||||
/// the caller's exclusive object lock. This does not rewrite payload or xl.meta.
|
||||
pub(crate) async fn restore_proof_replicas(
|
||||
expected: &FileInfo,
|
||||
disks: &[Option<DiskStore>],
|
||||
volume: &str,
|
||||
object: &str,
|
||||
) -> crate::disk::error::Result<usize> {
|
||||
use crate::disk::DeleteOptions;
|
||||
use crate::disk::RUSTFS_META_TMP_BUCKET;
|
||||
use crate::disk::error::DiskError;
|
||||
if contains_key_str(&expected.metadata, SUFFIX_INLINE_INTEGRITY) || expected.deleted || expected.is_remote() {
|
||||
return Ok(0);
|
||||
}
|
||||
let directory = expected.data_dir.filter(|id| !id.is_nil()).ok_or(DiskError::FileCorrupt)?;
|
||||
let mut repaired = 0;
|
||||
for part in &expected.parts {
|
||||
let part = part.integrity.as_ref().ok_or(DiskError::FileCorrupt)?;
|
||||
let path = format!("{object}/{directory}/{}", part.file_name());
|
||||
let checks = futures::future::join_all(disks.iter().map(|disk| async {
|
||||
let Some(disk) = disk else { return true };
|
||||
matches!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), verify_index_file(disk, volume, &path, part)).await,
|
||||
Ok(Ok(()))
|
||||
)
|
||||
}))
|
||||
.await;
|
||||
if checks.iter().all(|valid| *valid) {
|
||||
continue;
|
||||
}
|
||||
let proof = PartProofReader {
|
||||
part: part.clone(),
|
||||
sources: disks.iter().flatten().map(|disk| (disk.clone(), path.clone())).collect(),
|
||||
volume: volume.to_owned(),
|
||||
inline: None,
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
};
|
||||
let prepared = PreparedIntegrity::copy_from(&proof).await.map_err(DiskError::from)?;
|
||||
let temporary = format!("integrity-{}", uuid::Uuid::new_v4());
|
||||
let mut targets: Vec<_> = disks
|
||||
.iter()
|
||||
.zip(&checks)
|
||||
.map(|(disk, valid)| if *valid { None } else { disk.clone() })
|
||||
.collect();
|
||||
let result: crate::disk::error::Result<()> = async {
|
||||
prepared
|
||||
.write(&mut targets, volume, RUSTFS_META_TMP_BUCKET, &temporary)
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
let source_path = format!("{temporary}/{}", part.file_name());
|
||||
for (index, valid) in checks.iter().enumerate() {
|
||||
if *valid {
|
||||
continue;
|
||||
}
|
||||
let disk = targets[index].as_ref().ok_or(DiskError::ErasureWriteQuorum)?;
|
||||
disk.rename_file_durable(RUSTFS_META_TMP_BUCKET, &source_path, volume, &path)
|
||||
.await?;
|
||||
repaired += 1;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
for disk in disks.iter().flatten() {
|
||||
let _ = disk
|
||||
.delete(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&temporary,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
result?;
|
||||
}
|
||||
Ok(repaired)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn integrity_index_rejects_donor_payload_record_position_and_generation() {
|
||||
let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
|
||||
let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
|
||||
builder.part.generation = uuid::Uuid::parse_str("00112233-4455-4677-8899-aabbccddeeff").expect("fixed generation");
|
||||
let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
|
||||
builder.push(shards.into_iter()).await.expect("first stripe");
|
||||
builder.push(shards.into_iter()).await.expect("second stripe");
|
||||
let prepared = builder.finish(16).await.expect("index");
|
||||
let bytes = prepared.inline_bytes().expect("inline index");
|
||||
let part = prepared.part;
|
||||
// Frozen independently with Python hashlib from the v1 byte layout.
|
||||
let expected_root = [
|
||||
33, 254, 159, 226, 214, 235, 239, 175, 182, 132, 172, 81, 6, 234, 125, 187, 25, 98, 190, 6, 251, 241, 171, 33, 150,
|
||||
188, 179, 250, 26, 60, 47, 198,
|
||||
];
|
||||
assert_eq!(part.root, expected_root);
|
||||
let offset = part.record_offset(0).expect("offset");
|
||||
let record = &bytes[offset..offset + part.record_size()];
|
||||
part.verify_shard(0, 0, shards[0], record).expect("target payload");
|
||||
assert!(part.verify_shard(0, 0, shards[1], record).is_err());
|
||||
assert!(part.verify_record(1, record).is_err());
|
||||
let mut other = part.clone();
|
||||
other.generation = uuid::Uuid::new_v4();
|
||||
assert!(other.verify_record(0, record).is_err());
|
||||
for position in [0, 32, record.len() - 1] {
|
||||
let mut corrupt_record = record.to_vec();
|
||||
corrupt_record[position] ^= 1;
|
||||
assert!(part.verify_record(0, &corrupt_record).is_err());
|
||||
}
|
||||
let proof = Arc::new(PartProofReader {
|
||||
part,
|
||||
sources: vec![],
|
||||
volume: "bucket".to_owned(),
|
||||
inline: Some(bytes),
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
});
|
||||
let mut verifier = ShardVerifier::new(proof, 0, 0, None).expect("verifier");
|
||||
verifier.verify(shards[0]).await.expect("stripe 0");
|
||||
verifier.verify(shards[0]).await.expect("stripe 1");
|
||||
assert!(verifier.verify(shards[0]).await.is_err(), "no clean success past committed stripes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn integrity_reconstruction_rejects_wrong_decoder_output() {
|
||||
let mut builder = IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 1).expect("builder");
|
||||
builder
|
||||
.push([b"abcd".as_slice(), b"efgh", b"ijkl", b"mnop"].into_iter())
|
||||
.await
|
||||
.expect("stripe");
|
||||
let prepared = builder.finish(8).await.expect("index");
|
||||
let index = prepared.inline_bytes().expect("bytes");
|
||||
let proof = PartProofReader {
|
||||
part: prepared.part,
|
||||
inline: Some(index),
|
||||
sources: Vec::new(),
|
||||
volume: String::new(),
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
};
|
||||
let mut shards = vec![None, Some(b"efgh".to_vec()), Some(b"ijkl".to_vec()), Some(b"mnop".to_vec())];
|
||||
let verification = proof
|
||||
.reconstruction_proof(0, &shards)
|
||||
.await
|
||||
.expect("proof")
|
||||
.expect("missing data");
|
||||
shards[0] = Some(b"donr".to_vec());
|
||||
assert!(verification.verify(&shards).is_err());
|
||||
shards[0] = Some(b"abcd".to_vec());
|
||||
verification.verify(&shards).expect("verified reconstruction");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn integrity_builder_spills_without_changing_proofs() {
|
||||
let layout = IntegrityLayout::new(12, 4, 12, false).expect("layout");
|
||||
let mut builder = IntegrityBuilder::new(layout, 7).expect("builder");
|
||||
let shards = [b"x".as_slice(); 16];
|
||||
for _ in 0..2050 {
|
||||
builder.push(shards.into_iter()).await.expect("stripe");
|
||||
}
|
||||
assert!(builder.spill.is_some(), "digest rows exceed memory bound");
|
||||
let mut prepared = builder.finish(2050 * 12).await.expect("spilled tree");
|
||||
assert!(matches!(prepared.index, Spool::File(_)), "materialized index is also bounded");
|
||||
let mut record = vec![0; prepared.part.record_size()];
|
||||
for stripe in [0, 1025, 2049] {
|
||||
let offset = prepared.part.record_offset(stripe).expect("offset");
|
||||
prepared
|
||||
.index
|
||||
.read_at(u64::try_from(offset).expect("offset fits"), &mut record)
|
||||
.expect("proof");
|
||||
prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -882,6 +882,15 @@ pub enum WriteCompletion {
|
||||
TailDrained,
|
||||
}
|
||||
|
||||
/// Storage-owned write mode inherited by physical rewrites. This selects the
|
||||
/// destination format; the source reader must still validate every source byte.
|
||||
#[doc(hidden)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ShardIntegrityWriteMode {
|
||||
Legacy,
|
||||
Protected,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
@@ -942,6 +951,10 @@ pub struct ObjectOptions {
|
||||
|
||||
pub data_movement: bool,
|
||||
pub raw_data_movement_read: bool,
|
||||
/// Internal, in-memory protection context. Never populated from S3 metadata.
|
||||
/// None selects the rollout default only for a new write, not for a rewrite.
|
||||
#[doc(hidden)]
|
||||
pub shard_integrity_write_mode: Option<ShardIntegrityWriteMode>,
|
||||
/// Durable reservation identity carried only by decommission writes. Other
|
||||
/// data-movement users, including rebalance, leave it unset. Keep this
|
||||
/// context boxed because `ObjectOptions` is passed by value through deep
|
||||
@@ -1043,6 +1056,25 @@ pub enum ReplicationStatusWritebackMode {
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
|
||||
match self.shard_integrity_write_mode {
|
||||
Some(ShardIntegrityWriteMode::Protected) => true,
|
||||
Some(ShardIntegrityWriteMode::Legacy) => false,
|
||||
None if self.data_movement => false,
|
||||
None => {
|
||||
rustfs_utils::get_env_bool(rustfs_config::ENV_SHARD_INTEGRITY_WRITE, rustfs_config::DEFAULT_SHARD_INTEGRITY_WRITE)
|
||||
&& rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED,
|
||||
rustfs_config::DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn inherit_shard_integrity(&mut self, source: &ObjectInfo) {
|
||||
self.shard_integrity_write_mode = Some(source.shard_integrity_write_mode());
|
||||
}
|
||||
|
||||
pub(crate) fn with_capacity_expected_data_bytes(expected_data_bytes: Option<usize>) -> Self {
|
||||
Self {
|
||||
decommission_capacity: expected_data_bytes.map(|expected_data_bytes| {
|
||||
@@ -1438,6 +1470,23 @@ impl Clone for ObjectInfo {
|
||||
}
|
||||
|
||||
impl ObjectInfo {
|
||||
pub(crate) fn shard_integrity_write_mode(&self) -> ShardIntegrityWriteMode {
|
||||
// Any declaration requires protection. Malformed declarations remain
|
||||
// errors in the source reader and must never select the legacy path.
|
||||
if self.parts.iter().any(|part| part.integrity.is_some())
|
||||
|| [
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY,
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY,
|
||||
]
|
||||
.iter()
|
||||
.any(|suffix| rustfs_utils::http::contains_key_str(&self.user_defined, suffix))
|
||||
{
|
||||
ShardIntegrityWriteMode::Protected
|
||||
} else {
|
||||
ShardIntegrityWriteMode::Legacy
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture the source mutation snapshot used by replication workers when
|
||||
/// publishing terminal status. The semantic fingerprint is recomputed at
|
||||
/// the storage CAS boundary, so an older writer that preserves an unknown
|
||||
@@ -1840,6 +1889,7 @@ impl ObjectInfo {
|
||||
checksums: part.checksums.clone(),
|
||||
number: part.number,
|
||||
error: part.error.clone(),
|
||||
integrity: part.integrity.clone(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
|
||||
@@ -924,7 +924,7 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
|
||||
responses: &[Option<Vec<ObjectPartInfo>>],
|
||||
read_quorum: usize,
|
||||
) -> disk::error::Result<ObjectPartInfo> {
|
||||
let mut part_quorum: HashMap<(&str, usize, usize, i64), (usize, &ObjectPartInfo)> = HashMap::new();
|
||||
let mut part_quorum = HashMap::new();
|
||||
let mut present_count = 0usize;
|
||||
let mut missing_count = 0usize;
|
||||
let mut transient_error_count = 0usize;
|
||||
@@ -943,7 +943,7 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
|
||||
if !parts[part_idx].etag.is_empty() {
|
||||
present_count += 1;
|
||||
let part = &parts[part_idx];
|
||||
let key = (part.etag.as_str(), part.number, part.size, part.actual_size);
|
||||
let key = (part.etag.as_str(), part.number, part.size, part.actual_size, part.integrity.as_ref());
|
||||
let (count, _) = part_quorum.entry(key).or_insert((0, part));
|
||||
*count += 1;
|
||||
continue;
|
||||
@@ -1420,6 +1420,40 @@ pub(in crate::set_disk) fn get_bitrot_reader_setup_strategy(
|
||||
}
|
||||
|
||||
impl BitrotReaderSetup {
|
||||
pub(in crate::set_disk) fn bind_integrity(
|
||||
&mut self,
|
||||
expected: Option<&rustfs_filemeta::shard_integrity::PartIntegrity>,
|
||||
files: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
first_stripe: usize,
|
||||
) -> std::io::Result<()> {
|
||||
use crate::io_support::shard_integrity::{PartProofReader, ShardVerifier};
|
||||
let Some(expected) = expected else { return Ok(()) };
|
||||
let proof = PartProofReader::new(expected.clone(), files, disks, bucket, object)?;
|
||||
for (index, reader) in self.readers.iter_mut().enumerate() {
|
||||
if let Some(reader) = reader {
|
||||
let advanced = self.deferred_stripe_handles[index]
|
||||
.as_ref()
|
||||
.map(DeferredReaderStripeHandle::integrity_position);
|
||||
reader.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, first_stripe, advanced)?)?;
|
||||
}
|
||||
if let Some(reopen) = self.deferred_reopeners[index].take() {
|
||||
let proof = Arc::clone(&proof);
|
||||
self.deferred_reopeners[index] = Some(Arc::new(move |stripe| {
|
||||
let mut reader = reopen(stripe)?;
|
||||
let first = first_stripe.checked_add(stripe)?;
|
||||
reader
|
||||
.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, first, None).ok()?)
|
||||
.ok()?;
|
||||
Some(reader)
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn new(shards: usize) -> Self {
|
||||
Self {
|
||||
readers: (0..shards).map(|_| None).collect(),
|
||||
@@ -5698,6 +5732,8 @@ impl SetDisks {
|
||||
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
|
||||
) -> disk::error::Result<Vec<Option<DiskStore>>> {
|
||||
self.recover_part_transaction(dst_object, write_quorum).await?;
|
||||
let part = ObjectPartInfo::unmarshal(&meta)?;
|
||||
let integrity = part.integrity.map(Arc::new);
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
@@ -5711,12 +5747,39 @@ impl SetDisks {
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
let meta = meta.clone();
|
||||
let integrity = integrity.clone();
|
||||
async move {
|
||||
let disk = disk?;
|
||||
Some(
|
||||
disk.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
|
||||
.await,
|
||||
)
|
||||
let prepared = disk
|
||||
.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
|
||||
.await;
|
||||
if let Err(error) = prepared {
|
||||
return Some(Err(error));
|
||||
}
|
||||
if let Some(integrity) = integrity {
|
||||
let Some((directory, _)) = dst_object.rsplit_once('/') else {
|
||||
return Some(Err(DiskError::FileCorrupt));
|
||||
};
|
||||
let path = format!("{directory}/{}", integrity.file_name());
|
||||
// Older peers may return Ok from PreparePart without moving
|
||||
// the index. They must not enter the protected write quorum.
|
||||
let result = async {
|
||||
let mut reader = disk
|
||||
.read_file_stream(&dst_bucket, &path, 0, rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE)
|
||||
.await?;
|
||||
let mut header = [0; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE];
|
||||
tokio::io::AsyncReadExt::read_exact(&mut reader, &mut header)
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
if header != integrity.index_header() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
return Some(result);
|
||||
}
|
||||
Some(Ok(()))
|
||||
}
|
||||
});
|
||||
let prepare_results = join_all(prepare_tasks).await;
|
||||
|
||||
@@ -488,6 +488,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
pub(crate) fn hydrate_selected_fileinfo_part_checksums(fi: &mut FileInfo) -> disk::error::Result<()> {
|
||||
fi.hydrate_shard_integrity().map_err(DiskError::from)?;
|
||||
fi.hydrate_data_movement_part_checksums().map_err(DiskError::from)?;
|
||||
for part in &fi.parts {
|
||||
let Some(checksums) = part.checksums.as_ref() else {
|
||||
|
||||
@@ -6426,6 +6426,24 @@ async fn disks_with_all_parts(
|
||||
}
|
||||
}
|
||||
|
||||
if scan_mode == HealScanMode::Deep
|
||||
&& !latest_meta.deleted
|
||||
&& !latest_meta.is_remote()
|
||||
&& latest_meta.parts.iter().any(|part| part.integrity.is_some())
|
||||
{
|
||||
crate::io_support::shard_integrity::verify_deep_parts(
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
latest_meta,
|
||||
bucket,
|
||||
object,
|
||||
&mut data_errs_by_part,
|
||||
)
|
||||
.await?;
|
||||
populate_data_errs_by_disk(&mut data_errs_by_disk, &data_errs_by_part);
|
||||
return Ok((data_errs_by_disk, data_errs_by_part));
|
||||
}
|
||||
|
||||
// Check data for each disk
|
||||
for (index, disk) in online_disks.iter().enumerate() {
|
||||
if meta_errs[index].is_some() {
|
||||
@@ -6858,6 +6876,7 @@ fn completed_multipart_object_part(part_num: usize, ext_part: &ObjectPartInfo) -
|
||||
actual_size: ext_part.actual_size,
|
||||
index: ext_part.index.clone(),
|
||||
checksums: ext_part.checksums.clone(),
|
||||
integrity: ext_part.integrity.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -823,6 +823,13 @@ impl SetDisks {
|
||||
match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) {
|
||||
Ok(mut latest_meta) => {
|
||||
Self::hydrate_selected_fileinfo_part_checksums(&mut latest_meta)?;
|
||||
let protected =
|
||||
!latest_meta.parts.is_empty() && latest_meta.parts.iter().all(|part| part.integrity.is_some());
|
||||
result.integrity_verified = protected
|
||||
&& !latest_meta.deleted
|
||||
&& !latest_meta.is_remote()
|
||||
&& opts.scan_mode == HealScanMode::Deep
|
||||
&& !read_repair_uses_shared_lock;
|
||||
trace!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -926,7 +933,31 @@ impl SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() && !protected {
|
||||
result.detail =
|
||||
"Legacy object uses standard repair; independent object identity remains unverified".to_owned();
|
||||
}
|
||||
|
||||
if disks_to_heal_count == 0 {
|
||||
if result.integrity_verified && !opts.dry_run {
|
||||
match crate::io_support::shard_integrity::restore_proof_replicas(
|
||||
&latest_meta,
|
||||
&disks,
|
||||
bucket,
|
||||
object,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(repaired) if repaired > 0 => {
|
||||
result.detail = format!("Restored {repaired} independent integrity indexes")
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
result.integrity_verified = false;
|
||||
return Ok((result, Some(error)));
|
||||
}
|
||||
}
|
||||
}
|
||||
// The object is already healthy: no disk needs healing.
|
||||
// This is the common case for the very objects PR #4356
|
||||
// targets — a valid `xl.meta` plus a leaked pre-#3510
|
||||
@@ -1215,6 +1246,15 @@ impl SetDisks {
|
||||
let mut writer_failure_warned = false;
|
||||
|
||||
for (part_index, part) in latest_meta.parts.iter().enumerate() {
|
||||
use crate::io_support::shard_integrity::{PartProofReader, PreparedIntegrity, ShardVerifier};
|
||||
let proof = part
|
||||
.integrity
|
||||
.as_ref()
|
||||
.map(|part| {
|
||||
PartProofReader::new(part.clone(), &parts_metadata, &latest_disks, bucket, object)
|
||||
})
|
||||
.transpose()
|
||||
.map_err(DiskError::from)?;
|
||||
let till_offset = erasure.shard_file_offset(0, part.size, part.size);
|
||||
let use_mmap_read = object_mmap_read_enabled();
|
||||
|
||||
@@ -1265,7 +1305,15 @@ impl SetDisks {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(reader)) => {
|
||||
Ok(Some(mut reader)) => {
|
||||
if let Some(proof) = &proof {
|
||||
reader
|
||||
.set_integrity(
|
||||
ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None)
|
||||
.map_err(DiskError::from)?,
|
||||
)
|
||||
.map_err(DiskError::from)?;
|
||||
}
|
||||
readers.push(Some(reader));
|
||||
}
|
||||
Ok(None) => {
|
||||
@@ -1324,6 +1372,13 @@ impl SetDisks {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mut writer = writer;
|
||||
if let Some(proof) = &proof {
|
||||
writer.set_integrity(
|
||||
ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None)
|
||||
.map_err(DiskError::from)?,
|
||||
);
|
||||
}
|
||||
writers.push(Some(writer));
|
||||
} else {
|
||||
writers.push(None);
|
||||
@@ -1363,6 +1418,26 @@ impl SetDisks {
|
||||
return Err(e);
|
||||
}
|
||||
// close_bitrot_writers(&mut writers).await?;
|
||||
if !is_inline_buffer && let Some(proof) = &proof {
|
||||
let before = out_dated_disks.iter().filter(|disk| disk.is_some()).count();
|
||||
let proof_result = async {
|
||||
PreparedIntegrity::copy_from(proof)
|
||||
.await?
|
||||
.write(
|
||||
&mut out_dated_disks,
|
||||
bucket,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&format!("{tmp_id}/{dst_data_dir}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
if let Err(error) = proof_result {
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
return Err(error.into());
|
||||
}
|
||||
disks_to_heal_count -= before - out_dated_disks.iter().filter(|disk| disk.is_some()).count();
|
||||
}
|
||||
|
||||
for (index, disk_op) in out_dated_disks.iter_mut().enumerate() {
|
||||
if disk_op.is_none() {
|
||||
@@ -1385,6 +1460,13 @@ impl SetDisks {
|
||||
part.index.clone(),
|
||||
part.checksums.clone(),
|
||||
);
|
||||
if let Some(updated) = parts_metadata[index]
|
||||
.parts
|
||||
.iter_mut()
|
||||
.find(|updated| updated.number == part.number)
|
||||
{
|
||||
updated.integrity.clone_from(&part.integrity);
|
||||
}
|
||||
if is_inline_buffer {
|
||||
if let Some(writer) = writers[index].take() {
|
||||
// if let Some(w) = writer.as_any().downcast_ref::<BitrotFileWriter>() {
|
||||
@@ -1560,6 +1642,15 @@ impl SetDisks {
|
||||
));
|
||||
}
|
||||
|
||||
if result.integrity_verified
|
||||
&& let Err(error) =
|
||||
crate::io_support::shard_integrity::restore_proof_replicas(&latest_meta, &disks, bucket, object)
|
||||
.await
|
||||
{
|
||||
result.integrity_verified = false;
|
||||
return Ok((result, Some(error)));
|
||||
}
|
||||
|
||||
// The object is healthy here; sweep any data dirs left behind
|
||||
// by pre-#3510 unversioned overwrites, which the dangling paths
|
||||
// above never touch (issues #3231, #3191). Best effort — a
|
||||
@@ -1678,6 +1769,17 @@ impl SetDisks {
|
||||
let Some(candidate) = candidates.first().copied() else {
|
||||
return Ok(false);
|
||||
};
|
||||
let protected = rustfs_filemeta::shard_integrity::descriptor_from_metadata(&candidate.metadata)
|
||||
.map_err(DiskError::from)?
|
||||
.is_some()
|
||||
|| candidate.parts.iter().any(|part| part.integrity.is_some());
|
||||
// A minority metadata copy cannot establish an independent commitment.
|
||||
// Legacy recovery retains its existing bounds without certifying identity.
|
||||
if protected
|
||||
&& (candidates.len() < candidate.erasure.data_blocks || candidate.parts.iter().any(|part| part.integrity.is_none()))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let identity = Self::file_info_quorum_hash(candidate);
|
||||
if candidates
|
||||
.iter()
|
||||
@@ -1711,6 +1813,31 @@ impl SetDisks {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut verified_disks = vec![true; disks.len()];
|
||||
if protected {
|
||||
let mut proof_files = parts_metadata.to_vec();
|
||||
for (index, file) in proof_files.iter_mut().enumerate() {
|
||||
if matches!(
|
||||
errs.get(index).and_then(Option::as_ref),
|
||||
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
||||
) {
|
||||
*file = candidate.clone();
|
||||
file.erasure.index = candidate.erasure.distribution[index];
|
||||
}
|
||||
}
|
||||
let mut verified = (0..candidate.parts.len())
|
||||
.map(|part| (part, vec![crate::disk::CHECK_PART_UNKNOWN; disks.len()]))
|
||||
.collect();
|
||||
crate::io_support::shard_integrity::verify_deep_parts(&proof_files, disks, candidate, bucket, object, &mut verified)
|
||||
.await?;
|
||||
for (index, valid) in verified_disks.iter_mut().enumerate() {
|
||||
*valid = verified.values().all(|parts| parts.get(index) == Some(&CHECK_PART_SUCCESS));
|
||||
}
|
||||
if verified_disks.iter().filter(|valid| **valid).count() < candidate.erasure.data_blocks {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
let mut wrote = 0usize;
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let Some(disk) = disk else {
|
||||
@@ -1720,7 +1847,7 @@ impl SetDisks {
|
||||
errs.get(index).and_then(Option::as_ref),
|
||||
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
||||
);
|
||||
if !metadata_absent {
|
||||
if !metadata_absent || !verified_disks[index] {
|
||||
continue;
|
||||
}
|
||||
let Some(&shard_index) = candidate.erasure.distribution.get(index) else {
|
||||
@@ -5439,3 +5566,756 @@ mod heal_result_report_tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod shard_integrity_rollout_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod shard_identity_tests {
|
||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn shard_integrity_heal_rejects_complete_donor_shards_ec12_4() {
|
||||
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(16, 0, 4).await;
|
||||
let bucket = "donor-shard-identity";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create fixture bucket");
|
||||
}
|
||||
let expected = vec![0x5a; 5 * 1024 * 1024 + 123];
|
||||
let donor_body = vec![0xa6; expected.len()];
|
||||
let options = ObjectOptions {
|
||||
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
for (case, (coding_indexes, replace_descriptor)) in [
|
||||
(vec![1], false),
|
||||
(vec![6], false),
|
||||
(vec![12], false),
|
||||
(vec![13], false),
|
||||
(vec![16], false),
|
||||
(vec![1, 2, 3, 4], false),
|
||||
(vec![1, 2, 3, 4, 5], false),
|
||||
(vec![1], true),
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let target = format!("target-{case}");
|
||||
let donor = format!("donor-{case}");
|
||||
set.put_object(bucket, &target, &mut PutObjReader::from_vec(expected.clone()), &options)
|
||||
.await
|
||||
.expect("commit every target shard");
|
||||
set.put_object(bucket, &donor, &mut PutObjReader::from_vec(donor_body.clone()), &options)
|
||||
.await
|
||||
.expect("commit every donor shard");
|
||||
let mut target_meta = Vec::new();
|
||||
let mut donor_meta = Vec::new();
|
||||
for disk in &disks {
|
||||
target_meta.push(
|
||||
disk.read_version("", bucket, &target, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("target metadata"),
|
||||
);
|
||||
donor_meta.push(
|
||||
disk.read_version("", bucket, &donor, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("donor metadata"),
|
||||
);
|
||||
}
|
||||
assert_eq!((target_meta[0].erasure.data_blocks, target_meta[0].erasure.parity_blocks), (12, 4));
|
||||
let mut before = Vec::new();
|
||||
let mut paths = Vec::new();
|
||||
let mut meta_before = Vec::new();
|
||||
for (slot, meta) in target_meta.iter().enumerate() {
|
||||
let object_path = dirs[slot].path().join(bucket).join(&target);
|
||||
let path = object_path
|
||||
.join(meta.data_dir.expect("external data dir").to_string())
|
||||
.join("part.1");
|
||||
before.push(tokio::fs::read(&path).await.expect("original shard"));
|
||||
meta_before.push(tokio::fs::read(object_path.join("xl.meta")).await.expect("original xl.meta"));
|
||||
paths.push(path);
|
||||
}
|
||||
for index in &coding_indexes {
|
||||
let target_slot = target_meta
|
||||
.iter()
|
||||
.position(|m| m.erasure.index == *index)
|
||||
.expect("target coding index");
|
||||
let donor_slot = donor_meta
|
||||
.iter()
|
||||
.position(|m| m.erasure.index == *index)
|
||||
.expect("donor coding index");
|
||||
let path = dirs[donor_slot]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(&donor)
|
||||
.join(donor_meta[donor_slot].data_dir.expect("donor data dir").to_string())
|
||||
.join("part.1");
|
||||
let replacement = tokio::fs::read(path).await.expect("complete valid donor");
|
||||
assert_eq!(
|
||||
replacement.len(),
|
||||
before[target_slot].len(),
|
||||
"the adversarial file must have the same physical length"
|
||||
);
|
||||
assert_ne!(replacement, before[target_slot]);
|
||||
tokio::fs::write(&paths[target_slot], replacement)
|
||||
.await
|
||||
.expect("replace the complete shard without changing metadata");
|
||||
if replace_descriptor {
|
||||
let mut altered = target_meta[target_slot].clone();
|
||||
let suffix = rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY;
|
||||
let descriptor = rustfs_utils::http::get_consistent_str(&donor_meta[donor_slot].metadata, suffix)
|
||||
.expect("donor descriptor");
|
||||
rustfs_utils::http::insert_str(&mut altered.metadata, suffix, descriptor.to_owned());
|
||||
disks[target_slot]
|
||||
.write_metadata("", bucket, &target, altered)
|
||||
.await
|
||||
.expect("a minority descriptor must not authorize its own donor payload");
|
||||
}
|
||||
}
|
||||
let version = target_meta[0].version_id.expect("versioned fixture").to_string();
|
||||
if coding_indexes.len() <= 4 {
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, &target, None, Default::default(), &options)
|
||||
.await
|
||||
.expect("recoverable GET before explicit heal");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("verify and reconstruct every GET block");
|
||||
assert_eq!(body, expected, "GET must not expose donor data before explicit heal");
|
||||
}
|
||||
let heal = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
&target,
|
||||
&version,
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if coding_indexes.len() <= 4 {
|
||||
let (result, error) = heal.expect("recoverable donor mismatch");
|
||||
assert!(error.is_none(), "{error:?}");
|
||||
assert_eq!(
|
||||
result.drives_healed(),
|
||||
Some(coding_indexes.len()),
|
||||
"every complete donor must be detected"
|
||||
);
|
||||
for (slot, path) in paths.iter().enumerate() {
|
||||
assert_eq!(
|
||||
tokio::fs::read(path).await.expect("post-heal shard"),
|
||||
before[slot],
|
||||
"reconstruct exact original bytes at slot {slot}"
|
||||
);
|
||||
}
|
||||
for (range, expected_range) in [
|
||||
(None, expected.as_slice()),
|
||||
(
|
||||
Some(HTTPRangeSpec {
|
||||
start: 1024 * 1024 + 17,
|
||||
end: 2 * 1024 * 1024 + 99,
|
||||
is_suffix_length: false,
|
||||
}),
|
||||
&expected[1024 * 1024 + 17..2 * 1024 * 1024 + 100],
|
||||
),
|
||||
] {
|
||||
let mut reader = set
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
&target,
|
||||
range,
|
||||
Default::default(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(version.clone()),
|
||||
..options.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read repaired version");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("complete verified GET body");
|
||||
assert_eq!(body, expected_range, "full and range GET must match original bytes");
|
||||
}
|
||||
} else {
|
||||
assert!(
|
||||
heal.as_ref().map_or(true, |(_, error)| error.is_some()),
|
||||
"eleven authoritative sources must not produce a successful heal"
|
||||
);
|
||||
let read = set
|
||||
.get_object_reader(bucket, &target, None, Default::default(), &options)
|
||||
.await;
|
||||
if let Ok(mut reader) = read {
|
||||
let mut body = Vec::new();
|
||||
assert!(reader.stream.read_to_end(&mut body).await.is_err(), "quorum-minus-one must fail the body");
|
||||
assert!(body.is_empty(), "unverified first-stripe bytes must not escape");
|
||||
}
|
||||
for (slot, meta) in target_meta.iter().enumerate() {
|
||||
if !coding_indexes.contains(&meta.erasure.index) {
|
||||
assert_eq!(tokio::fs::read(&paths[slot]).await.expect("retained correct shard"), before[slot]);
|
||||
}
|
||||
assert_eq!(
|
||||
tokio::fs::read(dirs[slot].path().join(bucket).join(&target).join("xl.meta"))
|
||||
.await
|
||||
.expect("retained metadata"),
|
||||
meta_before[slot]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shard_integrity_inline_donor_repair_and_late_external_get() {
|
||||
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
|
||||
let bucket = "bound-inline-and-late";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let options = ObjectOptions {
|
||||
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let read_options = ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
};
|
||||
for (size, late) in [(4113usize, false), (3 * 1024 * 1024 + 123, true)] {
|
||||
let target = format!("target-{size}");
|
||||
let donor = format!("donor-{size}");
|
||||
let expected = vec![0x4d; size];
|
||||
set.put_object(bucket, &target, &mut PutObjReader::from_vec(expected.clone()), &options)
|
||||
.await
|
||||
.expect("target PUT");
|
||||
set.put_object(bucket, &donor, &mut PutObjReader::from_vec(vec![0x9a; size]), &options)
|
||||
.await
|
||||
.expect("donor PUT");
|
||||
let mut target_parts = Vec::new();
|
||||
let mut donor_parts = Vec::new();
|
||||
for disk in &disks {
|
||||
target_parts.push(
|
||||
disk.read_version("", bucket, &target, "", &read_options)
|
||||
.await
|
||||
.expect("target meta"),
|
||||
);
|
||||
donor_parts.push(
|
||||
disk.read_version("", bucket, &donor, "", &read_options)
|
||||
.await
|
||||
.expect("donor meta"),
|
||||
);
|
||||
}
|
||||
let slot = target_parts
|
||||
.iter()
|
||||
.position(|fi| fi.erasure.index == 1)
|
||||
.expect("target data slot");
|
||||
let donor_slot = donor_parts
|
||||
.iter()
|
||||
.position(|fi| fi.erasure.index == 1)
|
||||
.expect("donor data slot");
|
||||
if !late {
|
||||
let original = target_parts[slot].data.clone().expect("inline target");
|
||||
let donor_bytes = donor_parts[donor_slot].data.as_ref().expect("inline donor");
|
||||
assert_eq!(donor_bytes.len(), original.len());
|
||||
let path = dirs[slot].path().join(bucket).join(&target).join("xl.meta");
|
||||
let mut raw = tokio::fs::read(&path).await.expect("physical inline metadata");
|
||||
let offsets: Vec<_> = raw
|
||||
.windows(original.len())
|
||||
.enumerate()
|
||||
.filter_map(|(offset, bytes)| (bytes == original.as_ref()).then_some(offset))
|
||||
.collect();
|
||||
assert_eq!(offsets.len(), 1, "unique inline payload range");
|
||||
raw[offsets[0]..offsets[0] + original.len()].copy_from_slice(donor_bytes);
|
||||
tokio::fs::write(path, raw)
|
||||
.await
|
||||
.expect("replace inline bytes with target identity retained");
|
||||
let (result, error) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
&target,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("inline heal");
|
||||
assert!(error.is_none(), "{error:?}");
|
||||
assert_eq!(result.drives_healed(), Some(1));
|
||||
let repaired = disks[slot]
|
||||
.read_version("", bucket, &target, "", &read_options)
|
||||
.await
|
||||
.expect("repaired inline");
|
||||
assert_eq!(repaired.data, Some(original));
|
||||
} else {
|
||||
let target_path = dirs[slot]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(&target)
|
||||
.join(target_parts[slot].data_dir.expect("target dir").to_string())
|
||||
.join("part.1");
|
||||
let donor_path = dirs[donor_slot]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(&donor)
|
||||
.join(donor_parts[donor_slot].data_dir.expect("donor dir").to_string())
|
||||
.join("part.1");
|
||||
let mut bytes = tokio::fs::read(&target_path).await.expect("target bytes");
|
||||
let donor_bytes = tokio::fs::read(donor_path).await.expect("donor bytes");
|
||||
let frame = 32 + target_parts[slot].erasure.shard_size();
|
||||
bytes[frame..2 * frame].copy_from_slice(&donor_bytes[frame..2 * frame]);
|
||||
tokio::fs::write(target_path, bytes)
|
||||
.await
|
||||
.expect("replace a later complete frame");
|
||||
}
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, &target, None, Default::default(), &options)
|
||||
.await
|
||||
.expect("GET after corruption");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("late corruption must reconstruct without leaking donor bytes");
|
||||
assert_eq!(body, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shard_integrity_legacy_requires_target_digest_not_only_parity_consistency() {
|
||||
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
|
||||
let bucket = "legacy-shard-authority";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let options = ObjectOptions {
|
||||
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let expected = vec![0x4d; 1024 * 1024 + 123];
|
||||
let mut parts = Vec::new();
|
||||
let mut paths = Vec::new();
|
||||
for (object, body) in [("target", expected.clone()), ("donor", vec![0x9a; expected.len()])] {
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(body), &options)
|
||||
.await
|
||||
.expect("seed legacy fixture");
|
||||
let mut object_parts = Vec::new();
|
||||
let mut object_paths = Vec::new();
|
||||
for (slot, disk) in disks.iter().enumerate() {
|
||||
let mut fi = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("source meta");
|
||||
let path = dirs[slot]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(fi.data_dir.expect("data dir").to_string())
|
||||
.join("part.1");
|
||||
let encoded = tokio::fs::read(&path).await.expect("encoded source");
|
||||
let mut legacy = Vec::with_capacity(encoded.len());
|
||||
for frame in encoded.chunks(32 + fi.erasure.shard_size()) {
|
||||
let payload = &frame[32..];
|
||||
legacy.extend_from_slice(rustfs_utils::HashAlgorithm::HighwayHash256S.hash_encode(payload).as_ref());
|
||||
legacy.extend_from_slice(payload);
|
||||
}
|
||||
rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut fi.metadata);
|
||||
for part in &mut fi.parts {
|
||||
part.integrity = None;
|
||||
}
|
||||
tokio::fs::write(&path, legacy).await.expect("legacy framing");
|
||||
disk.write_metadata("", bucket, object, fi.clone())
|
||||
.await
|
||||
.expect("legacy metadata");
|
||||
object_parts.push(fi);
|
||||
object_paths.push(path);
|
||||
}
|
||||
parts.push(object_parts);
|
||||
paths.push(object_paths);
|
||||
}
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, "target", None, Default::default(), &options)
|
||||
.await
|
||||
.expect("healthy legacy object remains readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("legacy body");
|
||||
assert_eq!(body, expected);
|
||||
for (slot, fi) in parts[0].iter().enumerate() {
|
||||
let source = parts[1]
|
||||
.iter()
|
||||
.position(|donor| donor.erasure.index == fi.erasure.index)
|
||||
.expect("coding index");
|
||||
tokio::fs::copy(&paths[1][source], &paths[0][slot])
|
||||
.await
|
||||
.expect("install a complete parity-consistent foreign codeword");
|
||||
}
|
||||
// Compatibility keeps legacy GET semantics. A complete foreign codeword
|
||||
// remains an explicit residual risk until a trusted source rewrites it.
|
||||
let mut legacy = set
|
||||
.get_object_reader(bucket, "target", None, Default::default(), &options)
|
||||
.await
|
||||
.expect("legacy GET remains available");
|
||||
let mut foreign = Vec::new();
|
||||
legacy
|
||||
.stream
|
||||
.read_to_end(&mut foreign)
|
||||
.await
|
||||
.expect("legacy donor is internally consistent");
|
||||
assert_ne!(foreign, expected);
|
||||
for scan_mode in [HealScanMode::Normal, HealScanMode::Deep] {
|
||||
let heal = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
"target",
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let (result, error) = heal.expect("legacy scan");
|
||||
assert!(error.is_none(), "legacy scan remains observable");
|
||||
assert!(!result.integrity_verified, "a foreign codeword must not be certified");
|
||||
assert_eq!(result.drives_healed(), Some(0), "legacy scan must not rewrite data");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shard_integrity_multipart_replacement_and_metadata_copy_preserve_identity() {
|
||||
use crate::set_disk::{CompletePart, RUSTFS_META_MULTIPART_BUCKET, SetDisks};
|
||||
use rustfs_filemeta::ObjectPartInfo;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
|
||||
let bucket = "bound-multipart-copy";
|
||||
let object = "target";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let options = ObjectOptions {
|
||||
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let upload = set.new_multipart_upload(bucket, object, &options).await.expect("new upload");
|
||||
let (upload_meta, _) = set
|
||||
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
|
||||
.await
|
||||
.expect("upload metadata");
|
||||
let upload_path = SetDisks::get_multipart_upload_dir(bucket, object, &upload.upload_id, false);
|
||||
let part_path = dirs[0]
|
||||
.path()
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(upload_path)
|
||||
.join(upload_meta.data_dir.expect("staging directory").to_string())
|
||||
.join("part.2");
|
||||
let first = vec![0x36; 5 * 1024 * 1024];
|
||||
let second = vec![0x69; 1024 * 1024 + 123];
|
||||
let p1 = set
|
||||
.put_object_part(bucket, object, &upload.upload_id, 1, &mut PutObjReader::from_vec(first.clone()), &options)
|
||||
.await
|
||||
.expect("first part");
|
||||
set.put_object_part(
|
||||
bucket,
|
||||
object,
|
||||
&upload.upload_id,
|
||||
2,
|
||||
&mut PutObjReader::from_vec(vec![0xa5; second.len()]),
|
||||
&options,
|
||||
)
|
||||
.await
|
||||
.expect("original second part");
|
||||
let old_frame = tokio::fs::read(&part_path).await.expect("old part shard");
|
||||
let old_meta: ObjectPartInfo = rmp_serde::from_slice(
|
||||
&tokio::fs::read(part_path.with_extension("2.meta"))
|
||||
.await
|
||||
.expect("old part descriptor"),
|
||||
)
|
||||
.expect("decode old part descriptor");
|
||||
let p2 = set
|
||||
.put_object_part(
|
||||
bucket,
|
||||
object,
|
||||
&upload.upload_id,
|
||||
2,
|
||||
&mut PutObjReader::from_vec(second.clone()),
|
||||
&options,
|
||||
)
|
||||
.await
|
||||
.expect("replace second part");
|
||||
let mut completed = set
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&upload.upload_id,
|
||||
vec![
|
||||
CompletePart {
|
||||
part_num: p1.part_num,
|
||||
etag: p1.etag,
|
||||
..Default::default()
|
||||
},
|
||||
CompletePart {
|
||||
part_num: p2.part_num,
|
||||
etag: p2.etag,
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
&options,
|
||||
)
|
||||
.await
|
||||
.expect("complete replacement");
|
||||
let original = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("completed descriptor");
|
||||
assert_ne!(
|
||||
original.parts[1].integrity.as_ref().map(|proof| proof.generation),
|
||||
old_meta.integrity.as_ref().map(|proof| proof.generation)
|
||||
);
|
||||
assert_ne!(
|
||||
original.parts[0].integrity.as_ref().map(|proof| proof.generation),
|
||||
original.parts[1].integrity.as_ref().map(|proof| proof.generation)
|
||||
);
|
||||
let final_path = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(original.data_dir.expect("completed directory").to_string())
|
||||
.join("part.2");
|
||||
assert_eq!(old_frame.len(), tokio::fs::metadata(&final_path).await.unwrap().len() as usize);
|
||||
tokio::fs::write(final_path, old_frame)
|
||||
.await
|
||||
.expect("replay replaced part shard");
|
||||
let (result, error) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("heal replayed part");
|
||||
assert!(error.is_none(), "{error:?}");
|
||||
assert_eq!(result.drives_healed(), Some(1));
|
||||
|
||||
// REPLACE may omit every internal field; metadata-only COPY must keep
|
||||
// the existing payload domain even when it creates another version.
|
||||
completed.metadata_only = true;
|
||||
completed.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"x-amz-meta-label".to_string(),
|
||||
"replacement".to_string(),
|
||||
)]));
|
||||
set.copy_object(bucket, object, bucket, object, &mut completed, &options, &options)
|
||||
.await
|
||||
.expect("metadata replacement copy");
|
||||
let copied = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("copied descriptor");
|
||||
for part in [1, 2] {
|
||||
assert_eq!(copied.parts[part - 1].integrity, original.parts[part - 1].integrity);
|
||||
}
|
||||
let expected = [first, second].concat();
|
||||
for version in [None, original.version_id.map(|id| id.to_string())] {
|
||||
for range in [
|
||||
None,
|
||||
Some(HTTPRangeSpec {
|
||||
start: 5 * 1024 * 1024 - 17,
|
||||
end: 5 * 1024 * 1024 + 99,
|
||||
is_suffix_length: false,
|
||||
}),
|
||||
] {
|
||||
let expected_bytes = if range.is_some() {
|
||||
&expected[5 * 1024 * 1024 - 17..5 * 1024 * 1024 + 100]
|
||||
} else {
|
||||
&expected[..]
|
||||
};
|
||||
let mut reader = set
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
range,
|
||||
Default::default(),
|
||||
&ObjectOptions {
|
||||
version_id: version.clone(),
|
||||
..options.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("multipart GET");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("verified multipart body");
|
||||
assert_eq!(actual, expected_bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn shard_integrity_index_repair_preserves_payload_and_metadata() {
|
||||
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
|
||||
let bucket = "integrity-index-repair";
|
||||
let object = "target";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket");
|
||||
}
|
||||
let expected = vec![0x6a; 2 * 1024 * 1024 + 123];
|
||||
let opts = ObjectOptions {
|
||||
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(expected.clone()), &opts)
|
||||
.await
|
||||
.expect("PUT");
|
||||
let mut snapshots = Vec::new();
|
||||
for (dir, disk) in dirs.iter().zip(&disks) {
|
||||
let info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("metadata");
|
||||
let root = dir.path().join(bucket).join(object);
|
||||
let data = root.join(info.data_dir.expect("data dir").to_string());
|
||||
let proof = data.join(info.parts[0].integrity.as_ref().expect("commitment").file_name());
|
||||
snapshots.push((
|
||||
data.join("part.1"),
|
||||
tokio::fs::read(data.join("part.1")).await.expect("payload"),
|
||||
root.join("xl.meta"),
|
||||
tokio::fs::read(root.join("xl.meta")).await.expect("xl.meta"),
|
||||
proof.clone(),
|
||||
tokio::fs::read(proof).await.expect("index"),
|
||||
));
|
||||
}
|
||||
tokio::fs::remove_file(&snapshots[0].4).await.expect("missing index");
|
||||
let mut bad = snapshots[1].5.clone();
|
||||
bad[70] ^= 1;
|
||||
tokio::fs::write(&snapshots[1].4, bad).await.expect("corrupt index");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, Default::default(), &opts)
|
||||
.await
|
||||
.expect("GET with proof fallback");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("verified fallback body");
|
||||
assert_eq!(body, expected);
|
||||
let (healed, error) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("index repair");
|
||||
assert!(error.is_none(), "{error:?}");
|
||||
assert!(healed.integrity_verified);
|
||||
for ((path, payload, meta_path, meta, proof_path, proof), (disk, dir)) in snapshots.iter().zip(disks.iter().zip(&dirs)) {
|
||||
assert_eq!(tokio::fs::read(path).await.expect("payload"), *payload);
|
||||
assert_eq!(tokio::fs::read(meta_path).await.expect("metadata"), *meta);
|
||||
assert_eq!(tokio::fs::read(proof_path).await.expect("restored proof"), *proof);
|
||||
let relative = proof_path.strip_prefix(dir.path().join(bucket)).expect("relative proof path");
|
||||
disk.delete(bucket, relative.to_str().expect("proof path"), crate::disk::DeleteOptions::default())
|
||||
.await
|
||||
.expect("lose all independent proofs");
|
||||
}
|
||||
// All missing proofs cannot be recreated by trusting the surviving
|
||||
// payload's self-contained bitrot checksum.
|
||||
if let Ok(mut reader) = set.get_object_reader(bucket, object, None, Default::default(), &opts).await {
|
||||
let mut body = Vec::new();
|
||||
assert!(reader.stream.read_to_end(&mut body).await.is_err());
|
||||
}
|
||||
if let Ok((result, error)) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
assert!(error.is_some() || !result.integrity_verified);
|
||||
}
|
||||
for (path, payload, meta_path, meta, proof_path, _) in snapshots {
|
||||
assert_eq!(tokio::fs::read(path).await.expect("retained payload"), payload);
|
||||
assert_eq!(tokio::fs::read(meta_path).await.expect("retained metadata"), meta);
|
||||
assert!(!proof_path.exists(), "no proof minted from unverified payload");
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn shard_integrity_empty_and_no_parity_objects_remain_usable() {
|
||||
for parity in [0, 2] {
|
||||
let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, parity).await;
|
||||
let bucket = "integrity-empty-no-parity";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket");
|
||||
}
|
||||
for size in [0, 1, 1024 * 1024 + 123] {
|
||||
let object = format!("target-{size}");
|
||||
let expected = vec![0x51; size];
|
||||
let opts = ObjectOptions {
|
||||
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
set.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &opts)
|
||||
.await
|
||||
.expect("PUT");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, &object, None, Default::default(), &opts)
|
||||
.await
|
||||
.expect("GET");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("body");
|
||||
assert_eq!(body, expected);
|
||||
let (result, error) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
&object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("Deep scan");
|
||||
assert!(error.is_none(), "{parity} {size}: {error:?}");
|
||||
assert!(result.integrity_verified);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
||||
use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
|
||||
use crate::set_disk::CompletePart;
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
fn rollout_vars(requested: bool, confirmed: bool) -> [(&'static str, Option<&'static str>); 2] {
|
||||
[
|
||||
(rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some(if requested { "true" } else { "false" })),
|
||||
(
|
||||
rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED,
|
||||
Some(if confirmed { "true" } else { "false" }),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_metadata_heal_does_not_certify_unread_tier_payload() {
|
||||
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
|
||||
let bucket = "remote-integrity";
|
||||
let object = "transitioned";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("protected source");
|
||||
for (disk, dir) in disks.iter().zip(&dirs) {
|
||||
let mut info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("source metadata");
|
||||
assert!(info.parts[0].integrity.is_some());
|
||||
info.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_owned();
|
||||
info.transition_tier = "WARM".to_owned();
|
||||
info.transitioned_objname = "remote/transitioned".to_owned();
|
||||
let data_dir = info.data_dir.expect("external shard directory");
|
||||
disk.write_metadata("", bucket, object, info)
|
||||
.await
|
||||
.expect("publish remote metadata");
|
||||
tokio::fs::remove_dir_all(dir.path().join(bucket).join(object).join(data_dir.to_string()))
|
||||
.await
|
||||
.expect("tier transition releases local payload and proof indexes");
|
||||
}
|
||||
let (item, error) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remote metadata scan");
|
||||
assert!(error.is_none(), "{error:?}");
|
||||
assert_eq!(item.drives_healed(), Some(0));
|
||||
assert!(!item.integrity_verified, "local metadata presence cannot certify unread remote bytes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(shard_integrity_rollout)]
|
||||
async fn rollout_requires_both_flags_for_new_puts() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
|
||||
let bucket = "integrity-rollout";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for (requested, confirmed, protected) in [
|
||||
(false, false, false),
|
||||
(true, false, false),
|
||||
(false, true, false),
|
||||
(true, true, true),
|
||||
] {
|
||||
for size in [4096, 1024 * 1024 + 19] {
|
||||
let object = format!("put-{requested}-{confirmed}-{size}");
|
||||
let expected = vec![0x53; size];
|
||||
temp_env::async_with_vars(rollout_vars(requested, confirmed), async {
|
||||
set.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &opts)
|
||||
.await
|
||||
.expect("write at rollout gate");
|
||||
})
|
||||
.await;
|
||||
for disk in &disks {
|
||||
let meta = disk
|
||||
.read_version(
|
||||
"",
|
||||
bucket,
|
||||
&object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("published metadata");
|
||||
assert_eq!(meta.parts[0].integrity.is_some(), protected);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::contains_key_str(
|
||||
&meta.metadata,
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY
|
||||
),
|
||||
protected,
|
||||
"the public gate must control the committed extension"
|
||||
);
|
||||
assert_eq!(meta.erasure.get_checksum_info(1).algorithm, rustfs_utils::HashAlgorithm::HighwayHash256S);
|
||||
}
|
||||
let mut reader = temp_env::async_with_vars(
|
||||
rollout_vars(false, false),
|
||||
set.get_object_reader(bucket, &object, None, Default::default(), &opts),
|
||||
)
|
||||
.await
|
||||
.expect("read after disabling new protection");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("complete exact body");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_shard_repair_and_explicit_version_metadata_recovery_remain_available() {
|
||||
let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
|
||||
let bucket = "legacy-recovery";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
..Default::default()
|
||||
};
|
||||
let expected = vec![0x67; 1024 * 1024 + 37];
|
||||
for case in ["missing-part", "bad-bitrot", "missing-version-metadata"] {
|
||||
let object = case;
|
||||
let info = set
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(expected.clone()), &opts)
|
||||
.await
|
||||
.expect("write legacy fixture");
|
||||
let version = info.version_id.expect("versioned fixture").to_string();
|
||||
let meta = disks[0]
|
||||
.read_version("", bucket, object, &version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("legacy metadata");
|
||||
assert!(meta.parts.iter().all(|part| part.integrity.is_none()));
|
||||
let data_dir = meta.data_dir.expect("external part directory").to_string();
|
||||
let mut original = Vec::new();
|
||||
for dir in &dirs {
|
||||
original.push(
|
||||
tokio::fs::read(dir.path().join(bucket).join(object).join(&data_dir).join("part.1"))
|
||||
.await
|
||||
.expect("original framed shard"),
|
||||
);
|
||||
}
|
||||
match case {
|
||||
"missing-part" => {
|
||||
tokio::fs::remove_file(dirs[0].path().join(bucket).join(object).join(&data_dir).join("part.1"))
|
||||
.await
|
||||
.expect("remove one shard");
|
||||
}
|
||||
"bad-bitrot" => {
|
||||
let mut damaged = original[0].clone();
|
||||
damaged[32] ^= 1;
|
||||
tokio::fs::write(dirs[0].path().join(bucket).join(object).join(&data_dir).join("part.1"), damaged)
|
||||
.await
|
||||
.expect("corrupt one stored block");
|
||||
}
|
||||
_ => {
|
||||
for dir in dirs.iter().skip(1) {
|
||||
tokio::fs::remove_file(dir.path().join(bucket).join(object).join("xl.meta"))
|
||||
.await
|
||||
.expect("retain one consistent version metadata copy");
|
||||
}
|
||||
}
|
||||
}
|
||||
let (result, error) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
&version,
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("legacy recovery must run");
|
||||
assert!(error.is_none(), "{case}: {error:?}");
|
||||
assert!(!result.integrity_verified, "traditional recovery cannot certify original identity");
|
||||
if case != "missing-version-metadata" {
|
||||
assert_eq!(result.drives_healed(), Some(1), "physical repair must remain observable");
|
||||
}
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let restored = disk
|
||||
.read_version("", bucket, object, &version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("metadata must be restored for the explicit version");
|
||||
assert_eq!(restored.version_id, info.version_id);
|
||||
assert!(restored.parts.iter().all(|part| part.integrity.is_none()));
|
||||
let bytes = tokio::fs::read(
|
||||
dirs[index]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(restored.data_dir.expect("restored data directory").to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("restored payload");
|
||||
assert_eq!(bytes, original[index], "recover the original framed bytes on every drive");
|
||||
}
|
||||
let mut reader = set
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
Default::default(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(version),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read recovered historical version");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("complete recovered body");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(shard_integrity_rollout)]
|
||||
async fn multipart_mode_survives_rollout_switch_changes() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
|
||||
let bucket = "multipart-rollout";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
for protected in [false, true] {
|
||||
let object = format!("upload-{protected}");
|
||||
let upload =
|
||||
temp_env::async_with_vars(rollout_vars(protected, protected), set.new_multipart_upload(bucket, &object, &opts))
|
||||
.await
|
||||
.expect("initiate upload");
|
||||
let expected = vec![0x72; 128 * 1024 + 7];
|
||||
let complete = temp_env::async_with_vars(rollout_vars(!protected, !protected), async {
|
||||
let part = set
|
||||
.put_object_part(
|
||||
bucket,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
1,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("upload after switch change");
|
||||
set.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("complete using persisted upload mode")
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
complete.shard_integrity_write_mode(),
|
||||
if protected {
|
||||
ShardIntegrityWriteMode::Protected
|
||||
} else {
|
||||
ShardIntegrityWriteMode::Legacy
|
||||
}
|
||||
);
|
||||
let meta = disks[0]
|
||||
.read_version("", bucket, &object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("complete metadata");
|
||||
assert_eq!(meta.parts[0].integrity.is_some(), protected);
|
||||
assert!(!rustfs_utils::http::contains_key_str(
|
||||
&meta.metadata,
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY
|
||||
));
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, &object, None, Default::default(), &opts)
|
||||
.await
|
||||
.expect("completed upload");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("full multipart body");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(shard_integrity_rollout)]
|
||||
async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
|
||||
let bucket = "rewrite-rollout";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
for protected in [false, true] {
|
||||
let mode = if protected {
|
||||
ShardIntegrityWriteMode::Protected
|
||||
} else {
|
||||
ShardIntegrityWriteMode::Legacy
|
||||
};
|
||||
let object = format!("source-{protected}");
|
||||
let expected = vec![0x39; 1024 * 1024 + 11];
|
||||
let seed_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
..Default::default()
|
||||
};
|
||||
let mut source = set
|
||||
.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &seed_opts)
|
||||
.await
|
||||
.expect("source fixture");
|
||||
source.metadata_only = false;
|
||||
source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let copied = temp_env::async_with_vars(
|
||||
rollout_vars(!protected, !protected),
|
||||
set.copy_object(bucket, &object, bucket, &object, &mut source, &opts, &opts),
|
||||
)
|
||||
.await
|
||||
.expect("materialized self-copy");
|
||||
assert_eq!(
|
||||
copied.shard_integrity_write_mode(),
|
||||
mode,
|
||||
"a legacy rewrite is not a trusted migration and a protected rewrite cannot downgrade"
|
||||
);
|
||||
let meta = disks[0]
|
||||
.read_version("", bucket, &object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("rewritten metadata");
|
||||
assert_eq!(meta.parts[0].integrity.is_some(), protected);
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, &object, None, Default::default(), &opts)
|
||||
.await
|
||||
.expect("read physical rewrite");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("complete rewritten body");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
@@ -1626,17 +1626,20 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
};
|
||||
let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
let (reader, w_size) = match write_path {
|
||||
SmallWritePath::SingleBlockNonInline => {
|
||||
Arc::clone(&erasure)
|
||||
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
|
||||
.await?
|
||||
}
|
||||
SmallWritePath::PipelineBatchedLarge => {
|
||||
Arc::clone(&erasure).encode_batched(stream, &mut writers, write_quorum).await?
|
||||
}
|
||||
SmallWritePath::Inline | SmallWritePath::Pipeline => Arc::clone(&erasure).encode(stream, &mut writers, write_quorum).await?,
|
||||
let upload_suffix = rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY;
|
||||
let protected_upload = rustfs_utils::http::get_consistent_str(&fi.metadata, upload_suffix) == Some("1");
|
||||
if rustfs_utils::http::contains_key_str(&fi.metadata, upload_suffix) && !protected_upload {
|
||||
return Err(DiskError::FileCorrupt.into());
|
||||
}
|
||||
use crate::erasure::coding::encode::IntegrityEncodeMode;
|
||||
let mode = match write_path {
|
||||
SmallWritePath::SingleBlockNonInline => IntegrityEncodeMode::SingleBlock(small_size_hint),
|
||||
SmallWritePath::PipelineBatchedLarge => IntegrityEncodeMode::Batched,
|
||||
SmallWritePath::Inline | SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming,
|
||||
};
|
||||
let (reader, w_size, _, integrity) = Arc::clone(&erasure)
|
||||
.encode_with_shard_integrity(stream, &mut writers, write_quorum, part_id, mode, protected_upload)
|
||||
.await?;
|
||||
|
||||
if let Some(stage_start) = encode_stage_start {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
@@ -1674,6 +1677,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
)))?;
|
||||
}
|
||||
|
||||
let part_integrity = if let Some(integrity) = integrity {
|
||||
Some(integrity.write(&mut shuffle_disks, bucket, RUSTFS_META_TMP_BUCKET, &tmp_part).await?)
|
||||
} else { None };
|
||||
if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
let index_op = data
|
||||
.stream
|
||||
.try_get_index()
|
||||
@@ -1706,6 +1716,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
actual_size,
|
||||
index: index_op,
|
||||
checksums: if checksums.is_empty() { None } else { Some(checksums) },
|
||||
integrity: part_integrity,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -2065,6 +2076,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
|
||||
crate::hp_guard!("SetDisks::new_multipart_upload");
|
||||
let protect_upload = opts.shard_integrity_write_enabled();
|
||||
let storage_class_config = self.storage_class_config_snapshot();
|
||||
let mut _object_lock_guard = None;
|
||||
|
||||
@@ -2086,6 +2098,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let disks = disks.clone();
|
||||
|
||||
let mut user_defined = opts.user_defined.clone();
|
||||
rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut user_defined);
|
||||
if protect_upload {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY,
|
||||
"1".to_owned(),
|
||||
);
|
||||
}
|
||||
rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
|
||||
if !opts.data_movement {
|
||||
rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
|
||||
@@ -2579,6 +2599,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
part.index.clone(),
|
||||
part.checksums.clone(),
|
||||
);
|
||||
let inserted = fi
|
||||
.parts
|
||||
.iter_mut()
|
||||
.find(|entry| entry.number == part.number)
|
||||
.ok_or(Error::FileCorrupt)?;
|
||||
inserted.integrity.clone_from(&part.integrity);
|
||||
}
|
||||
|
||||
let (shuffle_disks, mut parts_metadatas) = Self::shuffle_disks_and_parts_metadata_by_index(&disks, &files_metas, &fi);
|
||||
@@ -2949,6 +2975,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY)
|
||||
&& (rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY)
|
||||
!= Some("1")
|
||||
|| fi.parts.iter().any(|part| part.integrity.is_none()))
|
||||
{
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_UPLOAD_INTEGRITY);
|
||||
fi.persist_shard_integrity()?;
|
||||
|
||||
for meta in parts_metadatas.iter_mut() {
|
||||
if meta.has_valid_erasure_geometry() {
|
||||
meta.size = fi.size;
|
||||
@@ -5355,7 +5391,16 @@ mod tests {
|
||||
upload_path,
|
||||
upload_meta.data_dir.expect("multipart upload should have a data directory")
|
||||
);
|
||||
let retry_meta = Bytes::from_static(b"interrupted retry metadata");
|
||||
let retry_meta = Bytes::from(
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 23,
|
||||
etag: "interrupted-retry".to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
.marshal_msg()
|
||||
.expect("valid legacy retry metadata"),
|
||||
);
|
||||
|
||||
for (index, disk) in disk_stores.iter().enumerate().take(3) {
|
||||
let retry_path = format!("{}/part.1", Uuid::new_v4());
|
||||
@@ -5423,7 +5468,16 @@ mod tests {
|
||||
&src_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&dst_path,
|
||||
Bytes::from_static(b"retry metadata"),
|
||||
Bytes::from(
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 9,
|
||||
etag: "retry".to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
.marshal_msg()
|
||||
.expect("valid legacy part metadata"),
|
||||
),
|
||||
3,
|
||||
None,
|
||||
)
|
||||
|
||||
@@ -3417,6 +3417,7 @@ impl SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
mut publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let protect_write = opts.shard_integrity_write_enabled();
|
||||
if publication_fence.is_none()
|
||||
&& opts.data_movement
|
||||
&& rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
@@ -3461,6 +3462,7 @@ impl SetDisks {
|
||||
|
||||
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
|
||||
let mut user_defined = opts.user_defined.clone();
|
||||
rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut user_defined);
|
||||
if let Some(eval_metadata) = &opts.eval_metadata {
|
||||
merge_evaluated_metadata(&mut user_defined, eval_metadata)?;
|
||||
}
|
||||
@@ -3556,7 +3558,9 @@ impl SetDisks {
|
||||
|
||||
let put_object_size = known_put_object_storage_size(data.size());
|
||||
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
|
||||
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
|
||||
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned)
|
||||
&& put_object_size >= 0
|
||||
&& usize::try_from(put_object_size).is_ok_and(|size| size <= erasure.block_size);
|
||||
|
||||
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
|
||||
let shard_file_size = shard_file_size_raw;
|
||||
@@ -3677,48 +3681,16 @@ impl SetDisks {
|
||||
};
|
||||
|
||||
let encode_stage_start = collect_stage_timing.then(Instant::now);
|
||||
let mut inline_shards = None;
|
||||
let (reader, w_size) = match write_path {
|
||||
SmallWritePath::Inline => match Arc::clone(&erasure)
|
||||
.encode_inline_shards_with_size_hint(stream, small_size_hint)
|
||||
.await
|
||||
{
|
||||
Ok((r, w, shards)) => {
|
||||
inline_shards = Some(shards);
|
||||
(r, w)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("encode_inline_small err {:?}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
},
|
||||
SmallWritePath::SingleBlockNonInline => match Arc::clone(&erasure)
|
||||
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
|
||||
.await
|
||||
{
|
||||
Ok((r, w)) => (r, w),
|
||||
Err(e) => {
|
||||
error!("encode_single_block_non_inline err {:?}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
},
|
||||
SmallWritePath::PipelineBatchedLarge => {
|
||||
match Arc::clone(&erasure).encode_batched(stream, &mut writers, write_quorum).await {
|
||||
Ok((r, w)) => (r, w),
|
||||
Err(e) => {
|
||||
error!("encode_batched err {:?}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
SmallWritePath::Pipeline => match Arc::clone(&erasure).encode(stream, &mut writers, write_quorum).await {
|
||||
Ok((r, w)) => (r, w),
|
||||
Err(e) => {
|
||||
error!("encode err {:?}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
},
|
||||
use crate::erasure::coding::encode::IntegrityEncodeMode;
|
||||
let mode = match write_path {
|
||||
SmallWritePath::Inline => IntegrityEncodeMode::Inline(small_size_hint),
|
||||
SmallWritePath::SingleBlockNonInline => IntegrityEncodeMode::SingleBlock(small_size_hint),
|
||||
SmallWritePath::PipelineBatchedLarge => IntegrityEncodeMode::Batched,
|
||||
SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming,
|
||||
};
|
||||
let (reader, w_size, inline_shards, integrity) = Arc::clone(&erasure)
|
||||
.encode_with_shard_integrity(stream, &mut writers, write_quorum, 1, mode, protect_write)
|
||||
.await?;
|
||||
let encode_elapsed = encode_stage_start.map(|stage_start| stage_start.elapsed());
|
||||
let encode_ms = encode_elapsed.map(|elapsed| elapsed.as_millis() as u64).unwrap_or_default();
|
||||
if let Some(encode_elapsed) = encode_elapsed {
|
||||
@@ -3820,6 +3792,35 @@ impl SetDisks {
|
||||
)));
|
||||
}
|
||||
|
||||
let part_integrity = if let Some(integrity) = integrity {
|
||||
Some(if is_inline_buffer {
|
||||
integrity.set_inline_metadata(&mut fi)?;
|
||||
integrity.part
|
||||
} else {
|
||||
integrity
|
||||
.write(
|
||||
&mut shuffle_disks,
|
||||
bucket,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&format!("{tmp_dir}/{}", fi.data_dir.ok_or(Error::FileCorrupt)?),
|
||||
)
|
||||
.await?
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
if let Some(inline_proof) =
|
||||
rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY)
|
||||
{
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY,
|
||||
inline_proof.to_owned(),
|
||||
);
|
||||
}
|
||||
fi.metadata = user_defined;
|
||||
if fi.version_id.is_none_or(|id| id.is_nil()) && !opts.data_movement && expected_restore_operation_id.is_none() {
|
||||
// Every disk must publish the same cleanup owner alongside a
|
||||
@@ -3832,6 +3833,8 @@ impl SetDisks {
|
||||
fi.size = w_size as i64;
|
||||
fi.versioned = opts.versioned || opts.version_suspended;
|
||||
fi.add_object_part(1, etag, w_size, mod_time, actual_size, index_op, None);
|
||||
fi.parts[0].integrity = part_integrity;
|
||||
fi.persist_shard_integrity()?;
|
||||
if opts.data_movement {
|
||||
fi.set_data_moved();
|
||||
}
|
||||
@@ -7446,7 +7449,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// Self-copy with a data reader: write tier data back locally (de-tiering).
|
||||
// Handles `mc cp --storage-class STANDARD obj obj` on a transitioned object.
|
||||
if let Some(mut put_reader) = src_info.put_object_reader.take() {
|
||||
return self.put_object(dst_bucket, dst_object, &mut put_reader, dst_opts).await;
|
||||
let mut put_opts = dst_opts.clone();
|
||||
put_opts.inherit_shard_integrity(src_info);
|
||||
return self.put_object(dst_bucket, dst_object, &mut put_reader, &put_opts).await;
|
||||
}
|
||||
// Same-key tiered copy without a pre-fetched reader: fall through to the metadata
|
||||
// path so the caller gets a disk/quorum error rather than NotImplemented.
|
||||
@@ -7613,6 +7618,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
None
|
||||
};
|
||||
let mut replacement_metadata = (*src_info.user_defined).clone();
|
||||
rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut replacement_metadata);
|
||||
for suffix in [
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY,
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_INLINE_INTEGRITY,
|
||||
] {
|
||||
if rustfs_utils::http::contains_key_str(&fi.metadata, suffix) {
|
||||
let value = rustfs_utils::http::get_consistent_str(&fi.metadata, suffix).ok_or(Error::FileCorrupt)?;
|
||||
rustfs_utils::http::insert_str(&mut replacement_metadata, suffix, value.to_owned());
|
||||
}
|
||||
}
|
||||
if let Some(part_checksums) = preserved_part_checksums {
|
||||
rustfs_utils::http::insert_str(&mut replacement_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums);
|
||||
}
|
||||
|
||||
@@ -725,6 +725,15 @@ impl SetDisks {
|
||||
skip_verify_bitrot,
|
||||
)
|
||||
.await?;
|
||||
if let Some(expected) = part.integrity.as_ref() {
|
||||
use crate::io_support::shard_integrity::{PartProofReader, ShardVerifier};
|
||||
let proof = PartProofReader::new(expected.clone(), files, disks, bucket, object)?;
|
||||
for (index, reader) in readers.iter_mut().enumerate() {
|
||||
if let Some(reader) = reader {
|
||||
reader.set_integrity(ShardVerifier::new(Arc::clone(&proof), index, 0, None)?)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
let reader_setup_elapsed = reader_setup_stage_start.elapsed();
|
||||
rustfs_io_metrics::record_get_object_shard_reader_setup_duration(reader_setup_elapsed.as_secs_f64());
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
@@ -777,6 +786,7 @@ impl SetDisks {
|
||||
erasure.data_shards,
|
||||
)
|
||||
.await;
|
||||
reader_setup.bind_integrity(part.integrity.as_ref(), &files, &disks, bucket, object, 0)?;
|
||||
let reader_setup_elapsed = reader_setup_stage_start.elapsed();
|
||||
rustfs_io_metrics::record_get_object_shard_reader_setup_duration(reader_setup_elapsed.as_secs_f64());
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
@@ -978,6 +988,7 @@ impl SetDisks {
|
||||
|
||||
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(&disks));
|
||||
let sync_spec = PartReaderSetupSpec {
|
||||
integrity: fi.parts[current_part].integrity.clone(),
|
||||
part_number,
|
||||
read_offset,
|
||||
read_length,
|
||||
@@ -1032,6 +1043,7 @@ impl SetDisks {
|
||||
let next_size = fi.parts[next_part].size;
|
||||
let next_length = next_size.min(remaining_after_current);
|
||||
let spec = PartReaderSetupSpec {
|
||||
integrity: fi.parts[next_part].integrity.clone(),
|
||||
part_number: next_number,
|
||||
read_offset: 0,
|
||||
read_length: erasure.shard_file_offset(0, next_length, next_size),
|
||||
@@ -1657,7 +1669,7 @@ impl SetDisks {
|
||||
});
|
||||
let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(disks));
|
||||
let reader_setup = create_bitrot_readers_until_quorum_with_preference(
|
||||
let mut reader_setup = create_bitrot_readers_until_quorum_with_preference(
|
||||
files,
|
||||
disks,
|
||||
bucket,
|
||||
@@ -1681,6 +1693,12 @@ impl SetDisks {
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let expected = fi
|
||||
.parts
|
||||
.iter()
|
||||
.find(|part| part.number == part_number)
|
||||
.and_then(|part| part.integrity.as_ref());
|
||||
reader_setup.bind_integrity(expected, files, disks, bucket, object, part_offset / erasure.block_size)?;
|
||||
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_SETUP, reader_setup_stage_start);
|
||||
|
||||
let available_shards = reader_setup.available_shards();
|
||||
@@ -1773,6 +1791,7 @@ impl SetDisks {
|
||||
|
||||
/// Per-part parameters for a multipart bitrot reader setup.
|
||||
struct PartReaderSetupSpec {
|
||||
integrity: Option<rustfs_filemeta::shard_integrity::PartIntegrity>,
|
||||
part_number: usize,
|
||||
read_offset: usize,
|
||||
read_length: usize,
|
||||
@@ -1914,7 +1933,7 @@ async fn setup_multipart_part_readers(
|
||||
metrics_size_bucket: &'static str,
|
||||
) -> (BitrotReaderSetup, Duration) {
|
||||
let started = Instant::now();
|
||||
let setup = create_bitrot_readers_until_quorum_with_preference(
|
||||
let mut setup = create_bitrot_readers_until_quorum_with_preference(
|
||||
files,
|
||||
disks,
|
||||
bucket,
|
||||
@@ -1938,6 +1957,13 @@ async fn setup_multipart_part_readers(
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
if setup
|
||||
.bind_integrity(spec.integrity.as_ref(), files, disks, bucket, object, spec.read_offset / shard_size)
|
||||
.is_err()
|
||||
{
|
||||
setup = BitrotReaderSetup::new(disks.len());
|
||||
setup.errors.fill(Some(DiskError::FileCorrupt));
|
||||
}
|
||||
(setup, started.elapsed())
|
||||
}
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ pub(crate) struct StripeReadState {
|
||||
shards: ShardBuffers,
|
||||
errors: ShardErrors,
|
||||
read_quorum: usize,
|
||||
pub(crate) integrity: Option<crate::io_support::shard_integrity::ReconstructionProof>,
|
||||
}
|
||||
|
||||
impl StripeReadState {
|
||||
@@ -70,6 +71,7 @@ impl StripeReadState {
|
||||
shards,
|
||||
errors,
|
||||
read_quorum,
|
||||
integrity: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,6 +80,7 @@ impl StripeReadState {
|
||||
shards: SmallVec::new(),
|
||||
errors: SmallVec::new(),
|
||||
read_quorum,
|
||||
integrity: None,
|
||||
};
|
||||
state.reset(slot_count, read_quorum);
|
||||
state
|
||||
@@ -89,6 +92,7 @@ impl StripeReadState {
|
||||
self.errors.clear();
|
||||
self.errors.resize_with(slot_count, || None);
|
||||
self.read_quorum = read_quorum;
|
||||
self.integrity = None;
|
||||
}
|
||||
|
||||
pub(crate) fn available_shards(&self) -> usize {
|
||||
@@ -124,6 +128,13 @@ impl StripeReadState {
|
||||
&mut self.shards
|
||||
}
|
||||
|
||||
pub(crate) fn verify_reconstructed_integrity(&self) -> std::io::Result<()> {
|
||||
if let Some(proof) = &self.integrity {
|
||||
proof.verify(&self.shards)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn into_parts(self) -> (ShardBuffers, ShardErrors) {
|
||||
(self.shards, self.errors)
|
||||
}
|
||||
|
||||
@@ -4336,6 +4336,7 @@ impl ECStore {
|
||||
let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]);
|
||||
|
||||
let mut dst_opts = dst_opts.clone();
|
||||
dst_opts.inherit_shard_integrity(src_info);
|
||||
if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() {
|
||||
dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?);
|
||||
}
|
||||
@@ -4455,6 +4456,7 @@ impl ECStore {
|
||||
let mut put_opts = ObjectOptions {
|
||||
user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
|
||||
versioned: dst_opts.versioned,
|
||||
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
no_lock: dst_opts.no_lock,
|
||||
mod_time: dst_opts.mod_time,
|
||||
@@ -4493,6 +4495,7 @@ impl ECStore {
|
||||
let mut put_opts = ObjectOptions {
|
||||
user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
|
||||
versioned: dst_opts.versioned,
|
||||
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
no_lock: dst_opts.no_lock,
|
||||
mod_time: dst_opts.mod_time,
|
||||
@@ -4541,6 +4544,7 @@ impl ECStore {
|
||||
let mut put_opts = ObjectOptions {
|
||||
user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
|
||||
versioned: dst_opts.versioned,
|
||||
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
no_lock: dst_opts.no_lock,
|
||||
mod_time: dst_opts.mod_time,
|
||||
|
||||
@@ -51,6 +51,8 @@ thiserror.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
regex.workspace = true
|
||||
arc-swap.workspace = true
|
||||
sha2.workspace = true
|
||||
base64-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
|
||||
@@ -35,7 +35,7 @@ use uuid::Uuid;
|
||||
pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
|
||||
pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
|
||||
|
||||
const MAX_ERASURE_SHARDS: usize = 16;
|
||||
pub(crate) const MAX_ERASURE_SHARDS: usize = 16;
|
||||
const MAX_FILEINFO_PARTS: usize = 10_000;
|
||||
const MAX_FILEINFO_CHECKSUMS: usize = 10_000;
|
||||
const FILEINFO_PART_BITMAP_WORD_BITS: usize = std::mem::size_of::<u64>() * 8;
|
||||
@@ -57,7 +57,7 @@ const ERR_RESTORE_HDR_MALFORMED: &str = "x-amz-restore header malformed";
|
||||
const RFC1123: &[FormatItem<'_>] =
|
||||
format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT");
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
|
||||
#[derive(Deserialize, Debug, PartialEq, Clone, Default)]
|
||||
pub struct ObjectPartInfo {
|
||||
pub etag: String,
|
||||
pub number: usize,
|
||||
@@ -69,6 +69,28 @@ pub struct ObjectPartInfo {
|
||||
// Checksums holds checksums of the part
|
||||
pub checksums: Option<HashMap<String, String>>,
|
||||
pub error: Option<String>,
|
||||
#[serde(default)]
|
||||
pub integrity: Option<crate::shard_integrity::PartIntegrity>,
|
||||
}
|
||||
|
||||
impl Serialize for ObjectPartInfo {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
|
||||
// Named fields let release readers ignore optional extensions. Appending
|
||||
// a ninth element to the old positional array makes those readers fail.
|
||||
let mut map = serializer.serialize_map(Some(8 + usize::from(self.integrity.is_some())))?;
|
||||
map.serialize_entry("etag", &self.etag)?;
|
||||
map.serialize_entry("number", &self.number)?;
|
||||
map.serialize_entry("size", &self.size)?;
|
||||
map.serialize_entry("actual_size", &self.actual_size)?;
|
||||
map.serialize_entry("mod_time", &self.mod_time)?;
|
||||
map.serialize_entry("index", &self.index)?;
|
||||
map.serialize_entry("checksums", &self.checksums)?;
|
||||
map.serialize_entry("error", &self.error)?;
|
||||
if let Some(integrity) = &self.integrity {
|
||||
map.serialize_entry("integrity", integrity)?;
|
||||
}
|
||||
map.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectPartInfo {
|
||||
@@ -639,7 +661,11 @@ impl<'de> Deserialize<'de> for FileInfo {
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_struct("FileInfo", FILE_INFO_FIELDS, FileInfoVisitor)
|
||||
let mut file = deserializer.deserialize_struct("FileInfo", FILE_INFO_FIELDS, FileInfoVisitor)?;
|
||||
if !file.parts.is_empty() {
|
||||
file.hydrate_shard_integrity().map_err(de::Error::custom)?;
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1090,6 +1116,7 @@ impl FileInfo {
|
||||
index,
|
||||
checksums,
|
||||
error: None,
|
||||
integrity: None,
|
||||
};
|
||||
|
||||
for p in self.parts.iter_mut() {
|
||||
@@ -2172,6 +2199,7 @@ mod tests {
|
||||
index,
|
||||
checksums,
|
||||
error,
|
||||
integrity: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2379,6 +2407,7 @@ mod tests {
|
||||
.collect(),
|
||||
),
|
||||
error: Some("part-error".to_string()),
|
||||
integrity: None,
|
||||
}],
|
||||
erasure: ErasureInfo {
|
||||
algorithm: "erasure-algorithm".to_string(),
|
||||
|
||||
@@ -2034,6 +2034,7 @@ impl From<MetaObjectV1Part> for ObjectPartInfo {
|
||||
index: value.index,
|
||||
checksums: value.checksums,
|
||||
error: value.error,
|
||||
integrity: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2711,6 +2712,33 @@ impl MetaObject {
|
||||
if all_parts && include_part_checksums {
|
||||
file_info.hydrate_data_movement_part_checksums()?;
|
||||
}
|
||||
if !self.part_numbers.is_empty()
|
||||
&& rustfs_utils::http::contains_key_str(&file_info.metadata, crate::shard_integrity::SUFFIX_UPLOAD_INTEGRITY)
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
if let Some(commitments) = crate::shard_integrity::descriptor_from_metadata(&file_info.metadata)? {
|
||||
let layout = crate::shard_integrity::IntegrityLayout::new(
|
||||
self.erasure_m,
|
||||
self.erasure_n,
|
||||
self.erasure_block_size,
|
||||
file_info.uses_legacy_checksum,
|
||||
)?;
|
||||
if commitments.len() != self.part_numbers.len() || commitments.len() != self.part_sizes.len() {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
for (i, commitment) in commitments.into_iter().enumerate() {
|
||||
if commitment.layout != layout
|
||||
|| usize::try_from(commitment.number).map_err(|_| Error::FileCorrupt)? != self.part_numbers[i]
|
||||
|| usize::try_from(commitment.size).map_err(|_| Error::FileCorrupt)? != self.part_sizes[i]
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
if all_parts {
|
||||
file_info.parts[i].integrity = Some(commitment);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(file_info)
|
||||
}
|
||||
|
||||
|
||||
@@ -28,3 +28,4 @@ pub use filemeta::*;
|
||||
pub use filemeta_inline::*;
|
||||
pub use metacache::*;
|
||||
pub use replication::*;
|
||||
pub mod shard_integrity;
|
||||
|
||||
@@ -0,0 +1,517 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Optional, quorum-owned SHA-256 commitments. The existing part payload and
|
||||
//! HighwayHash framing are unchanged. A proof file is only an untrusted index;
|
||||
//! authority comes from the root stored in the selected object's metadata.
|
||||
|
||||
use crate::{Error, FileInfo, Result};
|
||||
use rustfs_utils::http::{contains_key_str, get_consistent_str, insert_str, remove_str};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const SUFFIX_SHARD_INTEGRITY: &str = "shard-integrity-v1";
|
||||
pub const SUFFIX_INLINE_INTEGRITY: &str = "shard-integrity-inline-v1";
|
||||
pub const SUFFIX_UPLOAD_INTEGRITY: &str = "shard-integrity-upload-v1";
|
||||
pub const MAX_PARTS: usize = 10_000;
|
||||
pub const MAX_DESCRIPTOR_BYTES: usize = 1024 * 1024;
|
||||
pub const MAX_INLINE_PROOF_BYTES: usize = 64 * 1024;
|
||||
pub const INDEX_HEADER_SIZE: usize = 64;
|
||||
const TABLE_HEADER_SIZE: usize = 32;
|
||||
const PART_RECORD_SIZE: usize = 64;
|
||||
const TABLE_MAGIC: &[u8; 8] = b"RFSI\x01\0\0\0";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct IntegrityLayout {
|
||||
pub data: u16,
|
||||
pub parity: u16,
|
||||
pub block_size: u32,
|
||||
pub legacy: bool,
|
||||
}
|
||||
|
||||
impl IntegrityLayout {
|
||||
pub fn new(data: usize, parity: usize, block_size: usize, legacy: bool) -> Result<Self> {
|
||||
let layout = Self {
|
||||
data: data.try_into().map_err(|_| Error::FileCorrupt)?,
|
||||
parity: parity.try_into().map_err(|_| Error::FileCorrupt)?,
|
||||
block_size: block_size.try_into().map_err(|_| Error::FileCorrupt)?,
|
||||
legacy,
|
||||
};
|
||||
layout.validate()?;
|
||||
Ok(layout)
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.data == 0
|
||||
|| self.data < self.parity
|
||||
|| self.shards() > crate::fileinfo::MAX_ERASURE_SHARDS
|
||||
|| self.block_size == 0
|
||||
|| self.block_size > 64 * 1024 * 1024
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn shards(&self) -> usize {
|
||||
usize::from(self.data) + usize::from(self.parity)
|
||||
}
|
||||
|
||||
fn bytes(&self) -> [u8; 9] {
|
||||
let mut out = [0; 9];
|
||||
out[..2].copy_from_slice(&self.data.to_le_bytes());
|
||||
out[2..4].copy_from_slice(&self.parity.to_le_bytes());
|
||||
out[4..8].copy_from_slice(&self.block_size.to_le_bytes());
|
||||
out[8] = u8::from(self.legacy);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// One immutable UploadPart/PUT generation. A metadata-only copy retains this
|
||||
/// identity; overwriting even an identical part creates a new generation.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct PartIntegrity {
|
||||
pub layout: IntegrityLayout,
|
||||
pub number: u32,
|
||||
pub generation: Uuid,
|
||||
pub size: u64,
|
||||
pub stripes: u32,
|
||||
pub root: [u8; 32],
|
||||
}
|
||||
|
||||
impl PartIntegrity {
|
||||
pub fn new(layout: IntegrityLayout, number: usize) -> Result<Self> {
|
||||
let part = Self {
|
||||
layout,
|
||||
number: number.try_into().map_err(|_| Error::FileCorrupt)?,
|
||||
generation: Uuid::new_v4(),
|
||||
size: 0,
|
||||
stripes: 0,
|
||||
root: [0; 32],
|
||||
};
|
||||
part.validate()?;
|
||||
Ok(part)
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
self.layout.validate()?;
|
||||
if self.number == 0
|
||||
|| usize::try_from(self.number).map_err(|_| Error::FileCorrupt)? > MAX_PARTS
|
||||
|| self.generation.is_nil()
|
||||
|| self.size > i64::MAX.unsigned_abs()
|
||||
|| self.size.div_ceil(u64::from(self.layout.block_size)) != u64::from(self.stripes)
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn file_name(&self) -> String {
|
||||
format!("part.{}.integrity.{}", self.number, self.generation)
|
||||
}
|
||||
|
||||
pub fn height(&self) -> u32 {
|
||||
u32::BITS - self.stripes.max(1).saturating_sub(1).leading_zeros()
|
||||
}
|
||||
|
||||
pub fn record_size(&self) -> usize {
|
||||
// At most 16 digests and 32 siblings, independent of untrusted sizes.
|
||||
32 * (self.layout.shards() + usize::try_from(self.height()).unwrap_or(32))
|
||||
}
|
||||
|
||||
pub fn index_size(&self) -> Result<usize> {
|
||||
usize::try_from(self.stripes)
|
||||
.ok()
|
||||
.and_then(|n| n.checked_mul(self.record_size()))
|
||||
.and_then(|n| n.checked_add(INDEX_HEADER_SIZE))
|
||||
.ok_or(Error::FileCorrupt)
|
||||
}
|
||||
|
||||
pub fn record_offset(&self, stripe: u32) -> Result<usize> {
|
||||
if stripe >= self.stripes {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
usize::try_from(stripe)
|
||||
.ok()
|
||||
.and_then(|n| n.checked_mul(self.record_size()))
|
||||
.and_then(|n| n.checked_add(INDEX_HEADER_SIZE))
|
||||
.ok_or(Error::FileCorrupt)
|
||||
}
|
||||
|
||||
pub fn index_header(&self) -> [u8; INDEX_HEADER_SIZE] {
|
||||
let mut out = [0; INDEX_HEADER_SIZE];
|
||||
out[..8].copy_from_slice(b"RFSP\x01\0\0\0");
|
||||
out[8..17].copy_from_slice(&self.layout.bytes());
|
||||
out[20..24].copy_from_slice(&self.number.to_le_bytes());
|
||||
out[24..40].copy_from_slice(self.generation.as_bytes());
|
||||
out[40..48].copy_from_slice(&self.size.to_le_bytes());
|
||||
out[48..52].copy_from_slice(&self.stripes.to_le_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
fn hash_context(&self, domain: &[u8]) -> Sha256 {
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(domain);
|
||||
hash.update(self.layout.bytes());
|
||||
hash.update(self.number.to_le_bytes());
|
||||
hash.update(self.generation.as_bytes());
|
||||
hash
|
||||
}
|
||||
|
||||
pub fn shard_digest(&self, stripe: u32, coding_index: usize, payload: &[u8]) -> Result<[u8; 32]> {
|
||||
if coding_index >= self.layout.shards() {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/payload\0");
|
||||
hash.update(stripe.to_le_bytes());
|
||||
hash.update(u16::try_from(coding_index).map_err(|_| Error::FileCorrupt)?.to_le_bytes());
|
||||
hash.update(u64::try_from(payload.len()).map_err(|_| Error::FileCorrupt)?.to_le_bytes());
|
||||
hash.update(payload);
|
||||
Ok(hash.finalize().into())
|
||||
}
|
||||
|
||||
pub fn leaf_digest(&self, stripe: u32, digests: &[u8]) -> Result<[u8; 32]> {
|
||||
if digests.len() != self.layout.shards() * 32 {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/leaf\0");
|
||||
hash.update(stripe.to_le_bytes());
|
||||
hash.update(digests);
|
||||
Ok(hash.finalize().into())
|
||||
}
|
||||
|
||||
pub fn padding_digest(&self, stripe: u32) -> [u8; 32] {
|
||||
let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/padding\0");
|
||||
hash.update(stripe.to_le_bytes());
|
||||
hash.finalize().into()
|
||||
}
|
||||
|
||||
pub fn root_digest(&self, tree_root: &[u8; 32]) -> [u8; 32] {
|
||||
let mut hash = self.hash_context(b"rustfs/shard-integrity/v1/root\0");
|
||||
hash.update(self.size.to_le_bytes());
|
||||
hash.update(self.stripes.to_le_bytes());
|
||||
hash.update(tree_root);
|
||||
hash.finalize().into()
|
||||
}
|
||||
|
||||
pub fn verify_record(&self, stripe: u32, record: &[u8]) -> Result<()> {
|
||||
self.record_offset(stripe)?;
|
||||
if record.len() != self.record_size() {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let (digests, siblings) = record.split_at(self.layout.shards() * 32);
|
||||
let mut digest = self.leaf_digest(stripe, digests)?;
|
||||
let mut position = stripe;
|
||||
for sibling in siblings.as_chunks::<32>().0 {
|
||||
digest = if position & 1 == 0 {
|
||||
node_digest(&digest, sibling)
|
||||
} else {
|
||||
node_digest(sibling, &digest)
|
||||
};
|
||||
position >>= 1;
|
||||
}
|
||||
if self.root_digest(&digest) != self.root {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn verify_shard(&self, stripe: u32, coding_index: usize, payload: &[u8], record: &[u8]) -> Result<()> {
|
||||
self.verify_record(stripe, record)?;
|
||||
let digest = self.shard_digest(stripe, coding_index, payload)?;
|
||||
if record.get(coding_index * 32..(coding_index + 1) * 32) != Some(digest.as_slice()) {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn node_digest(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] {
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(b"rustfs/shard-integrity/v1/node\0");
|
||||
hash.update(left);
|
||||
hash.update(right);
|
||||
hash.finalize().into()
|
||||
}
|
||||
|
||||
pub fn encode_descriptor(parts: &[PartIntegrity]) -> Result<String> {
|
||||
let Some(first) = parts.first() else { return Err(Error::FileCorrupt) };
|
||||
if parts.len() > MAX_PARTS {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let mut out = vec![0; TABLE_HEADER_SIZE];
|
||||
out[..8].copy_from_slice(TABLE_MAGIC);
|
||||
out[8..17].copy_from_slice(&first.layout.bytes());
|
||||
out[28..32].copy_from_slice(&u32::try_from(parts.len()).map_err(|_| Error::FileCorrupt)?.to_le_bytes());
|
||||
let mut previous = 0;
|
||||
for part in parts {
|
||||
part.validate()?;
|
||||
if part.layout != first.layout || part.number <= previous {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
previous = part.number;
|
||||
out.extend_from_slice(&part.number.to_le_bytes());
|
||||
out.extend_from_slice(part.generation.as_bytes());
|
||||
out.extend_from_slice(&part.size.to_le_bytes());
|
||||
out.extend_from_slice(&part.stripes.to_le_bytes());
|
||||
out.extend_from_slice(&part.root);
|
||||
}
|
||||
Ok(base64_simd::STANDARD.encode_to_string(out))
|
||||
}
|
||||
|
||||
pub fn decode_descriptor(value: &str) -> Result<Vec<PartIntegrity>> {
|
||||
if value.len() > MAX_DESCRIPTOR_BYTES {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let bytes = base64_simd::STANDARD.decode_to_vec(value).map_err(|_| Error::FileCorrupt)?;
|
||||
// Reject non-canonical encodings as well as unsupported versions/reserved bits.
|
||||
if bytes.len() < TABLE_HEADER_SIZE
|
||||
|| &bytes[..8] != TABLE_MAGIC
|
||||
|| bytes[16] > 1
|
||||
|| bytes[17..28].iter().any(|v| *v != 0)
|
||||
|| base64_simd::STANDARD.encode_to_string(&bytes) != value
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let u32_at = |start| -> Result<u32> {
|
||||
Ok(u32::from_le_bytes(
|
||||
bytes
|
||||
.get(start..start + 4)
|
||||
.ok_or(Error::FileCorrupt)?
|
||||
.try_into()
|
||||
.map_err(|_| Error::FileCorrupt)?,
|
||||
))
|
||||
};
|
||||
let count = usize::try_from(u32_at(28)?).map_err(|_| Error::FileCorrupt)?;
|
||||
if count == 0 || count > MAX_PARTS || bytes.len() != TABLE_HEADER_SIZE + count * PART_RECORD_SIZE {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let layout = IntegrityLayout {
|
||||
data: u16::from_le_bytes([bytes[8], bytes[9]]),
|
||||
parity: u16::from_le_bytes([bytes[10], bytes[11]]),
|
||||
block_size: u32_at(12)?,
|
||||
legacy: bytes[16] == 1,
|
||||
};
|
||||
layout.validate()?;
|
||||
let mut parts = Vec::with_capacity(count);
|
||||
let mut previous = 0;
|
||||
for record in bytes[TABLE_HEADER_SIZE..].as_chunks::<PART_RECORD_SIZE>().0 {
|
||||
let part = PartIntegrity {
|
||||
layout,
|
||||
number: u32::from_le_bytes(record[..4].try_into().map_err(|_| Error::FileCorrupt)?),
|
||||
generation: Uuid::from_slice(&record[4..20])
|
||||
.ok()
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(Error::FileCorrupt)?,
|
||||
size: u64::from_le_bytes(record[20..28].try_into().map_err(|_| Error::FileCorrupt)?),
|
||||
stripes: u32::from_le_bytes(record[28..32].try_into().map_err(|_| Error::FileCorrupt)?),
|
||||
root: record[32..64].try_into().map_err(|_| Error::FileCorrupt)?,
|
||||
};
|
||||
part.validate()?;
|
||||
if part.number <= previous {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
previous = part.number;
|
||||
parts.push(part);
|
||||
}
|
||||
Ok(parts)
|
||||
}
|
||||
|
||||
pub fn descriptor_from_metadata(metadata: &HashMap<String, String>) -> Result<Option<Vec<PartIntegrity>>> {
|
||||
if !contains_key_str(metadata, SUFFIX_SHARD_INTEGRITY) {
|
||||
// An orphaned inline proof must not silently turn a protected object into legacy.
|
||||
if contains_key_str(metadata, SUFFIX_INLINE_INTEGRITY) {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
decode_descriptor(get_consistent_str(metadata, SUFFIX_SHARD_INTEGRITY).ok_or(Error::FileCorrupt)?).map(Some)
|
||||
}
|
||||
|
||||
pub fn clear_integrity_metadata(metadata: &mut HashMap<String, String>) {
|
||||
for suffix in [SUFFIX_SHARD_INTEGRITY, SUFFIX_INLINE_INTEGRITY, SUFFIX_UPLOAD_INTEGRITY] {
|
||||
remove_str(metadata, suffix);
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInfo {
|
||||
/// Decode once at the metadata boundary and retain each part's validated
|
||||
/// commitment. Reading part n must not reparse a 10,000-part descriptor.
|
||||
pub fn hydrate_shard_integrity(&mut self) -> Result<()> {
|
||||
if !self.parts.is_empty() && contains_key_str(&self.metadata, SUFFIX_UPLOAD_INTEGRITY) {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let Some(parts) = descriptor_from_metadata(&self.metadata)? else {
|
||||
if self.parts.iter().any(|part| part.integrity.is_some()) {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
let layout = IntegrityLayout::new(
|
||||
self.erasure.data_blocks,
|
||||
self.erasure.parity_blocks,
|
||||
self.erasure.block_size,
|
||||
self.uses_legacy_checksum,
|
||||
)?;
|
||||
if parts.len() != self.parts.len() {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
for (expected, actual) in parts.into_iter().zip(&mut self.parts) {
|
||||
if expected.layout != layout
|
||||
|| usize::try_from(expected.number).map_err(|_| Error::FileCorrupt)? != actual.number
|
||||
|| usize::try_from(expected.size).map_err(|_| Error::FileCorrupt)? != actual.size
|
||||
|| actual.integrity.as_ref().is_some_and(|value| value != &expected)
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
actual.integrity = Some(expected);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn persist_shard_integrity(&mut self) -> Result<()> {
|
||||
let parts: Vec<_> = self.parts.iter().filter_map(|part| part.integrity.clone()).collect();
|
||||
if parts.is_empty() {
|
||||
clear_integrity_metadata(&mut self.metadata);
|
||||
return Ok(());
|
||||
}
|
||||
if parts.len() != self.parts.len() {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
insert_str(&mut self.metadata, SUFFIX_SHARD_INTEGRITY, encode_descriptor(&parts)?);
|
||||
self.hydrate_shard_integrity()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ObjectPartInfo;
|
||||
|
||||
fn part(number: usize) -> PartIntegrity {
|
||||
let mut part =
|
||||
PartIntegrity::new(IntegrityLayout::new(12, 4, 1024 * 1024, false).expect("layout"), number).expect("part");
|
||||
part.size = 1024 * 1024 + 123;
|
||||
part.stripes = 2;
|
||||
part.root = [7; 32];
|
||||
part
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integrity_descriptor_is_bounded_canonical_and_complete() {
|
||||
let parts: Vec<_> = (1..=MAX_PARTS).map(part).collect();
|
||||
let encoded = encode_descriptor(&parts).expect("maximum multipart descriptor");
|
||||
assert_eq!(encoded.len(), 853_376);
|
||||
assert_eq!(decode_descriptor(&encoded).expect("decode maximum table"), parts);
|
||||
assert!(encode_descriptor(&[part(2), part(1)]).is_err());
|
||||
assert!(encode_descriptor(&[part(1), part(1)]).is_err());
|
||||
assert!(decode_descriptor(&format!("{encoded} ")).is_err());
|
||||
assert!(decode_descriptor(&"A".repeat(MAX_DESCRIPTOR_BYTES + 1)).is_err());
|
||||
let mut bytes = base64_simd::STANDARD
|
||||
.decode_to_vec(encode_descriptor(&[part(1)]).expect("encode"))
|
||||
.expect("base64");
|
||||
for offset in [4, 17, 28, 36] {
|
||||
let mut invalid = bytes.clone();
|
||||
if offset == 36 {
|
||||
invalid[36..52].fill(0);
|
||||
} else {
|
||||
invalid[offset] = 255;
|
||||
}
|
||||
assert!(
|
||||
decode_descriptor(&base64_simd::STANDARD.encode_to_string(invalid)).is_err(),
|
||||
"offset {offset}"
|
||||
);
|
||||
}
|
||||
bytes.push(0);
|
||||
assert!(decode_descriptor(&base64_simd::STANDARD.encode_to_string(bytes)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integrity_metadata_distinguishes_absence_corruption_and_part_mismatch() {
|
||||
let mut file = FileInfo::new("object", 12, 4);
|
||||
file.add_object_part(1, String::new(), 1024 * 1024 + 123, None, 0, None, None);
|
||||
assert!(descriptor_from_metadata(&file.metadata).expect("legacy absence").is_none());
|
||||
file.parts[0].integrity = Some(part(1));
|
||||
file.persist_shard_integrity().expect("persist");
|
||||
let mut decoded = file.clone();
|
||||
decoded.parts[0].integrity = None;
|
||||
decoded.hydrate_shard_integrity().expect("hydrate old peer response");
|
||||
assert_eq!(decoded.parts[0].integrity, file.parts[0].integrity);
|
||||
decoded.parts[0].size += 1;
|
||||
assert!(decoded.hydrate_shard_integrity().is_err());
|
||||
file.metadata
|
||||
.insert(format!("x-minio-internal-{SUFFIX_SHARD_INTEGRITY}"), "invalid".to_owned());
|
||||
assert!(descriptor_from_metadata(&file.metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integrity_descriptor_matches_independent_v1_vector() {
|
||||
// Header, UUID byte order, dimensions and root frozen independently
|
||||
// with Python struct/hashlib, rather than a Rust round trip.
|
||||
let encoded = "UkZTSQEAAAACAAIACAAAAAAAAAAAAAAAAAAAAAEAAAABAAAAABEiM0RVRneImaq7zN3u/xAAAAAAAAAAAgAAACH+n+LW6++vtoSsUQbqfbsZYr4G+/GrIZa8s/oaPC/G";
|
||||
let parts = decode_descriptor(encoded).expect("known descriptor");
|
||||
assert_eq!(parts.len(), 1);
|
||||
let part = &parts[0];
|
||||
assert_eq!(part.layout, IntegrityLayout::new(2, 2, 8, false).expect("layout"));
|
||||
assert_eq!(part.generation.to_string(), "00112233-4455-4677-8899-aabbccddeeff");
|
||||
assert_eq!((part.number, part.size, part.stripes), (1, 16, 2));
|
||||
assert_eq!(
|
||||
part.root,
|
||||
[
|
||||
33, 254, 159, 226, 214, 235, 239, 175, 182, 132, 172, 81, 6, 234, 125, 187, 25, 98, 190, 6, 251, 241, 171, 33,
|
||||
150, 188, 179, 250, 26, 60, 47, 198
|
||||
]
|
||||
);
|
||||
assert_eq!(encode_descriptor(&parts).expect("encode known descriptor"), encoded);
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct ReleasePart {
|
||||
etag: String,
|
||||
number: usize,
|
||||
size: usize,
|
||||
actual_size: i64,
|
||||
mod_time: Option<time::OffsetDateTime>,
|
||||
index: Option<bytes::Bytes>,
|
||||
checksums: Option<HashMap<String, String>>,
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integrity_part_extension_is_readable_by_release_decoder_and_reads_old_arrays() {
|
||||
let old = ReleasePart {
|
||||
etag: "same-etag".to_owned(),
|
||||
number: 1,
|
||||
size: 5,
|
||||
actual_size: 5,
|
||||
mod_time: None,
|
||||
index: None,
|
||||
checksums: None,
|
||||
error: None,
|
||||
};
|
||||
let tuple = rmp_serde::to_vec(&old).expect("release positional writer");
|
||||
assert_eq!(tuple[0], 0x98, "release wrote eight positional fields");
|
||||
let mut new = ObjectPartInfo::unmarshal(&tuple).expect("read release part");
|
||||
assert!(new.integrity.is_none());
|
||||
new.integrity = Some(part(1));
|
||||
let bytes = new.marshal_msg().expect("new writer");
|
||||
assert_eq!(bytes[0], 0x89, "extension uses nine named fields");
|
||||
let decoded: ReleasePart = rmp_serde::from_slice(&bytes).expect("release decoder ignores unknown named field");
|
||||
assert_eq!(decoded.etag, old.etag);
|
||||
assert_eq!(decoded.size, 5);
|
||||
assert_eq!(ObjectPartInfo::unmarshal(&bytes).expect("new decoder").integrity, new.integrity);
|
||||
}
|
||||
}
|
||||
@@ -456,7 +456,7 @@ fn warn_mrf_journal_write(err: &super::DiskError) {
|
||||
|
||||
/// Translate an intent into the prioritized heal request the issue specifies:
|
||||
/// decode failures go Urgent ECDecode, metadata corruption goes High
|
||||
/// Metadata, partial writes go Normal object heal.
|
||||
/// Metadata, partial writes go Normal-priority object heal with Deep verification.
|
||||
pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
|
||||
let bucket = intent.bucket.to_string();
|
||||
let object = intent.object.to_string();
|
||||
@@ -484,6 +484,11 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
|
||||
),
|
||||
};
|
||||
let mut options = HealOptions::default();
|
||||
if matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::PartialWrite) {
|
||||
// Presence-only repair cannot discharge a protected object's durable
|
||||
// obligation. Verify payloads even after new protection is disabled.
|
||||
options.scan_mode = rustfs_heal_contracts::heal_channel::HealScanMode::Deep;
|
||||
}
|
||||
if !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption)
|
||||
&& let Some(scope) = intent.scope
|
||||
{
|
||||
@@ -2716,5 +2721,6 @@ mod tests {
|
||||
});
|
||||
assert!(matches!(partial.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(partial.priority, HealPriority::Normal);
|
||||
assert_eq!(partial.options.scan_mode, rustfs_heal_contracts::heal_channel::HealScanMode::Deep);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1169,7 +1169,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else if error.is_none() && !opts.dry_run {
|
||||
} else if error.is_none() && !opts.dry_run && item.integrity_verified {
|
||||
let ok_drive_state = DriveState::Ok.to_string();
|
||||
let all_after_drives_ok = item.after.drives.iter().all(|drive| drive.state == ok_drive_state);
|
||||
match (
|
||||
|
||||
@@ -762,10 +762,12 @@ mod absence_receipt_regressions {
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 3);
|
||||
assert_eq!(outcome.counters.healed, 1, "completed cleanup must be repaired: {outcome:?}");
|
||||
assert_eq!(outcome.counters.unchanged, 2);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
// Exact historical absence has its own proof. The two live legacy
|
||||
// versions remain readable but carry no independent payload receipt.
|
||||
assert_eq!(outcome.counters.unchanged, 0);
|
||||
assert_eq!(outcome.counters.unknown, 2);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert_eq!(outcome.counters.skipped, 0);
|
||||
assert_eq!(outcome.counters.skipped, 2);
|
||||
assert_versions(&store, bucket, &old, ¤t).await;
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use http::HeaderMap;
|
||||
use rustfs_heal::heal::{
|
||||
outcome::{HealObjectDisposition, HealTraversalCoverage},
|
||||
@@ -24,7 +26,8 @@ use tokio::io::AsyncReadExt as _;
|
||||
|
||||
mod storage_api;
|
||||
use storage_api::integration::{
|
||||
DiskAPI as _, DiskError, DiskOption, Endpoint, NamespaceLocking as _, ObjectIO as _, ReadOptions, new_disk,
|
||||
DiskAPI as _, DiskError, DiskOption, Endpoint, NamespaceLocking as _, ObjectIO as _, ReadOptions, ShardIntegrityWriteMode,
|
||||
new_disk,
|
||||
};
|
||||
|
||||
fn deep_heal_task(storage: &Arc<ECStoreHealStorage>, bucket: &str, object: &str, dry_run: bool) -> HealTask {
|
||||
@@ -75,6 +78,7 @@ async fn deep_heal_truncated_xlmeta_scenario() {
|
||||
let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
let bucket = "truncated-xlmeta";
|
||||
env.make_bucket(bucket, false).await;
|
||||
env.make_bucket("truncated-xlmeta-versioned", true).await;
|
||||
let payload = vec![0x7b; 4 * 1024 * 1024];
|
||||
let mut endpoint = Endpoint::try_from(env.disk_paths[0].to_str().expect("UTF-8 disk path")).expect("target endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
@@ -88,13 +92,13 @@ async fn deep_heal_truncated_xlmeta_scenario() {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for damage in ["length-prefix", "metadata-body", "crc-tail", "versioned"] {
|
||||
for (damage, protected) in ["length-prefix", "metadata-body", "crc-tail", "versioned"]
|
||||
.into_iter()
|
||||
.flat_map(|damage| [false, true].map(|protected| (damage, protected)))
|
||||
{
|
||||
let versioned = damage == "versioned";
|
||||
let bucket = if versioned { "truncated-xlmeta-versioned" } else { bucket };
|
||||
if versioned {
|
||||
env.make_bucket(bucket, true).await;
|
||||
}
|
||||
let object = format!("{damage}/object.bin");
|
||||
let object = format!("{damage}-{protected}/object.bin");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
env.ecstore
|
||||
.put_object(
|
||||
@@ -103,6 +107,11 @@ async fn deep_heal_truncated_xlmeta_scenario() {
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
shard_integrity_write_mode: Some(if protected {
|
||||
ShardIntegrityWriteMode::Protected
|
||||
} else {
|
||||
ShardIntegrityWriteMode::Legacy
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -168,15 +177,26 @@ async fn deep_heal_truncated_xlmeta_scenario() {
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(outcome.counters.healed, 1, "{damage}: {outcome:?}");
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.counters.skipped, 0);
|
||||
// Both modes restore metadata and shards. Only independently verified
|
||||
// payloads authorize a strong repair receipt.
|
||||
assert_eq!(outcome.counters.healed, u64::from(protected), "{damage}: {outcome:?}");
|
||||
assert_eq!(outcome.counters.unknown, u64::from(!protected));
|
||||
assert_eq!(outcome.counters.skipped, u64::from(!protected));
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert_eq!(outcome.counters.attempt_failures, 0);
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Repaired);
|
||||
assert_eq!(
|
||||
outcome.objects[0].disposition,
|
||||
if protected {
|
||||
HealObjectDisposition::Repaired
|
||||
} else {
|
||||
HealObjectDisposition::Unknown
|
||||
}
|
||||
);
|
||||
assert_eq!(read_error, DiskError::FileCorrupt);
|
||||
let results = task.get_result_items().await;
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].drives_healed(), Some(1));
|
||||
assert_eq!(results[0].integrity_verified, protected);
|
||||
assert_eq!(results[0].before.drives.len(), 16);
|
||||
assert_eq!(results[0].after.drives.len(), 16);
|
||||
assert_eq!(results[0].before.drives[0].state, DriveState::Corrupt.to_string());
|
||||
@@ -217,8 +237,8 @@ async fn deep_heal_truncated_xlmeta_scenario() {
|
||||
repeat.execute().await.expect("repeated heal completes");
|
||||
let repeat_outcome = repeat.get_outcome().await;
|
||||
assert_eq!(repeat_outcome.counters.healed, 0);
|
||||
assert_eq!(repeat_outcome.counters.unchanged, 1);
|
||||
assert_eq!(repeat_outcome.counters.unknown, 0);
|
||||
assert_eq!(repeat_outcome.counters.unchanged, u64::from(protected));
|
||||
assert_eq!(repeat_outcome.counters.unknown, u64::from(!protected));
|
||||
assert_eq!(tokio::fs::read(&target_meta).await.expect("read repeated-heal target"), healed_bytes);
|
||||
let mut reader = env
|
||||
.ecstore
|
||||
@@ -273,8 +293,9 @@ async fn deep_heal_truncated_xlmeta_scenario() {
|
||||
let outcome = quorum_task.get_outcome().await;
|
||||
if damaged == 4 {
|
||||
result.expect("twelve authoritative members must repair four damaged copies");
|
||||
assert_eq!(outcome.counters.healed, 1);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.counters.healed, u64::from(protected));
|
||||
assert_eq!(outcome.counters.unknown, u64::from(!protected));
|
||||
assert_eq!(quorum_task.get_result_items().await[0].drives_healed(), Some(4));
|
||||
assert!(
|
||||
quorum_task.get_result_items().await[0]
|
||||
.after
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
//! tests never start a scanner, auto-heal, or an explicit Admin Heal request.
|
||||
//! Run with nextest: channel and storage singletons require separate processes.
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
mrf_queue::{self, snapshot::inspect_local_committed_snapshot},
|
||||
@@ -178,6 +180,8 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() {
|
||||
[
|
||||
("RUSTFS_HEAL_MRF_ENABLE", Some("true")),
|
||||
("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", Some("false")),
|
||||
("RUSTFS_SHARD_INTEGRITY_WRITE", Some("false")),
|
||||
("RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED", Some("false")),
|
||||
],
|
||||
async {
|
||||
let root = tempfile::tempdir().expect("test directory should be created");
|
||||
@@ -313,14 +317,21 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() {
|
||||
);
|
||||
assert!(
|
||||
wait_until(|| async {
|
||||
inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
|
||||
.await
|
||||
.expect("cleanup snapshot must validate")
|
||||
.is_none()
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
snapshot.queue_length == 0 && snapshot.active_tasks == 0
|
||||
})
|
||||
.await,
|
||||
"verified repair must release durable responsibility"
|
||||
"legacy repair attempts must finish without inventing verification"
|
||||
);
|
||||
assert!(snapshot_contains("new.bin").await);
|
||||
assert!(snapshot_contains("versioned.bin").await);
|
||||
manager.stop().await.expect("test manager should stop");
|
||||
let before_purge = inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
|
||||
.await
|
||||
.expect("legacy checkpoint must validate")
|
||||
.expect("unverified legacy responsibility must remain")
|
||||
.payload()
|
||||
.to_vec();
|
||||
// Deleting an existing marker by VersionId removes a version; it
|
||||
// must not create a new partial-write repair responsibility.
|
||||
for path in &env.disk_paths[12..] {
|
||||
@@ -346,12 +357,14 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() {
|
||||
)
|
||||
.await
|
||||
.expect("explicit marker purge should retain quorum");
|
||||
assert!(
|
||||
assert_eq!(
|
||||
inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
|
||||
.await
|
||||
.expect("purge must not create a checkpoint")
|
||||
.is_none(),
|
||||
"a physical marker purge must not be admitted as a marker creation repair"
|
||||
.expect("purge checkpoint must validate")
|
||||
.expect("existing legacy checkpoint must remain")
|
||||
.payload(),
|
||||
before_purge,
|
||||
"a physical marker purge must not add a marker creation repair"
|
||||
);
|
||||
for (path, disk) in env.disk_paths[12..].iter().zip(&all[12..]) {
|
||||
tokio::fs::remove_file(path).await.expect("remove purge outage sentinel");
|
||||
@@ -361,7 +374,6 @@ async fn partial_write_ec12_4_ack_rejoin_repairs_versions_and_delete_marker() {
|
||||
disk.reset_health_for_store_init_retry();
|
||||
}
|
||||
*set.disks.write().await = all.iter().cloned().map(Some).collect();
|
||||
manager.stop().await.expect("test manager should stop");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -391,6 +403,15 @@ fn partial_write_crash_fixture() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_write_sigkill_replay_rearms_and_repairs() {
|
||||
partial_write_sigkill_replay_scenario(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_sigkill_replay_repairs_without_releasing_unverified_responsibility() {
|
||||
partial_write_sigkill_replay_scenario(false).await;
|
||||
}
|
||||
|
||||
async fn partial_write_sigkill_replay_scenario(protected: bool) {
|
||||
use std::process::{Command, Stdio};
|
||||
let root = tempfile::tempdir().expect("crash test directory");
|
||||
let log = std::fs::File::create(root.path().join("child.log")).expect("child log");
|
||||
@@ -399,6 +420,8 @@ async fn partial_write_sigkill_replay_rearms_and_repairs() {
|
||||
.env("RUSTFS_TEST_PARTIAL_WRITE_CRASH_ROOT", root.path())
|
||||
.env("RUSTFS_HEAL_MRF_ENABLE", "true")
|
||||
.env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false")
|
||||
.env("RUSTFS_SHARD_INTEGRITY_WRITE", protected.to_string())
|
||||
.env("RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED", protected.to_string())
|
||||
.stdout(Stdio::from(log.try_clone().expect("clone child log")))
|
||||
.stderr(Stdio::from(log))
|
||||
.spawn()
|
||||
@@ -432,15 +455,27 @@ async fn partial_write_sigkill_replay_rearms_and_repairs() {
|
||||
"replayed responsibility must heal the returning member"
|
||||
);
|
||||
assert_payload(&env, "partial-crash", "crash.bin", None, b"durable partial write across SIGKILL").await;
|
||||
assert!(
|
||||
wait_until(|| async {
|
||||
inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
|
||||
.await
|
||||
.expect("valid checkpoint")
|
||||
.is_none()
|
||||
})
|
||||
.await,
|
||||
"replayed responsibility must be released only after verified repair"
|
||||
);
|
||||
if protected {
|
||||
assert!(
|
||||
wait_until(|| async {
|
||||
inspect_local_committed_snapshot(SNAPSHOT_LIMIT)
|
||||
.await
|
||||
.expect("valid checkpoint")
|
||||
.is_none()
|
||||
})
|
||||
.await,
|
||||
"replayed responsibility must be released only after verified repair"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
wait_until(|| async {
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
snapshot.queue_length == 0 && snapshot.active_tasks == 0
|
||||
})
|
||||
.await,
|
||||
"legacy replay attempts must finish"
|
||||
);
|
||||
assert!(snapshot_contains("crash.bin").await, "unverified legacy responsibility must remain");
|
||||
}
|
||||
manager.stop().await.expect("restarted manager should stop");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use rustfs_heal::heal::{
|
||||
outcome::HealObjectDisposition,
|
||||
storage::{ECStoreHealStorage, HealStorageAPI},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_test_utils::TestECStoreEnv;
|
||||
use serial_test::serial;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
mod storage_api;
|
||||
use storage_api::integration::{DiskAPI, ObjectIO, ObjectOptions, PutObjReader, ReadOptions, ShardIntegrityWriteMode};
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn legacy_repair_reports_execution_without_strong_receipt() {
|
||||
let root = tempfile::tempdir().expect("legacy receipt fixture");
|
||||
let env = TestECStoreEnv::builder()
|
||||
.base_dir(root.path())
|
||||
.prefix("legacy_receipt")
|
||||
.build()
|
||||
.await;
|
||||
let bucket = "legacy-receipt";
|
||||
let object = "missing-shard";
|
||||
env.make_bucket(bucket, false).await;
|
||||
let expected = vec![0x5c; 1024 * 1024 + 37];
|
||||
env.ecstore
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("legacy object");
|
||||
let set = env.ecstore.pools[0].get_disks(0);
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
let meta = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("metadata");
|
||||
tokio::fs::remove_file(
|
||||
env.disk_paths[0]
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(meta.data_dir.expect("external directory").to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("remove one shard");
|
||||
let result = ECStoreHealStorage::new(env.ecstore.clone())
|
||||
.heal_object_with_receipt(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("legacy repair");
|
||||
assert!(result.error.is_none(), "{:?}", result.error);
|
||||
assert_eq!(result.item.drives_healed(), Some(1));
|
||||
assert!(!result.item.integrity_verified);
|
||||
assert!(result.receipt.is_none(), "physical repair does not prove original identity");
|
||||
let mut reader = env
|
||||
.ecstore
|
||||
.get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read repaired object");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("exact recovered body");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn receipt_requires_independent_deep_verification() {
|
||||
let root = tempfile::tempdir().expect("receipt fixture");
|
||||
let env = TestECStoreEnv::builder()
|
||||
.base_dir(root.path())
|
||||
.prefix("shard_identity_receipt")
|
||||
.build()
|
||||
.await;
|
||||
let bucket = "shard-identity-receipt";
|
||||
env.make_bucket(bucket, false).await;
|
||||
let set = env.ecstore.pools[0].get_disks(0);
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
let storage = ECStoreHealStorage::new(env.ecstore.clone());
|
||||
for corrupt_count in [1usize, 3] {
|
||||
let object = format!("target-{corrupt_count}");
|
||||
let donor = format!("donor-{corrupt_count}");
|
||||
for (name, byte) in [(&object, 0x3c), (&donor, 0xa9)] {
|
||||
env.ecstore
|
||||
.put_object(
|
||||
bucket,
|
||||
name,
|
||||
&mut PutObjReader::from_vec(vec![byte; 1024 * 1024 + 123]),
|
||||
&ObjectOptions {
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("commit all fixture shards");
|
||||
}
|
||||
let mut target_meta = Vec::new();
|
||||
let mut donor_meta = Vec::new();
|
||||
for disk in &disks {
|
||||
target_meta.push(
|
||||
disk.read_version("", bucket, &object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("target metadata"),
|
||||
);
|
||||
donor_meta.push(
|
||||
disk.read_version("", bucket, &donor, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("donor metadata"),
|
||||
);
|
||||
}
|
||||
let mut retained = Vec::new();
|
||||
for (slot, target) in target_meta.iter().enumerate() {
|
||||
let target_path = env.disk_paths[slot]
|
||||
.join(bucket)
|
||||
.join(&object)
|
||||
.join(target.data_dir.expect("target directory").to_string())
|
||||
.join("part.1");
|
||||
let original = tokio::fs::read(&target_path).await.expect("original shard");
|
||||
if target.erasure.index <= corrupt_count {
|
||||
let donor_slot = donor_meta
|
||||
.iter()
|
||||
.position(|part| part.erasure.index == target.erasure.index)
|
||||
.expect("same donor coding index");
|
||||
let donor_path = env.disk_paths[donor_slot]
|
||||
.join(bucket)
|
||||
.join(&donor)
|
||||
.join(donor_meta[donor_slot].data_dir.expect("donor directory").to_string())
|
||||
.join("part.1");
|
||||
let replacement = tokio::fs::read(donor_path).await.expect("complete donor shard");
|
||||
assert_eq!(replacement.len(), original.len());
|
||||
tokio::fs::write(&target_path, replacement)
|
||||
.await
|
||||
.expect("replace intact shard");
|
||||
} else {
|
||||
retained.push((target_path, original));
|
||||
}
|
||||
}
|
||||
let normal = storage
|
||||
.heal_object_with_receipt(
|
||||
bucket,
|
||||
&object,
|
||||
None,
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("normal presence scan");
|
||||
assert!(normal.receipt.is_none(), "a presence scan cannot certify payload integrity");
|
||||
let result = storage
|
||||
.heal_object_with_receipt(
|
||||
bucket,
|
||||
&object,
|
||||
None,
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if corrupt_count == 1 {
|
||||
let result = result.expect("recoverable repair");
|
||||
assert!(result.error.is_none());
|
||||
assert_eq!(
|
||||
result.receipt.expect("verified repair receipt").disposition,
|
||||
HealObjectDisposition::Repaired,
|
||||
"presence alone must not produce VerifiedHealthy"
|
||||
);
|
||||
} else if let Ok(result) = result {
|
||||
assert!(result.error.is_some(), "below quorum cannot be a success");
|
||||
assert!(result.receipt.is_none(), "no completion receipt without authoritative sources");
|
||||
}
|
||||
for (path, original) in retained {
|
||||
assert_eq!(tokio::fs::read(path).await.expect("retained correct shard"), original);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ pub(crate) mod integration {
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, RUSTFS_META_BUCKET, ReadOptions, new_disk,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore;
|
||||
pub(crate) use rustfs_storage_api::BucketOperations;
|
||||
pub(crate) use rustfs_storage_api::MakeBucketOptions;
|
||||
|
||||
@@ -36,6 +36,10 @@ const DRIVE_STATE_OK: &str = "ok";
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct HealResultItem {
|
||||
/// Only the coordinator's authenticated full scan may set this. Transported
|
||||
/// or old-peer results default to unproven and cannot advance heal receipts.
|
||||
#[serde(skip)]
|
||||
pub integrity_verified: bool,
|
||||
#[serde(rename = "resultId")]
|
||||
pub result_index: usize,
|
||||
#[serde(rename = "type")]
|
||||
|
||||
@@ -288,6 +288,9 @@ pub struct RenameFileRequest {
|
||||
pub dst_volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "5")]
|
||||
pub dst_path: ::prost::alloc::string::String,
|
||||
/// Apply the destination's payload/metadata durability policy before success.
|
||||
#[prost(bool, tag = "6")]
|
||||
pub durable: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct RenameFileResponse {
|
||||
@@ -295,6 +298,8 @@ pub struct RenameFileResponse {
|
||||
pub success: bool,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
#[prost(bool, tag = "3")]
|
||||
pub durability_applied: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct WriteRequest {
|
||||
|
||||
@@ -1139,6 +1139,11 @@ pub fn canonical_rename_file_request_body(
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
// Preserve the release signature for ordinary renames. An older server
|
||||
// computes a different digest for a durable request and rejects mutation.
|
||||
if request.durable {
|
||||
body.push_bool(true);
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
@@ -1450,6 +1455,7 @@ mod disk_mutation_canonical_tests {
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let rename_file = RenameFileRequest {
|
||||
durable: false,
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
@@ -1458,6 +1464,7 @@ mod disk_mutation_canonical_tests {
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_file_request_body(&rename_file).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut RenameFileRequest| r.durable = true,
|
||||
|r: &mut RenameFileRequest| r.disk = "d2".into(),
|
||||
|r: &mut RenameFileRequest| r.src_volume = "sv2".into(),
|
||||
|r: &mut RenameFileRequest| r.src_path = "sp2".into(),
|
||||
@@ -1632,10 +1639,28 @@ mod disk_mutation_canonical_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_rename_extension_preserves_old_wire_defaults_and_signatures() {
|
||||
use crate::proto_gen::node_service::RenameFileResponse;
|
||||
use prost::Message;
|
||||
// Release wire messages omitted request tag 6 and response tag 3.
|
||||
let mut request =
|
||||
RenameFileRequest::decode(b"\x0a\x01d\x12\x01s\x1a\x01p\x22\x01v\x2a\x01q".as_slice()).expect("release request");
|
||||
assert!(!request.durable);
|
||||
let old_body = b"rustfs-rename-file-request-v1\0\0\0\0\0\0\0\0\x01d\0\0\0\0\0\0\0\x01s\0\0\0\0\0\0\0\x01p\0\0\0\0\0\0\0\x01v\0\0\0\0\0\0\0\x01q";
|
||||
assert_eq!(canonical_rename_file_request_body(&request).expect("body"), old_body);
|
||||
request.durable = true;
|
||||
assert_ne!(canonical_rename_file_request_body(&request).expect("durable body"), old_body);
|
||||
let old_success = RenameFileResponse::decode(b"\x08\x01".as_slice()).expect("release response");
|
||||
assert!(old_success.success);
|
||||
assert!(!old_success.durability_applied, "old success cannot certify durable publication");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_mutation_canonical_domains_are_distinct_per_message() {
|
||||
// The same field values must never authenticate one RPC's request as another's.
|
||||
let rename_file = RenameFileRequest {
|
||||
durable: false,
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
|
||||
@@ -219,11 +219,14 @@ message RenameFileRequest {
|
||||
string src_path = 3;
|
||||
string dst_volume = 4;
|
||||
string dst_path = 5;
|
||||
// Apply the destination's payload/metadata durability policy before success.
|
||||
bool durable = 6;
|
||||
}
|
||||
|
||||
message RenameFileResponse {
|
||||
bool success = 1;
|
||||
optional Error error = 2;
|
||||
bool durability_applied = 3;
|
||||
}
|
||||
|
||||
message WriteRequest {
|
||||
|
||||
@@ -24,3 +24,6 @@ procedure is required.
|
||||
|
||||
For storage dashboards, see [Storage metrics and observer selection](operations/storage-metrics.md):
|
||||
drive ownership, snapshot freshness, counter queries, and rolling upgrades.
|
||||
|
||||
For optional shard commitments, see [Independent shard integrity rollout](operations/shard-integrity-rollout.md):
|
||||
activation, legacy repair results, multipart mode changes, and rollback limits.
|
||||
|
||||
@@ -156,6 +156,73 @@ Each shard file is self-verifying against silent disk corruption.
|
||||
|
||||
---
|
||||
|
||||
### 5.1 Independent shard commitments
|
||||
|
||||
New PUTs and newly initiated multipart uploads retain `CSumAlgo = 1` and the
|
||||
existing `[HighwayHash256][shard]` frames. Independent commitments are disabled
|
||||
for new writes by default; both `RUSTFS_SHARD_INTEGRITY_WRITE` and
|
||||
`RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED` must be enabled to start protecting new
|
||||
writes. Existing protected objects and uploads retain their mode. An independent SHA-256
|
||||
commitment protects against replacing a complete frame with a same-length donor
|
||||
frame whose self-contained checksum is valid (backlog#2497).
|
||||
|
||||
The commitment describes an immutable part generation, with a random UUID,
|
||||
part number, exact encoded length, erasure geometry, codec mode, stripe count,
|
||||
and Merkle root. Each leaf commits to the ordered SHA-256 digests of **all**
|
||||
encoded shards in one stripe. Payload digests include generation, part, stripe,
|
||||
coding index, length, and geometry. The root also commits to the final part size.
|
||||
A valid metadata quorum selects the expected root; neither RS consistency nor
|
||||
an intact adjacent HighwayHash checksum establishes the root.
|
||||
|
||||
The dual-prefix internal `shard-integrity-v1` metadata value encodes a canonical
|
||||
Base64 table: a 32-byte header and 64 bytes per part, up to 10,000 sorted unique
|
||||
parts. The same table is stored under both internal prefixes. Readers reject a
|
||||
malformed, conflicting, or incomplete descriptor instead of treating it as
|
||||
legacy metadata. Each external part has an immutable
|
||||
`part.N.integrity.<generation UUID>` sidecar, replicated on every participating
|
||||
disk. Its 64-byte header is followed by stripe records containing all shard
|
||||
digests and a Merkle path. An inline object stores the small proof under the
|
||||
dual-prefix `shard-integrity-inline-v1` key.
|
||||
|
||||
GET authenticates a stripe record against the selected root, then verifies each
|
||||
source shard before decoding or emitting bytes. Reconstructed data is also
|
||||
checked against its expected digest. Range reads fetch only the proof records
|
||||
for touched stripes; deferred parity readers retain their exact stripe position.
|
||||
A missing or corrupt index replica can use another authenticated replica. Deep
|
||||
Heal verifies at the coordinator, including data returned by older disk servers,
|
||||
and restores missing indexes only from proofs matching the existing root. It
|
||||
never mints a new commitment from suspect stored bytes. Index-only repair leaves
|
||||
payload and `xl.meta` bytes unchanged and requires acknowledged durable publish.
|
||||
|
||||
Writes publish the payload and proof on the same write quorum. UploadPart uses
|
||||
a fresh generation for each replacement, includes that generation and root in
|
||||
part-metadata quorum selection, and publishes its index before the existing
|
||||
part transaction switches data and metadata. Settlement removes only the
|
||||
obsolete generation; rollback retains the old index. Interrupted preparation
|
||||
can leave unreferenced files until the upload directory is reclaimed. Ordinary
|
||||
metadata COPY preserves commitments. Physical rewrites inherit their source
|
||||
mode: protected sources create new commitments after reading through their
|
||||
verifier; legacy sources remain legacy. Rewriting existing bytes is not a
|
||||
trusted migration of their historical identity. An upload's persisted marker,
|
||||
not the current node's write switch, determines its UploadPart/Complete mode.
|
||||
|
||||
Digest/index builders spill above a 1 MiB buffer limit, and request readers keep
|
||||
a bounded stripe cache. For EC 12+4, a 5 GiB part with 1 MiB stripes has a
|
||||
4,751,424-byte index on each disk (about 1.42% of logical data across 16 disks).
|
||||
The maximum descriptor is 853,376 Base64 bytes per prefix, about 1.63 MiB for
|
||||
both copies. These are format bounds, not measured throughput guarantees.
|
||||
|
||||
Legacy objects retain their existing GET and traditional Heal behavior and
|
||||
therefore their residual complete-donor substitution risk. Ordinary shard repair
|
||||
and the existing explicit-version metadata recovery path remain available, but
|
||||
do not create commitments or certify object identity. Actual drive repairs are
|
||||
reported separately from strong integrity receipts. Normal
|
||||
presence scans do not issue strong integrity receipts even for protected
|
||||
objects. Only a completed exclusive Deep scan/repair with authenticated sources
|
||||
can do so. See the [upgrade contract](minio-file-format-compat.md#independent-integrity-upgrade-contract)
|
||||
for mixed-version and migration constraints and the
|
||||
[rollout runbook](../operations/shard-integrity-rollout.md) for activation and rollback.
|
||||
|
||||
## 6. On-disk format (`xl.meta`)
|
||||
|
||||
This section is the load-bearing compatibility contract for stored metadata. It is byte-compatible with MinIO's `xl.meta`; interop proof, the fixture corpus, and the out-of-scope list are owned by [minio-file-format-compat.md](minio-file-format-compat.md) — cite it, do not re-derive interop claims.
|
||||
@@ -252,7 +319,7 @@ Version-aware heal ([set_disk/ops/heal.rs](../../crates/ecstore/src/set_disk/ops
|
||||
|
||||
- **INVARIANT — reconstructability.** Heal refuses when `meta_to_heal_count > parity_blocks` (relaxed only if a quorum etag exists) or when any part loses more than `parity_blocks` shards.
|
||||
- **INVARIANT — geometry match.** `latest_meta.erasure.distribution.len()` must equal the online-disk, outdated-disk, and parts-metadata counts, else heal refuses ("backend disks manually modified"). A real object missing `data_dir` is `FileCorrupt`.
|
||||
- **Data-safety guard (backlog#920).** If data shards survive on ≥ `data_blocks` disks, regenerate the missing `xl.meta` from a valid FileInfo and re-drive heal rather than dangling-delete; torn writes (< `data_blocks`) fall through to dangling-delete handling.
|
||||
- **Data-safety guard (backlog#920).** Legacy explicit-version metadata recovery retains its validated geometry, consistent candidate identity, online-disk and available-data bounds. It does not establish independent payload identity. Protected recovery additionally requires a matching metadata quorum and authenticated data on ≥ `data_blocks` disks. Conflicting metadata and uncertain deletion conditions retain their existing refusal/grace behavior.
|
||||
- Healed shards are written to the outdated disks, each recording `erasure.index = slot + 1`, then `rename_data` to final. Heal admission / scanner budget is owned by [placement-repair-invariants.md](placement-repair-invariants.md).
|
||||
|
||||
---
|
||||
|
||||
@@ -146,3 +146,55 @@ Not fixture-proven: transitioned `xl.meta`; CORS, public-access-block, and bucke
|
||||
- Any change to `crates/filemeta` or `crates/ecstore/src/bucket` metadata encoding is a storage-format change and follows the migration and readiness contracts in [README.md](README.md) and the ecstore layout boundary rules.
|
||||
- Do not bump a Version Anchor without a read path for the prior value; see [erasure-coding.md](erasure-coding.md) for the accept-older, reject-newer rule.
|
||||
- `.github/workflows/ci.yml`, `.github/workflows/cache-warm.yml`, and `ARCHITECTURE.md` cite the [rio-v2 variant lifecycle](#rio-v2-variant-lifecycle) heading; keep it when editing this file.
|
||||
|
||||
## Independent integrity upgrade contract
|
||||
|
||||
The optional shard-integrity extension preserves the release checksum algorithm
|
||||
and payload framing. It does not bump XL container/header/metadata versions.
|
||||
`ObjectPartInfo` extensions use named MessagePack fields; older eight-field
|
||||
readers can ignore the new field, and new readers still decode old positional
|
||||
arrays. This is a decoder compatibility claim, not certification of arbitrary
|
||||
mixed-version write, repair, tiering, or downgrade workflows.
|
||||
|
||||
| Workflow | Contract |
|
||||
|---|---|
|
||||
| New reader, old object | Existing reads and traditional shard/explicit-version metadata recovery remain available; independent donor-substitution protection is absent until a trusted rewrite. |
|
||||
| Old reader, new object | Existing checksum/frame and metadata decoders remain usable; old readers do not enforce the independent proof. |
|
||||
| New coordinator, old disk server | GET/Deep verification runs at the coordinator. A protected UploadPart requires a write quorum that publishes its index. An old RenameFile response cannot acknowledge durable index repair. |
|
||||
| Old write/repair coordinator | Unsupported for protected data: it may omit, discard, or preserve stale integrity metadata. New protection is disabled by default so the coordinator rollout can precede activation. |
|
||||
| Pre-upgrade in-progress multipart upload | Remains legacy when completed. Reinitiate and reupload from a trusted source to obtain protection. |
|
||||
| New multipart upload completed by an old coordinator | Unsupported; a retained upload marker without a completed descriptor fails validation. |
|
||||
| Write switch changed during an upload | The persisted upload mode wins: legacy uploads remain legacy and protected uploads must finish with all proofs. |
|
||||
| Write switch disabled after activation | Existing protected reads, repair and rewrites still require protection. This does not make an old binary safe to reintroduce. |
|
||||
| Automatic COPY, materialization or data movement | Preserve the source protection mode. Rewriting legacy bytes is not independent evidence of their original content. |
|
||||
| Full downgrade after protected writes | No blanket guarantee; preserve a snapshot and validate read, write, repair, COPY, and multipart behavior before considering a downgrade. |
|
||||
|
||||
Upgrade all readers, writers and background coordinators before explicitly
|
||||
enabling both `RUSTFS_SHARD_INTEGRITY_WRITE` and
|
||||
`RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED`. Both default to false. The confirmation
|
||||
is an operator attestation, not automatic capability discovery or an old-node
|
||||
fence. Validate actual mixed-version writes and restart recovery before claiming
|
||||
a rolling-upgrade or pre-activation rollback guarantee. See the
|
||||
[rollout runbook](../operations/shard-integrity-rollout.md).
|
||||
There is no automatic migration of existing payloads and no need
|
||||
to rewrite them merely to preserve reading. To protect legacy objects, compare
|
||||
against a separately trusted source or end-to-end digest and rewrite/reupload
|
||||
through upgraded coordinators. Recomputing a hash from existing suspect shards,
|
||||
or agreeing RS parity alone, does not establish their original identity.
|
||||
|
||||
Normal scans, tier metadata scans and unproven legacy scans cannot produce `VerifiedHealthy` or
|
||||
`Repaired` integrity receipts. Legacy repair still runs and reports actual
|
||||
before/after drive changes; an unknown strong result does not mean the repair
|
||||
was never attempted. Authoritative historical-version absence/cleanup proofs
|
||||
remain separate from live payload verification. Retain trusted backups for
|
||||
legacy recovery. These commitments protect against misplaced or corrupted
|
||||
shards under an authoritative metadata quorum; they are not signatures against
|
||||
an attacker who can replace that quorum too.
|
||||
|
||||
The public MinIO source inspected at
|
||||
[`7aac2a2`](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/bitrot-streaming.go)
|
||||
uses adjacent streaming HighwayHash checksums. Its
|
||||
[XL metadata codec](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/xl-storage-format-v2.go)
|
||||
does not establish the independent part-generation commitment described here.
|
||||
This source comparison does not claim a live MinIO donor-shard reproduction or
|
||||
behavior of proprietary MinIO editions.
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# Independent shard integrity rollout
|
||||
|
||||
Independent shard commitments detect internally valid shards that belong to a
|
||||
different part generation. The feature preserves the existing checksum frames
|
||||
and adds optional metadata and proof indexes. It does not authenticate legacy
|
||||
data retroactively. The [format contract](../architecture/erasure-coding.md#51-independent-shard-commitments)
|
||||
defines the stored representation.
|
||||
|
||||
## Configuration and object lifetime
|
||||
|
||||
| Variable | Default | Effect |
|
||||
|---|---|---|
|
||||
| `RUSTFS_SHARD_INTEGRITY_WRITE` | `false` | Requests protection for new writes without an inherited object/upload mode. |
|
||||
| `RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED` | `false` | Operator confirmation that every reader, writer and background coordinator supports the extension. |
|
||||
|
||||
Both must be true before new writes opt in. Setting only the request switch
|
||||
does not enable protection. These flags do not perform peer capability discovery
|
||||
or stop an older binary from joining the cluster.
|
||||
|
||||
A PUT fixes its mode before encoding. An MPU fixes its mode at initiation and
|
||||
persists it with the upload. Completing an old upload after activation does not
|
||||
promote it. Disabling the switches during a protected upload does not permit
|
||||
parts or completion to drop the proof requirement.
|
||||
|
||||
Reads always follow the object's metadata. Turning off creation does not disable
|
||||
verification of existing protected objects. A corrupt, conflicting or incomplete
|
||||
protection declaration is an error, not a legacy object. Metadata-only COPY
|
||||
preserves the existing commitment. COPY and physical background rewrites inherit
|
||||
the source mode; they do not turn an unverified legacy object into a historically
|
||||
verified object simply by calculating a new hash.
|
||||
|
||||
## Upgrade and activation
|
||||
|
||||
1. Deploy a version that understands and maintains the extension, leaving both
|
||||
switches false. Existing legacy reads, shard repair and explicit-version
|
||||
metadata recovery continue under their existing bounds.
|
||||
2. Validate old/new process combinations for PUT, multipart completion, COPY,
|
||||
Heal and restart recovery in the actual deployment. Decoder fixture tests
|
||||
alone do not certify these operations.
|
||||
3. Finish upgrading every reader and write/repair/movement coordinator. Prevent
|
||||
old services or rollback images from rejoining before enabling protection.
|
||||
4. Qualify protected writes, range reads, repair, missing indexes and deployment
|
||||
restart/failure behavior. Measure small-object throughput, range latency and
|
||||
maximum multipart metadata overhead against the same release configuration.
|
||||
5. Enable both switches consistently. Verify new objects with an exclusive Deep
|
||||
scan before treating the enhanced mode as qualified for production traffic.
|
||||
|
||||
Existing rollout or performance gaps remain gates for production activation;
|
||||
this runbook is not a claim that a distributed upgrade has been exercised.
|
||||
|
||||
## Legacy repair and interpretation
|
||||
|
||||
Legacy objects remain exposed to substitution by a complete, internally valid
|
||||
donor shard. Traditional reconstruction can restore missing or detectably
|
||||
damaged shards, but cannot prove that all surviving bytes are the original
|
||||
content. It therefore does not create an independent digest or a positive strong
|
||||
integrity receipt.
|
||||
|
||||
Use the heal item's before/after drives and actual repaired-drive count to
|
||||
observe ordinary recovery. Strong outcome counters may remain unknown/skipped
|
||||
because object identity is unproven; that is distinct from execution progress.
|
||||
MRF likewise retains durable legacy repair obligations after physical recovery
|
||||
when no independent verification receipt can discharge them. An idle attempt
|
||||
queue does not imply that these persistent obligations have been cleared.
|
||||
Partial-write MRF replay uses Deep verification so a protected object's repair
|
||||
can discharge its obligation after verifying the payload. This adds full-object
|
||||
read work to those background attempts, including healthy replay targets.
|
||||
Normal presence scans likewise cannot certify protected payloads. Local Heal
|
||||
of transitioned objects checks metadata without reading the tier payload, so it
|
||||
does not issue a payload-integrity receipt even if a descriptor remains. An
|
||||
authoritative historical-version cleanup or absence proof has its own identity
|
||||
and commit checks and does not depend on a live payload digest.
|
||||
|
||||
To protect a legacy version as a trusted migration, validate against an
|
||||
independently trusted original or suitable end-to-end digest, then reupload.
|
||||
An ETag is not universally a plaintext MD5. Rehashing existing bytes or checking
|
||||
RS parity alone does not establish the original identity. Keep this distinction
|
||||
when measuring migration coverage; an ordinary server-side rewrite is not proof
|
||||
of historical correctness.
|
||||
|
||||
## Rollback
|
||||
|
||||
Before any protected objects or uploads exist, a rollback still needs actual
|
||||
old-version write/repair/restart qualification. Metadata decoding compatibility
|
||||
is insufficient.
|
||||
|
||||
After activation, the supported rollback floor is a version that understands
|
||||
and preserves the extension. Turning off creation leaves protected objects,
|
||||
historical versions and in-progress protected uploads in place. It neither
|
||||
converts them to legacy nor authorizes an arbitrary older reader/writer/repair
|
||||
coordinator. Preserve recovery snapshots and use an explicitly validated
|
||||
migration procedure if returning to an older format implementation is required.
|
||||
@@ -4126,6 +4126,41 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_rename_requires_its_own_authenticated_body_and_success() {
|
||||
let service = make_server();
|
||||
let mut message = RenameFileRequest {
|
||||
disk: "http://node-a:9000/data/rustfs0".to_owned(),
|
||||
src_volume: ".rustfs.sys/tmp".to_owned(),
|
||||
src_path: "integrity-stage/index".to_owned(),
|
||||
dst_volume: "bucket".to_owned(),
|
||||
dst_path: "object/index".to_owned(),
|
||||
durable: false,
|
||||
};
|
||||
let old_body = rustfs_protos::canonical_rename_file_request_body(&message).expect("ordinary rename body");
|
||||
message.durable = true;
|
||||
let mut tampered = Request::new(message.clone());
|
||||
set_tonic_canonical_body_digest(&mut tampered, &old_body).expect("digest");
|
||||
mark_v2_authenticated(&mut tampered);
|
||||
let error = service
|
||||
.rename_file(tampered)
|
||||
.await
|
||||
.expect_err("durability flag must be authenticated");
|
||||
assert_eq!(error.code(), tonic::Code::PermissionDenied);
|
||||
|
||||
let body = rustfs_protos::canonical_rename_file_request_body(&message).expect("durable rename body");
|
||||
let mut request = Request::new(message);
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("digest");
|
||||
mark_v2_authenticated(&mut request);
|
||||
let response = service
|
||||
.rename_file(request)
|
||||
.await
|
||||
.expect("valid digest passes the gate")
|
||||
.into_inner();
|
||||
assert!(!response.success, "unknown disk cannot apply the rename");
|
||||
assert!(!response.durability_applied, "failed publication cannot acknowledge durability");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_mutation_body_digest_gate_runs_before_disk_lookup() {
|
||||
let service = make_server();
|
||||
@@ -4340,6 +4375,7 @@ mod tests {
|
||||
assert_gated!(
|
||||
rename_file,
|
||||
RenameFileRequest {
|
||||
durable: false,
|
||||
disk: disk.clone(),
|
||||
src_volume: "src".into(),
|
||||
src_path: "sp".into(),
|
||||
@@ -5512,6 +5548,7 @@ mod tests {
|
||||
let service = create_test_node_service();
|
||||
|
||||
let request = Request::new(RenameFileRequest {
|
||||
durable: false,
|
||||
disk: "invalid-disk-path".to_string(),
|
||||
src_volume: "src-volume".to_string(),
|
||||
src_path: "src-path".to_string(),
|
||||
|
||||
@@ -1379,22 +1379,29 @@ impl NodeService {
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
match disk
|
||||
.rename_file(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path)
|
||||
.await
|
||||
{
|
||||
let result = if request.durable {
|
||||
disk.rename_file_durable(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path)
|
||||
.await
|
||||
} else {
|
||||
disk.rename_file(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path)
|
||||
.await
|
||||
};
|
||||
match result {
|
||||
Ok(_) => Ok(Response::new(RenameFileResponse {
|
||||
success: true,
|
||||
durability_applied: request.durable,
|
||||
error: None,
|
||||
})),
|
||||
Err(err) => Ok(Response::new(RenameFileResponse {
|
||||
success: false,
|
||||
durability_applied: false,
|
||||
error: Some(err.into()),
|
||||
})),
|
||||
}
|
||||
} else {
|
||||
Ok(Response::new(RenameFileResponse {
|
||||
success: false,
|
||||
durability_applied: false,
|
||||
error: Some(DiskError::other("cannot find disk".to_string()).into()),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -1395,6 +1395,15 @@ pub(crate) trait StorageDiskRpcExt {
|
||||
async fn read_file(&self, volume: &str, path: &str) -> DiskResult<FileReader>;
|
||||
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult<FileReader>;
|
||||
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()>;
|
||||
async fn rename_file_durable(
|
||||
&self,
|
||||
_src_volume: &str,
|
||||
_src_path: &str,
|
||||
_dst_volume: &str,
|
||||
_dst_path: &str,
|
||||
) -> DiskResult<()> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn rename_part(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
@@ -1550,6 +1559,10 @@ where
|
||||
ecstore_disk::DiskAPI::rename_file(self, src_volume, src_path, dst_volume, dst_path).await
|
||||
}
|
||||
|
||||
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()> {
|
||||
ecstore_disk::DiskAPI::rename_file_durable(self, src_volume, src_path, dst_volume, dst_path).await
|
||||
}
|
||||
|
||||
async fn rename_part(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
|
||||
Reference in New Issue
Block a user