gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 16:46:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(ecstore): complete EC validation coverage gate

Browse Source 
* test(ecstore): complete EC validation coverage gate

* test(ecstore): stabilize validation suite after rebase

* test(ecstore): fix rio-v2 clippy lint
This commit is contained in:
Zhengchao An
2026-07-09 03:12:48 +08:00
committed by GitHub
parent 7c701d9f2c
commit ed81d2f6b8
18 changed files with 4916 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/core/io_primitives.rs
+547 -20
View File
@@ -3060,35 +3060,28 @@ impl SetDisks {
}
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
let mut revert_futures = Vec::with_capacity(disks.len());
for (i, err) in errs.iter().enumerate() {
if err.is_some() {
continue;
}
if let Some(disk) = disks[i].as_ref() {
let disk = disk.clone();
let bucket = bucket.to_string();
let path = path_join_buf(&[prefix, STORAGE_FORMAT_FILE]);
revert_futures.push(async move {
if let Err(err) = disk
.delete(
&bucket,
&path,
DeleteOptions {
recursive: true,
..Default::default()
},
)
.await
{
warn!("write meta revert err {:?}", err);
}
});
if let Err(err) = disk
.delete(
bucket,
&path,
DeleteOptions {
recursive: true,
..Default::default()
},
)
.await
{
warn!("write meta revert err {:?}", err);
}
}
}
join_all(revert_futures).await;
return Err(err);
}
Ok(())
@@ -3811,6 +3804,7 @@ pub(in crate::set_disk) mod cleanup_fault_injection {
mod tests {
use super::*;
use std::io::Cursor;
use tempfile::TempDir;
use tokio::io::AsyncReadExt;
fn metadata_test_fileinfo(object: &str) -> FileInfo {
@@ -3840,6 +3834,49 @@ mod tests {
}
}
async fn read_multiple_test_disk(bucket: &str, objects: &[(&str, &[u8])]) -> (TempDir, DiskStore) {
let dir = tempfile::tempdir().expect("tempdir should be created");
let endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("disk should be created");
match disk.make_volume(bucket).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(err) => panic!("bucket should be available: {err:?}"),
}
for (object, body) in objects {
disk.write_all(bucket, object, Bytes::copy_from_slice(body))
.await
.expect("object should be written");
}
(dir, disk)
}
async fn io_primitives_test_set(disks: Vec<Option<DiskStore>>, default_parity_count: usize) -> Arc<SetDisks> {
let set_drive_count = disks.len();
SetDisks::new(
"io-primitives-test".to_string(),
Arc::new(RwLock::new(disks)),
set_drive_count,
default_parity_count,
0,
0,
Vec::new(),
FormatV3::new(1, 1),
Vec::new(),
)
.await
}
fn failed_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
Box::pin(async { ReadRepairAdmissionOutcome::Failed("injected submit failure".to_string()) })
}
@@ -3848,6 +3885,38 @@ mod tests {
Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
}
fn dropped_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
Box::pin(async {
ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Dropped(
rustfs_common::heal_channel::HealAdmissionDropReason::PolicyDropped,
))
})
}
fn test_object_bitrot_reader() -> ObjectBitrotReader {
BitrotReader::new(
Box::new(Cursor::new(vec![1u8, 2, 3, 4])) as Box<dyn AsyncRead + Send + Sync + Unpin>,
4,
HashAlgorithm::None,
false,
)
}
fn test_deferred_object_bitrot_reader() -> (ObjectBitrotReader, DeferredReaderStripeHandle) {
create_deferred_bitrot_reader_with_stripe_handle(
Some(Bytes::from_static(&[1, 2, 3, 4])),
None,
"bucket",
"object/part.1",
0,
4,
4,
HashAlgorithm::None,
false,
false,
)
}
#[test]
fn dangling_delete_grace_defaults_to_one_hour() {
temp_env::with_var(ENV_HEAL_DANGLING_DELETE_GRACE_SECS, None::<&str>, || {
@@ -4009,6 +4078,36 @@ mod tests {
assert!(part.etag.is_empty());
}
#[test]
fn resolve_read_part_returns_part_when_etag_reaches_quorum() {
let responses = vec![
Some(vec![read_part_test_part(1, "winner")]),
Some(vec![read_part_test_part(1, "loser")]),
Some(vec![read_part_test_part(1, "winner")]),
];
let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
.expect("etag quorum should resolve the present part");
assert_eq!(part.etag, "winner");
}
#[test]
fn resolve_read_part_diagnostic_branch_keeps_read_quorum_error() {
temp_env::with_var("RUSTFS_ISSUE3031_DIAG_ENABLE", Some("true"), || {
let responses = vec![
Some(Vec::new()),
None,
Some(vec![read_part_test_error(1, "permission denied")]),
];
let err = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
.expect_err("diagnostic logging must not change the read-quorum result");
assert_eq!(err, DiskError::ErasureReadQuorum);
});
}
// Runs under a Tokio runtime like the sibling reservation tests:
// shard_read_costs_for_disks consults process-global topology state
// (local_endpoint_hosts_for_shard_costs), whose fast-lock manager lazily
@@ -4021,6 +4120,349 @@ mod tests {
assert!(shard_read_costs_for_disks(&[]).is_empty());
}
#[test]
fn shard_read_cost_for_endpoint_and_missing_disk_cover_all_cost_classes() {
let same_node_hosts = vec!["node-a:9000".to_string()];
assert_eq!(shard_read_cost_for_disk(None, &same_node_hosts), ShardReadCost::Unknown);
assert_eq!(shard_read_cost_for_endpoint(true, "", &same_node_hosts), ShardReadCost::Local);
assert_eq!(
shard_read_cost_for_endpoint(false, "node-a:9000", &same_node_hosts),
ShardReadCost::SameNode
);
assert_eq!(
shard_read_cost_for_endpoint(false, "node-b:9000", &same_node_hosts),
ShardReadCost::Remote
);
assert!(local_endpoint_hosts_for_shard_costs().is_empty());
}
#[test]
#[serial_test::serial]
fn bitrot_reader_setup_tracks_strategy_counters_and_deferred_readers() {
temp_env::with_var(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true"), || {
assert!(matches!(
get_bitrot_reader_setup_strategy(BitrotReaderSetupMode::ReadQuorum, false),
BitrotReaderSetupStrategy::DataBlocksFirst
));
});
temp_env::with_var(ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, Some("true"), || {
assert!(matches!(
get_bitrot_reader_setup_strategy(BitrotReaderSetupMode::VerifyReconstruction, false),
BitrotReaderSetupStrategy::DataBlocksFirst
));
});
assert_eq!(BitrotReaderSetupMode::ReadQuorum.as_str(), "read_quorum");
assert_eq!(BitrotReaderSetupMode::VerifyReconstruction.as_str(), "verify_reconstruction");
assert_eq!(BitrotReaderSetupStrategy::AllShards.as_str(), "all_shards");
assert_eq!(BitrotReaderSetupStrategy::DataBlocksFirst.as_str(), "data_blocks_first");
assert_eq!(BitrotReaderSetupStrategy::DataBlocksOnly.as_str(), "data_blocks_only");
let mut setup = BitrotReaderSetup::new(4);
assert_eq!(setup.scheduled_shards(), 0);
assert!(setup.mark_scheduled(0));
assert!(!setup.mark_scheduled(0));
assert_eq!(setup.scheduled_shards(), 1);
assert_eq!(setup.pending_scheduled_shards(), 1);
assert_eq!(setup.available_shards(), 0);
assert_eq!(setup.completed_failed_shards(), 0);
assert_eq!(setup.reconstruction_verification_target(3, 2), 3);
assert!(!setup.has_setup_quorum(3, 2, BitrotReaderSetupMode::ReadQuorum));
assert!(!setup.data_shards_attempted(3));
assert_eq!(setup.scheduling_target(3, 2, BitrotReaderSetupMode::VerifyReconstruction), 3);
setup.apply_reader_result(0, Ok(Some(test_object_bitrot_reader())));
setup.apply_reader_result(1, Ok(None));
setup.apply_reader_result(2, Err(DiskError::FileCorrupt));
assert_eq!(setup.attempted_shards(), 3);
assert_eq!(setup.pending_scheduled_shards(), 0);
assert_eq!(setup.available_shards(), 1);
assert_eq!(setup.available_data_shards(3), 1);
assert_eq!(setup.completed_failed_shards(), 2);
assert!(setup.data_shards_attempted(3));
assert_eq!(setup.reconstruction_verification_target(3, 2), 3);
assert_eq!(setup.setup_target(3, 2, BitrotReaderSetupMode::VerifyReconstruction), 3);
assert_eq!(setup.scheduling_target(3, 2, BitrotReaderSetupMode::VerifyReconstruction), 3);
let mut verification_setup = BitrotReaderSetup::new(4);
verification_setup.apply_reader_result(0, Ok(Some(test_object_bitrot_reader())));
verification_setup.apply_reader_result(1, Ok(Some(test_object_bitrot_reader())));
verification_setup.apply_reader_result(2, Err(DiskError::FileCorrupt));
verification_setup.apply_reader_result(3, Ok(Some(test_object_bitrot_reader())));
assert_eq!(verification_setup.reconstruction_verification_target(3, 2), 4);
assert_eq!(verification_setup.setup_target(3, 2, BitrotReaderSetupMode::VerifyReconstruction), 4);
assert!(verification_setup.has_setup_quorum(3, 2, BitrotReaderSetupMode::ReadQuorum));
let (deferred_reader, stripe_handle) = test_deferred_object_bitrot_reader();
setup.retain_deferred_reader(3, deferred_reader, stripe_handle);
assert_eq!(setup.deferred_shards(), 1);
assert!(setup.readers[3].is_some());
assert!(setup.errors[3].is_none());
}
#[tokio::test]
async fn write_unique_file_info_reverts_metadata_when_write_quorum_fails() {
let bucket = "write-unique-bucket";
let object = "object";
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let files = vec![metadata_test_fileinfo(object), metadata_test_fileinfo(object)];
let result = SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &files, 2).await;
assert!(result.is_err(), "missing disk must prevent the requested write quorum");
assert!(
matches!(
disk.read_all(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE])).await,
Err(DiskError::FileNotFound)
),
"successful metadata write must be reverted when quorum is not reached"
);
}
#[tokio::test]
async fn update_object_meta_handles_empty_metadata_and_missing_quorum() {
let set = io_primitives_test_set(vec![None, None], 1).await;
let mut empty = metadata_test_fileinfo("object");
empty.metadata.clear();
set.update_object_meta("bucket", "object", empty, &[None, None])
.await
.expect("empty metadata update without replacement should be a no-op");
let mut with_metadata = metadata_test_fileinfo("object");
with_metadata
.metadata
.insert("x-amz-meta-test".to_string(), "value".to_string());
let result = set.update_object_meta("bucket", "object", with_metadata, &[None, None]).await;
assert!(result.is_err(), "missing disks must prevent metadata write quorum");
}
#[tokio::test]
async fn load_file_info_versions_exact_returns_versions_from_read_quorum() {
let bucket = "exact-versions-bucket";
let object = "exact-object";
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let mut fi = metadata_test_fileinfo(object);
fi.version_id = Some(Uuid::new_v4());
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, object, fi.clone())
.await
.expect("metadata should be written");
let set = io_primitives_test_set(vec![Some(disk)], 0).await;
let versions = set
.load_file_info_versions_exact(bucket, object)
.await
.expect("exact version load should succeed")
.expect("exact version load should find metadata");
assert_eq!(versions.versions.len(), 1);
assert_eq!(versions.versions[0].version_id, fi.version_id);
assert_eq!(versions.versions[0].name, object);
}
#[tokio::test]
async fn commit_rename_data_dir_reclaims_old_data_dir_and_reports_receipt() {
let bucket = "commit-rename-bucket";
let object = "object";
let old_data_dir = "11111111-1111-1111-1111-111111111111";
let committed_data_dir = "22222222-2222-2222-2222-222222222222";
let path = format!("{object}/{old_data_dir}/part.1");
let (_dir1, disk1) = read_multiple_test_disk(bucket, &[(&path, b"one".as_slice())]).await;
let (_dir2, disk2) = read_multiple_test_disk(bucket, &[(&path, b"two".as_slice())]).await;
let set = io_primitives_test_set(vec![Some(disk1.clone()), Some(disk2.clone())], 1).await;
let cleanup = set
.commit_rename_data_dir(
&[Some(disk1.clone()), Some(disk2.clone())],
bucket,
object,
old_data_dir,
committed_data_dir,
2,
)
.await;
assert_eq!(cleanup.attempted, 2);
assert_eq!(cleanup.reclaimed, 2);
assert!(!cleanup.has_residue());
assert!(matches!(disk1.read_all(bucket, &path).await, Err(DiskError::FileNotFound)));
assert!(matches!(disk2.read_all(bucket, &path).await, Err(DiskError::FileNotFound)));
let missing = set
.commit_rename_data_dir(&[None, None], bucket, object, old_data_dir, committed_data_dir, 1)
.await;
assert_eq!(missing.attempted, 0);
assert_eq!(missing.reclaimed, 0);
assert!(!missing.has_residue(), "missing disk slots must not be counted as cleanup residue");
assert!(missing.below_quorum, "missing disk slots should remain visible in the quorum lens");
}
#[tokio::test]
async fn delete_prefix_removes_present_disks_and_ignores_missing_disk_slots() {
let bucket = "delete-prefix-bucket";
let (_dir, disk) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"payload".as_slice())]).await;
let set = io_primitives_test_set(vec![Some(disk.clone()), None], 1).await;
set.delete_prefix(bucket, "prefix")
.await
.expect("missing disk slots should not block prefix deletion");
assert!(matches!(disk.read_all(bucket, "prefix/object.txt").await, Err(DiskError::FileNotFound)));
}
#[tokio::test]
async fn delete_if_dangling_respects_recent_write_grace_without_deleting_metadata() {
let bucket = "dangling-grace-bucket";
let object = "object";
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let set = io_primitives_test_set(vec![Some(disk.clone()), None, None], 1).await;
let mut fi = metadata_test_fileinfo(object);
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, object, fi.clone())
.await
.expect("metadata should be written before dangling check");
let err = set
.delete_if_dangling(
bucket,
object,
&[fi, FileInfo::default(), FileInfo::default()],
&[None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)],
&HashMap::new(),
ObjectOptions::default(),
)
.await
.expect_err("recent dangling metadata must stay protected by grace");
assert_eq!(err, DiskError::ErasureReadQuorum);
disk.read_all(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE]))
.await
.expect("metadata should remain during dangling grace");
}
#[tokio::test]
async fn delete_if_dangling_returns_stale_metadata_when_all_slots_are_already_absent() {
let bucket = "dangling-delete-bucket";
let object = "object";
let set = io_primitives_test_set(vec![None, None, None], 1).await;
let mut fi = metadata_test_fileinfo(object);
fi.mod_time = Some(OffsetDateTime::now_utc() - time::Duration::hours(2));
let deleted = set
.delete_if_dangling(
bucket,
object,
&[fi.clone(), FileInfo::default(), FileInfo::default()],
&[
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
],
&HashMap::new(),
ObjectOptions::default(),
)
.await
.expect("stale dangling metadata should pass when every delete target was already absent");
assert_eq!(deleted.name, object);
assert_eq!(deleted.mod_time, fi.mod_time);
}
#[tokio::test]
async fn delete_if_dangling_cleans_when_only_part_results_prove_invalid_metadata_is_dangling() {
let bucket = "dangling-invalid-meta-bucket";
let object = "object";
let set = io_primitives_test_set(vec![None, None, None, None], 1).await;
let mut data_errs_by_part = HashMap::new();
data_errs_by_part.insert(
1,
vec![
CHECK_PART_FILE_NOT_FOUND,
CHECK_PART_FILE_NOT_FOUND,
CHECK_PART_FILE_NOT_FOUND,
CHECK_PART_DISK_NOT_FOUND,
],
);
let deleted = set
.delete_if_dangling(
bucket,
object,
&[
FileInfo::default(),
FileInfo::default(),
FileInfo::default(),
FileInfo::default(),
],
&[
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
Some(DiskError::FileNotFound),
Some(DiskError::DiskNotFound),
],
&data_errs_by_part,
ObjectOptions::default(),
)
.await
.expect("invalid metadata should be cleanable when part results prove the object is dangling");
assert!(!deleted.is_valid());
}
#[tokio::test]
async fn read_multiple_files_returns_quorum_data_and_fails_closed_for_partial_file() {
let bucket = "read-multiple-bucket";
let prefix = "prefix";
let (_dir1, disk1) = read_multiple_test_disk(
bucket,
&[
("prefix/shared.txt", b"longer shared payload".as_slice()),
("prefix/partial.txt", b"only one disk".as_slice()),
],
)
.await;
let (_dir2, disk2) = read_multiple_test_disk(bucket, &[("prefix/shared.txt", b"short".as_slice())]).await;
let req = ReadMultipleReq {
bucket: bucket.to_string(),
prefix: prefix.to_string(),
files: vec!["shared.txt".to_string(), "partial.txt".to_string()],
max_size: 0,
metadata_only: false,
abort404: false,
max_results: 0,
};
let responses = SetDisks::read_multiple_files(&[Some(disk1), Some(disk2)], req, 2).await;
assert_eq!(responses.len(), 2);
assert!(responses[0].exists);
assert_eq!(responses[0].data, b"longer shared payload");
assert!(!responses[1].exists);
assert_eq!(responses[1].error, Error::ErasureReadQuorum.to_string());
}
#[tokio::test]
async fn read_multiple_files_returns_read_quorum_error_when_no_disk_can_answer() {
let req = ReadMultipleReq {
bucket: "bucket".to_string(),
prefix: "prefix".to_string(),
files: vec!["missing.txt".to_string()],
max_size: 0,
metadata_only: false,
abort404: false,
max_results: 0,
};
let responses = SetDisks::read_multiple_files(&[None, None], req, 1).await;
assert_eq!(responses.len(), 1);
assert!(!responses[0].exists);
assert_eq!(responses[0].error, Error::ErasureReadQuorum.to_string());
}
#[tokio::test]
async fn reserve_read_repair_heal_dedupes_by_object_version_and_set() {
let object = format!("object-{}", Uuid::new_v4());
@@ -4042,6 +4484,63 @@ mod tests {
release_read_repair_heal_reservation(&retry_key).await;
}
#[test]
fn record_read_repair_dedup_accepts_known_reasons() {
record_read_repair_dedup("duplicate");
record_read_repair_dedup("policy_drop");
}
#[tokio::test]
async fn reserve_read_repair_heal_prunes_oldest_entry_at_capacity() {
let bucket = format!("bucket-{}", Uuid::new_v4());
let first_object = format!("object-{}", Uuid::new_v4());
let first_key = reserve_read_repair_heal(&bucket, &first_object, None, 1, 1)
.await
.expect("first reservation should be accepted");
let mut keys = vec![first_key.clone()];
for index in 0..(READ_REPAIR_HEAL_DEDUP_MAX_ENTRIES + 8) {
let object = format!("object-{index}-{}", Uuid::new_v4());
if let Some(key) = reserve_read_repair_heal(&bucket, &object, None, 1, 1).await {
keys.push(key);
}
}
let replaced_first = reserve_read_repair_heal(&bucket, &first_object, None, 1, 1).await;
for key in keys {
release_read_repair_heal_reservation(&key).await;
}
if let Some(key) = replaced_first.as_ref() {
release_read_repair_heal_reservation(key).await;
}
assert!(
replaced_first.is_some(),
"capacity pruning should evict the oldest read-repair reservation"
);
}
#[tokio::test]
async fn submit_read_repair_heal_wrapper_records_reservation_without_external_channel() {
let object = format!("object-{}", Uuid::new_v4());
submit_read_repair_heal("bucket", &object, None, 4, 5, Some(7), "test").await;
for _ in 0..20 {
if let Some(key) = reserve_read_repair_heal("bucket", &object, None, 4, 5).await {
release_read_repair_heal_reservation(&key).await;
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
if let Some(key) = reserve_read_repair_heal("bucket", &object, None, 4, 5).await {
release_read_repair_heal_reservation(&key).await;
} else {
let admitted_key = ReadRepairHealCacheKey::new("bucket", &object, None, 4, 5);
release_read_repair_heal_reservation(&admitted_key).await;
}
}
#[tokio::test]
async fn submit_read_repair_heal_releases_reservation_after_submitter_failure() {
let object = format!("object-{}", Uuid::new_v4());
@@ -4070,6 +4569,34 @@ mod tests {
panic!("failed read-repair submission should release its dedup reservation");
}
#[tokio::test]
async fn submit_read_repair_heal_releases_reservation_after_not_admitted_response() {
let object = format!("object-{}", Uuid::new_v4());
submit_read_repair_heal_with_submitter(
ReadRepairHealSubmission {
bucket: "bucket",
object: &object,
version_id: Some("version-1"),
pool_index: 2,
set_index: 3,
part_number: Some(2),
reason: "test",
},
dropped_read_repair_submitter,
)
.await;
for _ in 0..20 {
if let Some(key) = reserve_read_repair_heal("bucket", &object, Some("version-1"), 2, 3).await {
release_read_repair_heal_reservation(&key).await;
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
panic!("not-admitted read-repair submission should release its dedup reservation");
}
#[tokio::test]
async fn submit_read_repair_heal_keeps_admitted_reservation_deduped() {
let object = format!("object-{}", Uuid::new_v4());
crates/ecstore/src/set_disk/metadata.rs
+98
View File
@@ -1086,6 +1086,85 @@ mod tests {
assert_eq!(err, DiskError::ErasureReadQuorum);
}
#[test]
fn metadata_quorum_covers_etag_fallback_and_object_quorum_failures() {
let mut parts_metadata = (1..=3)
.map(|index| {
let mut fi = metadata_quorum_test_fileinfo(OffsetDateTime::now_utc(), index);
fi.mod_time = None;
fi
})
.collect::<Vec<_>>();
parts_metadata[2]
.metadata
.insert("etag".to_string(), "minority-etag".to_string());
let errs = vec![None; parts_metadata.len()];
let disks = vec![None; parts_metadata.len()];
let (_online, mod_time, etag) = SetDisks::list_online_disks(&disks, &parts_metadata, &errs, 2);
assert!(mod_time.is_none());
assert_eq!(etag.as_deref(), Some("object-etag"));
let zero_parity = SetDisks::object_quorum_from_meta(&parts_metadata, &errs, 0)
.expect("zero default parity should require all metadata shards");
assert_eq!(zero_parity, (3, 3));
let invalid = vec![FileInfo::default(); 4];
let err = SetDisks::object_quorum_from_meta(&invalid, &vec![None; 4], 2)
.expect_err("invalid metadata without a common parity must fail closed");
assert_eq!(err, DiskError::ErasureReadQuorum);
}
#[test]
fn fileinfo_quorum_hash_includes_optional_checksums_and_ignores_replication_noise() {
let mod_time = OffsetDateTime::now_utc();
let mut left = metadata_quorum_test_fileinfo(mod_time, 1);
left.mode = Some(0o640);
left.written_by_version = Some(42);
left.checksum = Some(Bytes::from_static(b"object-checksum"));
left.parts[0].index = Some(Bytes::from_static(b"part-index"));
left.parts[0].error = Some("repair-pending".to_string());
left.parts[0].checksums = Some(HashMap::from([
("sha256".to_string(), "left".to_string()),
("crc32".to_string(), "right".to_string()),
]));
left.metadata.insert(
format!("{}{}", http::RUSTFS_INTERNAL_PREFIX, http::SUFFIX_REPLICATION_STATUS),
"replica-a".to_string(),
);
let mut right = left.clone();
right.parts[0].checksums = Some(HashMap::from([
("crc32".to_string(), "right".to_string()),
("sha256".to_string(), "left".to_string()),
]));
right.metadata.insert(
format!("{}{}", http::MINIO_INTERNAL_PREFIX, http::SUFFIX_REPLICATION_STATUS),
"replica-b".to_string(),
);
assert_eq!(
SetDisks::file_info_quorum_hash(&left),
SetDisks::file_info_quorum_hash(&right),
"checksum map ordering and replication metadata must not split quorum identity"
);
right.parts[0]
.checksums
.as_mut()
.expect("checksums should exist")
.insert("sha256".to_string(), "changed".to_string());
assert_ne!(SetDisks::file_info_quorum_hash(&left), SetDisks::file_info_quorum_hash(&right));
}
#[test]
fn quorum_helpers_reject_zero_quorum_and_shuffle_check_parts_by_distribution() {
let err = SetDisks::find_file_info_in_quorum(&[], &None, &None, 0).expect_err("zero quorum cannot select metadata");
assert_eq!(err, DiskError::ErasureReadQuorum);
assert_eq!(SetDisks::shuffle_check_parts(&[2, 1, 0], &[]), vec![2, 1, 0]);
assert_eq!(SetDisks::shuffle_check_parts(&[2, 1, 0], &[3, 1, 2]), vec![1, 0, 2]);
}
#[test]
fn metadata_quorum_uses_simple_majority_for_transitioned_objects() {
let parts_metadata = (1..=6).map(transition_metadata_quorum_fileinfo).collect::<Vec<_>>();
@@ -1274,4 +1353,23 @@ mod tests {
let (d3, _) = SetDisks::shuffle_disks_and_parts_metadata_by_index_owned(disks, parts, &fi);
assert_eq!(d3.len(), slots);
}
#[tokio::test]
async fn shuffle_variants_skip_missing_disks_and_invalid_metadata() {
let tempdir = tempfile::tempdir().expect("tempdir should be created");
let (fi, mut parts) = shuffle_fixture(true);
let mut disks = shuffle_test_disks(&tempdir, parts.len()).await;
disks[0] = None;
parts[1] = FileInfo::default();
let (by_index_disks, by_index_parts) = SetDisks::shuffle_disks_and_parts_metadata_by_index(&disks, &parts, &fi);
assert!(
by_index_disks.iter().filter(|disk| disk.is_some()).count() <= disks.iter().filter(|disk| disk.is_some()).count()
);
assert!(by_index_parts.iter().any(|part| !part.is_valid()));
let (fallback_disks, fallback_parts) = SetDisks::shuffle_disks_and_parts_metadata(&disks, &parts, &fi);
assert!(fallback_disks.iter().any(Option::is_none));
assert!(fallback_parts.iter().any(|part| !part.is_valid()));
}
}
crates/ecstore/src/set_disk/mod.rs
+755 -2
View File
@@ -91,7 +91,7 @@ use crate::storage_api_contracts::{
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
},
namespace::NamespaceLocking as _,
object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
range::HTTPRangeSpec,
};
use crate::store::utils::is_reserved_or_invalid_bucket;
@@ -3562,10 +3562,11 @@ mod tests {
use crate::disk::health_state::RuntimeDriveHealthState;
use crate::disk::new_disk;
use crate::layout::endpoints::SetupType;
use crate::object_api::BLOCK_SIZE_V2;
use crate::object_api::ObjectInfo;
use crate::storage_api_contracts::{
heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart,
namespace::NamespaceLocking as _, object::ObjectOperations as _,
namespace::NamespaceLocking as _, object::ObjectIO as _, object::ObjectOperations as _,
};
use crate::store::init_format::save_format_file;
use crate::store::list_objects::ListPathOptions;
@@ -3579,6 +3580,7 @@ mod tests {
use tempfile::TempDir;
use time::OffsetDateTime;
use tokio::fs;
use tokio::io::AsyncReadExt;
#[test]
fn complete_part_error_maps_confirmed_missing_to_invalid_part() {
@@ -7706,6 +7708,757 @@ mod tests {
assert!(walked_names.iter().any(|name| name == "object"));
}
#[tokio::test]
async fn set_level_put_get_delete_restores_large_object_and_fails_closed_after_delete() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-object-roundtrip";
let object = "nested/object.bin";
let payload = (0..(BLOCK_SIZE_V2 + 17)).map(|idx| (idx % 251) as u8).collect::<Vec<_>>();
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
let written = set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("large object should be written");
assert_eq!(written.size, payload.len() as i64);
let mut get_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
.expect("large object reader should open");
let mut restored = Vec::new();
get_reader
.stream
.read_to_end(&mut restored)
.await
.expect("large object should stream");
assert_eq!(restored, payload);
let deleted = set_disks
.delete_object(bucket, object, opts.clone())
.await
.expect("object delete should succeed");
assert_eq!(deleted.name, object);
let err = match set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
{
Ok(_) => panic!("deleted object must not be readable"),
Err(err) => err,
};
assert!(
is_err_object_not_found(&err),
"deleted object read must fail closed with object-not-found, got {err:?}"
);
}
#[tokio::test]
async fn set_level_batched_large_put_get_restores_body() {
const BATCHED_LARGE_SIZE: usize = 64 * 1024 * 1024;
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-batched-large-object";
let object = "large/batched.bin";
let payload = (0..BATCHED_LARGE_SIZE)
.map(|idx| ((idx as u64).wrapping_mul(31).wrapping_add(17) % 251) as u8)
.collect::<Vec<_>>();
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
let written = set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("batched-large object should be written");
assert_eq!(written.size, payload.len() as i64);
let mut get_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
.expect("batched-large object reader should open");
let mut restored = Vec::with_capacity(payload.len());
get_reader
.stream
.read_to_end(&mut restored)
.await
.expect("batched-large object should stream");
assert_eq!(restored, payload);
}
#[tokio::test]
async fn set_level_put_object_fails_closed_when_writer_quorum_is_unavailable() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-put-writer-quorum";
let object = "object.txt";
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before disk loss");
{
let mut disks = set_disks.disks.write().await;
disks[1] = None;
}
let mut reader = PutObjReader::from_vec(b"quorum guarded body".to_vec());
let err = set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect_err("missing writer quorum must fail the put");
assert!(
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
"expected write quorum failure, got {err:?}"
);
let read_err = match set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
{
Ok(_) => panic!("failed put must not leave a readable object"),
Err(err) => err,
};
assert!(
is_err_object_not_found(&read_err) || matches!(read_err, Error::ErasureReadQuorum),
"failed put must fail closed on read, got {read_err:?}"
);
}
#[tokio::test]
async fn set_level_put_object_short_reader_fails_closed_across_write_paths() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-put-short-reader";
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let cases = [
("single-block", b"short body".to_vec(), 32),
("pipeline", b"pipeline short body".to_vec(), BLOCK_SIZE_V2 as i64 + 1),
("batched-large", b"batched short body".to_vec(), 64 * 1024 * 1024),
];
for (name, payload, declared_size) in cases {
let object = format!("{name}/object.txt");
let hash_reader = HashReader::from_stream(Cursor::new(payload), declared_size, declared_size, None, None, false)
.expect("test reader should be constructed");
let mut reader = PutObjReader::new(hash_reader);
let err = set_disks
.put_object(bucket, &object, &mut reader, &opts)
.await
.expect_err("short reader must fail the put");
let err_text = format!("{err:?}");
assert!(
err_text.contains("UnexpectedEof") || err_text.contains("IncompleteBody"),
"{name} short reader should fail with an EOF/incomplete-body error, got {err:?}"
);
let read_err = match set_disks
.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
.await
{
Ok(_) => panic!("{name} short failed put must not leave a readable object"),
Err(err) => err,
};
assert!(
is_err_object_not_found(&read_err) || matches!(read_err, Error::ErasureReadQuorum),
"{name} short failed put must fail closed on read, got {read_err:?}"
);
}
}
#[tokio::test]
async fn set_level_get_restores_body_when_one_shard_is_missing_after_write() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-read-repair-missing-shard";
let object = "object.bin";
let payload = (0..(BLOCK_SIZE_V2 + 211))
.map(|idx| ((idx * 13) % 251) as u8)
.collect::<Vec<_>>();
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("object should be written before shard loss");
{
let mut disks = set_disks.disks.write().await;
disks[1] = None;
}
let mut get_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
.expect("object should remain readable with one missing shard");
let mut restored = Vec::new();
get_reader
.stream
.read_to_end(&mut restored)
.await
.expect("degraded read should stream");
assert_eq!(restored, payload);
}
#[tokio::test]
async fn set_level_overwrite_restores_new_body_and_cleans_old_data_dir() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-object-overwrite";
let object = "nested/object.bin";
let first_payload = (0..(BLOCK_SIZE_V2 + 31)).map(|idx| (idx % 239) as u8).collect::<Vec<_>>();
let second_payload = (0..(BLOCK_SIZE_V2 + 97))
.map(|idx| ((idx * 7) % 251) as u8)
.collect::<Vec<_>>();
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut first_reader = PutObjReader::from_vec(first_payload);
set_disks
.put_object(bucket, object, &mut first_reader, &opts)
.await
.expect("first object body should be written");
let mut second_reader = PutObjReader::from_vec(second_payload.clone());
let second = set_disks
.put_object(bucket, object, &mut second_reader, &opts)
.await
.expect("overwrite body should commit and clean the old data dir");
assert_eq!(second.size, second_payload.len() as i64);
let mut get_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
.expect("overwritten object reader should open");
let mut restored = Vec::new();
get_reader
.stream
.read_to_end(&mut restored)
.await
.expect("overwritten object should stream");
assert_eq!(restored, second_payload);
}
#[tokio::test]
async fn set_level_write_preconditions_fail_closed_and_allow_matching_etags() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-write-preconditions";
let object = "object.txt";
let write_opts = ObjectOptions {
no_lock: true,
preserve_etag: Some("conditional-etag".to_string()),
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(b"conditional body".to_vec());
set_disks
.put_object(bucket, object, &mut reader, &write_opts)
.await
.expect("object should be written with deterministic etag");
let reject_existing = ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("conditional-etag".to_string()),
..Default::default()
}),
..Default::default()
};
assert!(matches!(
set_disks.check_write_precondition(bucket, object, &reject_existing).await,
Some(StorageError::PreconditionFailed)
));
let allow_matching = ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_match: Some("\"conditional-etag\"".to_string()),
..Default::default()
}),
..Default::default()
};
assert!(
set_disks
.check_write_precondition(bucket, object, &allow_matching)
.await
.is_none()
);
let missing_if_match = ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_match: Some("missing-etag".to_string()),
..Default::default()
}),
..Default::default()
};
assert!(matches!(
set_disks
.check_write_precondition(bucket, "missing-object.txt", &missing_if_match)
.await,
Some(StorageError::ObjectNotFound(_, _))
));
}
#[tokio::test]
async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-versioned-delete";
let object = "object.txt";
let opts = ObjectOptions {
no_lock: true,
versioned: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(b"versioned object body".to_vec());
let written = set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("versioned object should be written");
assert!(written.version_id.is_some());
let marker = set_disks
.delete_object(bucket, object, opts.clone())
.await
.expect("versioned delete should create a marker");
assert!(marker.delete_marker);
assert!(marker.version_id.is_some());
let err = match set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
{
Ok(_) => panic!("latest delete marker must hide object body"),
Err(err) => err,
};
assert!(
is_err_object_not_found(&err) || matches!(err, Error::MethodNotAllowed),
"delete marker read must fail closed, got {err:?}"
);
}
#[tokio::test]
async fn set_level_metadata_self_copy_preserves_body_and_updates_metadata() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-metadata-copy";
let object = "object.txt";
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
let payload = b"metadata copy must not lose committed bytes".to_vec();
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("object should be written");
let mut source = set_disks
.get_object_info(bucket, object, &opts)
.await
.expect("object info should be readable");
let mut metadata = (*source.user_defined).clone();
metadata.insert("x-amz-meta-copy-check".to_string(), "present".to_string());
source.user_defined = Arc::new(metadata);
source.metadata_only = true;
source.etag = Some("metadata-copy-etag".to_string());
let copied = set_disks
.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
.await
.expect("metadata self-copy should succeed");
assert_eq!(copied.user_defined.get("x-amz-meta-copy-check").map(String::as_str), Some("present"));
let mut get_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
.expect("copied object reader should open");
let mut restored = Vec::new();
get_reader
.stream
.read_to_end(&mut restored)
.await
.expect("copied object should stream");
assert_eq!(restored, payload);
let reread = set_disks
.get_object_info(bucket, object, &opts)
.await
.expect("copied object info should be readable");
assert_eq!(reread.user_defined.get("x-amz-meta-copy-check").map(String::as_str), Some("present"));
}
#[tokio::test]
async fn set_level_empty_object_read_uses_buffered_empty_body_with_locks() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-empty-object";
let object = "empty.bin";
let opts = ObjectOptions::default();
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(Vec::new());
let written = set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("empty object should be written");
assert_eq!(written.size, 0);
let mut get_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
.expect("empty object reader should open");
assert_eq!(get_reader.object_info.size, 0);
assert_eq!(get_reader.buffered_body.as_ref().map(Bytes::len), Some(0));
let mut restored = Vec::new();
get_reader
.stream
.read_to_end(&mut restored)
.await
.expect("empty object should stream");
assert!(restored.is_empty());
let info = set_disks
.get_object_info(bucket, object, &opts)
.await
.expect("empty object info should be readable");
assert_eq!(info.size, 0);
}
#[tokio::test]
async fn set_level_put_object_options_preserve_etag_and_normalize_standard_storage_class() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-put-options";
let object = "object.txt";
let mod_time = OffsetDateTime::from_unix_timestamp(1_717_171_717).expect("fixed timestamp should parse");
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD.to_string());
user_defined.insert(SUFFIX_COMPRESSION.to_string(), "zstd".to_string());
let mut eval_metadata = HashMap::new();
eval_metadata.insert("x-amz-meta-evaluated".to_string(), "yes".to_string());
let opts = ObjectOptions {
mod_time: Some(mod_time),
preserve_etag: Some("preserved-etag".to_string()),
user_defined,
eval_metadata: Some(eval_metadata),
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(b"option matrix body".to_vec());
let written = set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("object should be written with option matrix");
assert_eq!(written.etag.as_deref(), Some("preserved-etag"));
assert_eq!(written.mod_time, Some(mod_time));
assert_eq!(written.user_defined.get("x-amz-meta-evaluated").map(String::as_str), Some("yes"));
assert!(!written.user_defined.contains_key(AMZ_STORAGE_CLASS));
let info = set_disks
.get_object_info(bucket, object, &opts)
.await
.expect("object info should be readable");
assert_eq!(info.etag.as_deref(), Some("preserved-etag"));
assert_eq!(info.mod_time, Some(mod_time));
assert_eq!(info.user_defined.get("x-amz-meta-evaluated").map(String::as_str), Some("yes"));
assert!(!info.user_defined.contains_key(AMZ_STORAGE_CLASS));
}
#[tokio::test]
async fn set_level_put_object_metadata_updates_headers_without_rewriting_body() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-put-metadata";
let object = "object.txt";
let payload = b"metadata update must preserve bytes".to_vec();
let write_opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
set_disks
.put_object(bucket, object, &mut reader, &write_opts)
.await
.expect("object should be written");
let mut eval_metadata = HashMap::new();
eval_metadata.insert("x-amz-meta-updated".to_string(), "true".to_string());
let update_time = OffsetDateTime::from_unix_timestamp(1_717_181_818).expect("fixed timestamp should parse");
let update_opts = ObjectOptions {
eval_metadata: Some(eval_metadata),
mod_time: Some(update_time),
..Default::default()
};
let updated = set_disks
.put_object_metadata(bucket, object, &update_opts)
.await
.expect("metadata update should succeed");
assert_eq!(updated.mod_time, Some(update_time));
assert_eq!(updated.user_defined.get("x-amz-meta-updated").map(String::as_str), Some("true"));
let mut get_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &write_opts)
.await
.expect("updated object reader should open");
let mut restored = Vec::new();
get_reader
.stream
.read_to_end(&mut restored)
.await
.expect("updated object should stream");
assert_eq!(restored, payload);
}
#[tokio::test]
async fn set_level_copy_object_with_prefetched_reader_restores_body() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-copy-reader";
let object = "object.txt";
let payload = b"copy reader body".to_vec();
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut source = ObjectInfo {
metadata_only: false,
put_object_reader: Some(PutObjReader::from_vec(payload.clone())),
..Default::default()
};
let copied = set_disks
.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
.await
.expect("copy with prefetched reader should write object data");
assert_eq!(copied.size, payload.len() as i64);
let mut get_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
.expect("copied reader should open");
let mut restored = Vec::new();
get_reader
.stream
.read_to_end(&mut restored)
.await
.expect("copied object should stream");
assert_eq!(restored, payload);
}
#[tokio::test]
async fn set_level_version_suspended_delete_creates_null_delete_marker() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-version-suspended-delete";
let object = "object.txt";
let opts = ObjectOptions {
no_lock: true,
version_suspended: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("suspended-version object should be written");
let marker = set_disks
.delete_object(bucket, object, opts.clone())
.await
.expect("version-suspended delete should create a null marker");
assert!(marker.delete_marker);
assert_eq!(marker.version_id, Some(Uuid::nil()));
let err = match set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
{
Ok(_) => panic!("null delete marker must hide object body"),
Err(err) => err,
};
assert!(
is_err_object_not_found(&err) || matches!(err, Error::MethodNotAllowed),
"null delete marker read must fail closed, got {err:?}"
);
}
#[tokio::test]
async fn set_level_delete_prefix_removes_nested_objects() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-delete-prefix";
let object = "prefix/object.txt";
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(b"prefix body".to_vec());
set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("prefix object should be written");
set_disks
.delete_object(
bucket,
"prefix/",
ObjectOptions {
delete_prefix: true,
no_lock: true,
..Default::default()
},
)
.await
.expect("prefix delete should succeed");
let err = match set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
{
Ok(_) => panic!("object under deleted prefix must not be readable"),
Err(err) => err,
};
assert!(
is_err_object_not_found(&err),
"prefix-deleted object must fail closed with object-not-found, got {err:?}"
);
}
#[tokio::test]
#[serial]
async fn set_level_delete_objects_batch_removes_live_data_and_tolerates_missing_keys() {
temp_env::async_with_vars([("RUSTFS_ISSUE3031_DIAG_ENABLE", Some("true"))], async {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-delete-objects";
let existing = "existing.txt";
let missing = "missing.txt";
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(b"batch delete body".to_vec());
set_disks
.put_object(bucket, existing, &mut reader, &opts)
.await
.expect("object should be written before batch delete");
let (deleted, errors) = set_disks
.delete_objects(
bucket,
vec![
ObjectToDelete {
object_name: existing.to_string(),
..Default::default()
},
ObjectToDelete {
object_name: missing.to_string(),
..Default::default()
},
],
opts.clone(),
)
.await;
assert_eq!(deleted.len(), 2);
assert_eq!(errors.len(), 2);
assert!(errors.iter().all(Option::is_none));
assert_eq!(deleted[0].object_name, existing);
assert!(deleted[0].found);
assert!(!deleted[0].delete_marker);
assert_eq!(deleted[1].object_name, missing);
assert!(!deleted[1].found);
assert!(!deleted[1].delete_marker);
let err = match set_disks
.get_object_reader(bucket, existing, None, HeaderMap::new(), &opts)
.await
{
Ok(_) => panic!("batch-deleted object must not be readable as latest"),
Err(err) => err,
};
assert!(
is_err_object_not_found(&err),
"batch-deleted object must fail closed with object-not-found, got {err:?}"
);
})
.await;
}
#[tokio::test]
async fn set_level_heal_format_repairs_unformatted_disk() {
let set_disks = make_local_bucket_test_set_disks_with_missing_format().await;
crates/ecstore/src/set_disk/read.rs
+594
View File
@@ -1641,6 +1641,21 @@ mod metadata_cache_tests {
.await
}
async fn new_read_version_test_disk(bucket: &str) -> (tempfile::TempDir, DiskStore) {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint =
crate::layout::endpoint::Endpoint::try_from(dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
let local_disk = crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("local disk should open");
let disk: DiskStore = Arc::new(crate::disk::Disk::Local(Box::new(crate::disk::disk_store::LocalDiskWrapper::new(
Arc::new(local_disk),
false,
))));
disk.make_volume(bucket).await.expect("bucket volume should be created");
(dir, disk)
}
#[test]
#[serial]
fn get_object_metadata_cache_capacity_uses_default_and_env_override() {
@@ -1667,6 +1682,249 @@ mod metadata_cache_tests {
fi
}
#[tokio::test]
async fn get_object_with_fileinfo_rejects_invalid_ranges_before_reader_setup() {
let bucket = "bucket";
let object = "object";
let mut output = Vec::new();
let err = SetDisks::get_object_with_fileinfo(
bucket,
object,
2,
1,
&mut output,
valid_test_fileinfo(object),
Vec::new(),
&[],
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
)
.await
.expect_err("offset beyond object size must fail before reader setup");
assert!(err.to_string().contains("offset out of range"));
let mut overflow = valid_test_fileinfo(object);
overflow.size = -1;
let err = SetDisks::get_object_with_fileinfo(
bucket,
object,
usize::MAX,
1,
&mut output,
overflow,
Vec::new(),
&[],
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
)
.await
.expect_err("offset plus length overflow must fail before reader setup");
assert!(err.to_string().contains("offset out of range"));
let err = SetDisks::get_object_with_fileinfo(
bucket,
object,
1,
1,
&mut output,
valid_test_fileinfo(object),
Vec::new(),
&[],
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
)
.await
.expect_err("end offset beyond object size must fail before reader setup");
assert!(err.to_string().contains("offset out of range"));
let mut invalid_erasure = valid_test_fileinfo(object);
invalid_erasure.erasure.block_size = 0;
let err = SetDisks::get_object_with_fileinfo(
bucket,
object,
0,
1,
&mut output,
invalid_erasure,
Vec::new(),
&[],
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
)
.await
.expect_err("invalid erasure metadata must fail before reader setup");
assert!(err.to_string().contains("invalid erasure metadata"));
assert!(output.is_empty());
}
#[tokio::test]
async fn get_object_with_fileinfo_fails_closed_without_read_quorum() {
let bucket = "bucket";
let object = "object";
let mut fi = valid_test_fileinfo(object);
fi.erasure.block_size = 1;
fi.erasure.distribution = vec![1, 2, 3, 4];
fi.parts.push(ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
});
let mut output = Vec::new();
let err = SetDisks::get_object_with_fileinfo(
bucket,
object,
0,
1,
&mut output,
fi,
Vec::new(),
&[],
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
)
.await
.expect_err("object read must fail closed when no shards can be read");
let err = err.to_string().to_ascii_lowercase();
assert!(!err.is_empty(), "read should fail with a concrete error");
assert!(output.is_empty());
}
#[tokio::test]
async fn read_version_optimized_reads_local_metadata_and_fails_closed_without_quorum() {
let bucket = "read-version-optimized-bucket";
let object = "object";
let (_dir, disk) = new_read_version_test_disk(bucket).await;
let mut fi = valid_test_fileinfo(object);
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, object, fi.clone())
.await
.expect("metadata should be written before optimized read");
let set = SetDisks::new(
"read-version-optimized-test".to_string(),
Arc::new(RwLock::new(vec![Some(disk)])),
1,
0,
0,
0,
Vec::new(),
FormatV3::new(1, 1),
Vec::new(),
)
.await;
let versions = set
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect("single readable disk should satisfy optimized read quorum");
assert_eq!(versions.len(), 1);
assert_eq!(versions[0].name, object);
assert_eq!(versions[0].metadata.get("etag").map(String::as_str), Some("etag-1"));
let missing_quorum = new_metadata_cache_test_set()
.await
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect_err("empty disk set must fail closed");
assert!(
missing_quorum.to_string().to_ascii_lowercase().contains("file"),
"optimized read failure should map to file-not-found style error: {missing_quorum}"
);
}
#[tokio::test]
async fn get_object_info_and_quorum_maps_delete_marker_and_purge_states() {
let bucket = "get-object-info-marker-bucket";
let (_dir, disk) = new_read_version_test_disk(bucket).await;
let set = SetDisks::new(
"get-object-info-marker-test".to_string(),
Arc::new(RwLock::new(vec![Some(disk.clone())])),
1,
0,
0,
0,
Vec::new(),
FormatV3::new(1, 1),
Vec::new(),
)
.await;
let latest_marker = FileInfo {
volume: bucket.to_string(),
name: "latest-delete-marker".to_string(),
version_id: Some(Uuid::new_v4()),
deleted: true,
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
disk.write_metadata(bucket, bucket, "latest-delete-marker", latest_marker)
.await
.expect("latest marker metadata should be written");
let (_, _, latest_err) = set
.get_object_info_and_quorum(bucket, "latest-delete-marker", &ObjectOptions::default())
.await;
assert!(
matches!(latest_err, Some(StorageError::ObjectNotFound(_, _))),
"latest delete marker should hide the object, got {latest_err:?}"
);
let marker_version = Uuid::new_v4();
let version_marker = FileInfo {
volume: bucket.to_string(),
name: "version-delete-marker".to_string(),
version_id: Some(marker_version),
deleted: true,
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
disk.write_metadata(bucket, bucket, "version-delete-marker", version_marker)
.await
.expect("version marker metadata should be written");
let (_, _, version_err) = set
.get_object_info_and_quorum(
bucket,
"version-delete-marker",
&ObjectOptions {
version_id: Some(marker_version.to_string()),
..Default::default()
},
)
.await;
assert!(
matches!(version_err, Some(StorageError::MethodNotAllowed)),
"explicit delete marker read should be method-not-allowed, got {version_err:?}"
);
}
#[test]
fn get_object_metadata_cache_request_eligibility_is_conservative() {
let opts = ObjectOptions::default();
@@ -2055,6 +2313,26 @@ mod tests {
const CODEC_STREAMING_TEST_BUCKET: &str = "bucket";
const CODEC_STREAMING_TEST_OBJECT: &str = "object";
async fn local_test_disks(count: usize, bucket: &str) -> (Vec<tempfile::TempDir>, Vec<Option<crate::disk::DiskStore>>) {
let mut dirs = Vec::with_capacity(count);
let mut disks = Vec::with_capacity(count);
for _ in 0..count {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = crate::layout::endpoint::Endpoint::try_from(dir.path().to_string_lossy().as_ref())
.expect("endpoint should parse");
let local_disk = crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("local disk should open");
let disk: crate::disk::DiskStore = Arc::new(crate::disk::Disk::Local(Box::new(
crate::disk::disk_store::LocalDiskWrapper::new(Arc::new(local_disk), false),
)));
disk.make_volume(bucket).await.expect("bucket volume should be created");
dirs.push(dir);
disks.push(Some(disk));
}
(dirs, disks)
}
#[test]
fn shard_read_cost_for_endpoint_maps_topology_classes() {
let local_hosts = vec!["node-a:9000".to_string()];
@@ -2772,6 +3050,299 @@ mod tests {
assert!(result.is_err(), "zero block_size metadata must be rejected, not panic");
}
#[tokio::test]
async fn codec_streaming_fileinfo_rejects_invalid_size_and_multipart_mismatch() {
let mut single_part = codec_streaming_test_fileinfo(8, 1);
single_part.size = -1;
let invalid_size = SetDisks::get_object_decode_reader_with_fileinfo(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
&single_part,
&[],
&[],
0,
0,
false,
"test-object-class",
"test-size-bucket",
false,
)
.await;
assert!(invalid_size.is_err(), "negative object size must reject before reader construction");
let mut multipart = codec_streaming_test_fileinfo(17, 2);
multipart.parts[0].size = 8;
multipart.parts[1].size = 8;
let mismatch = temp_env::async_with_vars([(ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, Some("true"))], async {
SetDisks::get_object_decode_reader_with_fileinfo(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
&multipart,
&[],
&[],
0,
0,
false,
"test-object-class",
"test-size-bucket",
false,
)
.await
})
.await;
assert!(mismatch.is_err(), "multipart part sizes must match object size");
}
#[tokio::test]
async fn codec_streaming_fileinfo_reports_multipart_fallbacks_at_entry() {
let multipart = codec_streaming_test_fileinfo(16, 2);
let disabled = temp_env::async_with_vars([(ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, Some("false"))], async {
SetDisks::get_object_decode_reader_with_fileinfo(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
&multipart,
&[],
&[],
0,
0,
false,
"test-object-class",
"test-size-bucket",
false,
)
.await
})
.await
.expect("multipart disabled should return a structured fallback");
assert!(matches!(
disabled,
GetCodecStreamingReaderBuildOutcome::Fallback(GetCodecStreamingFallbackReason::Multipart)
));
let part_limit = temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS, Some("1")),
],
async {
SetDisks::get_object_decode_reader_with_fileinfo(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
&multipart,
&[],
&[],
0,
0,
false,
"test-object-class",
"test-size-bucket",
false,
)
.await
},
)
.await
.expect("multipart part limit should return a structured fallback");
assert!(matches!(
part_limit,
GetCodecStreamingReaderBuildOutcome::Fallback(GetCodecStreamingFallbackReason::MultipartPartLimit)
));
}
#[tokio::test]
async fn codec_streaming_fileinfo_builds_lazy_multipart_reader_from_inline_shards() {
let part_data = b"abcdefgh";
let erasure = coding::Erasure::new(4, 2, part_data.len());
let mut fi = codec_streaming_test_fileinfo(16, 2);
fi.erasure.block_size = part_data.len();
fi.erasure.distribution = (1..=erasure.total_shard_count()).collect();
for part in &mut fi.parts {
part.size = part_data.len();
part.actual_size = i64::try_from(part_data.len()).expect("test part size should fit i64");
}
let files = codec_streaming_inline_files(&erasure, part_data).await;
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint =
crate::layout::endpoint::Endpoint::try_from(dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
let local_disk = crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("local disk should open");
let disk: crate::disk::DiskStore = Arc::new(crate::disk::Disk::Local(Box::new(
crate::disk::disk_store::LocalDiskWrapper::new(Arc::new(local_disk), false),
)));
let disks = vec![Some(disk); files.len()];
let outcome = temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS, Some("4")),
],
async {
SetDisks::get_object_decode_reader_with_fileinfo(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
&fi,
&files,
&disks,
0,
0,
false,
"test-object-class",
"test-size-bucket",
false,
)
.await
},
)
.await
.expect("lazy multipart reader should be constructed");
let GetCodecStreamingReaderBuildOutcome::Reader(mut reader) = outcome else {
panic!("expected lazy multipart reader");
};
let mut body = Vec::new();
reader
.read_to_end(&mut body)
.await
.expect("lazy multipart reader should restore both parts");
assert_eq!(body, [part_data.as_slice(), part_data.as_slice()].concat());
}
#[tokio::test]
#[serial_test::serial]
async fn codec_streaming_fileinfo_builds_single_part_reader_from_inline_shards() {
let part_data = b"abcdefgh";
let erasure = coding::Erasure::new(4, 2, part_data.len());
let mut fi = codec_streaming_test_fileinfo(part_data.len() as i64, 1);
fi.erasure.block_size = part_data.len();
fi.erasure.distribution = (1..=erasure.total_shard_count()).collect();
fi.parts[0].size = part_data.len();
fi.parts[0].actual_size = i64::try_from(part_data.len()).expect("test part size should fit i64");
let files = codec_streaming_inline_files(&erasure, part_data).await;
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint =
crate::layout::endpoint::Endpoint::try_from(dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
let local_disk = crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("local disk should open");
let disk: crate::disk::DiskStore = Arc::new(crate::disk::Disk::Local(Box::new(
crate::disk::disk_store::LocalDiskWrapper::new(Arc::new(local_disk), false),
)));
let disks = vec![Some(disk); files.len()];
let outcome = temp_env::async_with_vars([("RUSTFS_SHARD_LOCALITY_SCHEDULING", Some("on"))], async {
SetDisks::get_object_decode_reader_with_fileinfo(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
&fi,
&files,
&disks,
0,
0,
false,
"test-object-class",
"test-size-bucket",
false,
)
.await
})
.await
.expect("single part reader should be constructed");
let GetCodecStreamingReaderBuildOutcome::Reader(mut reader) = outcome else {
panic!("expected single part reader");
};
let mut body = Vec::new();
reader
.read_to_end(&mut body)
.await
.expect("single part codec reader should restore payload");
assert_eq!(body, part_data);
}
#[tokio::test]
async fn get_object_with_fileinfo_restores_missing_inline_data_shard_and_submits_repair() {
let part_data = b"abcdefgh";
let erasure = coding::Erasure::new(4, 2, part_data.len());
let mut fi = codec_streaming_test_fileinfo(part_data.len() as i64, 1);
fi.erasure.block_size = part_data.len();
fi.erasure.distribution = (1..=erasure.total_shard_count()).collect();
fi.parts[0].size = part_data.len();
fi.parts[0].actual_size = i64::try_from(part_data.len()).expect("test part size should fit i64");
let mut files = codec_streaming_inline_files(&erasure, part_data).await;
files[0].data = None;
let (_dirs, disks) = local_test_disks(files.len(), CODEC_STREAMING_TEST_BUCKET).await;
let mut output = Vec::new();
SetDisks::get_object_with_fileinfo(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
0,
part_data.len() as i64,
&mut output,
fi,
files,
&disks,
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"test-object-class",
"test-size-bucket",
)
.await
.expect("missing data shard should be reconstructed from parity");
assert_eq!(output, part_data);
}
#[tokio::test]
async fn codec_streaming_part_reader_rejects_oversized_part_and_missing_quorum() {
let erasure = coding::Erasure::new(4, 2, 8);
let fi = codec_streaming_test_fileinfo(8, 1);
let oversized = SetDisks::build_codec_streaming_part_reader(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
&fi,
&[],
&[],
&erasure,
1,
0,
9,
8,
false,
"test-object-class",
"test-size-bucket",
false,
)
.await;
assert!(oversized.is_err(), "part_length > part_size must be rejected");
let missing_quorum = SetDisks::build_codec_streaming_part_reader(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
&fi,
&[],
&[],
&erasure,
1,
0,
8,
8,
false,
"test-object-class",
"test-size-bucket",
false,
)
.await;
assert!(missing_quorum.is_err(), "reader setup must fail closed when no shard can answer");
}
#[tokio::test]
async fn multipart_codec_streaming_reader_reads_parts_in_order() {
let readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>> = vec![
@@ -3190,6 +3761,29 @@ mod tests {
Ok(decoded)
}
async fn codec_streaming_inline_files(erasure: &coding::Erasure, part_data: &'static [u8]) -> Vec<FileInfo> {
let shards = erasure.encode_data(part_data).expect("test part should encode");
let distribution = (1..=erasure.total_shard_count()).collect::<Vec<_>>();
let mut files = Vec::with_capacity(shards.len());
for shard in shards {
let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), erasure.shard_size(), HashAlgorithm::HighwayHash256S);
writer.write(&shard).await.expect("test shard should write with bitrot hash");
let mut fi = FileInfo::new(CODEC_STREAMING_TEST_OBJECT, erasure.data_shards, erasure.parity_shards);
fi.volume = CODEC_STREAMING_TEST_BUCKET.to_string();
fi.name = CODEC_STREAMING_TEST_OBJECT.to_string();
fi.size = i64::try_from(part_data.len()).expect("test part size should fit i64");
fi.erasure.block_size = erasure.block_size;
fi.erasure.index = files.len() + 1;
fi.erasure.distribution = distribution.clone();
fi.data = Some(Bytes::from(writer.into_inner().into_inner()));
files.push(fi);
}
files
}
#[tokio::test]
async fn bitrot_reader_setup_stops_at_read_quorum() {
let setup = setup_inline_bitrot_readers(
crates/ecstore/src/set_disk/shard_source.rs
+12
View File
@@ -240,6 +240,18 @@ mod tests {
assert_eq!(ShardReadCost::SameNode.as_str(), GET_SHARD_READ_COST_SAME_NODE);
}
#[test]
fn shard_read_cost_reports_all_labels_and_remote_classification() {
assert_eq!(ShardReadCost::Local.as_str(), GET_SHARD_READ_COST_LOCAL);
assert_eq!(ShardReadCost::SameNode.as_str(), GET_SHARD_READ_COST_SAME_NODE);
assert_eq!(ShardReadCost::Remote.as_str(), GET_SHARD_READ_COST_REMOTE);
assert_eq!(ShardReadCost::Unknown.as_str(), GET_SHARD_READ_COST_UNKNOWN);
assert!(ShardReadCost::Remote.is_remote());
assert!(!ShardReadCost::Local.is_remote());
assert!(!ShardReadCost::Unknown.is_low_cost());
}
#[test]
fn stripe_read_state_reports_complete_data_shards_without_parity() {
let state = StripeReadState::from_parts(vec![Some(vec![1]), Some(vec![2]), None], Vec::new(), 2);