mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 23:47:28 +00:00
test(ecstore): complete EC validation coverage gate
* test(ecstore): complete EC validation coverage gate * test(ecstore): stabilize validation suite after rebase * test(ecstore): fix rio-v2 clippy lint
This commit is contained in:
Zhengchao An
@@ -0,0 +1,434 @@
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::format::FormatV3;
|
||||
use crate::disk::{DiskAPI, DiskOption, DiskStore, WalkDirOptions, new_disk};
|
||||
use crate::error::Error;
|
||||
use crate::io_support::rio::HashReader;
|
||||
use crate::object_api::{BLOCK_SIZE_V2, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use crate::store::init_format::save_format_file;
|
||||
use http::HeaderMap;
|
||||
use rustfs_filemeta::{MetacacheReader, MetacacheWriter};
|
||||
use std::io::Cursor;
|
||||
use std::mem;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> Arc<SetDisks> {
|
||||
let format = FormatV3::new(1, drive_count);
|
||||
let mut endpoints = Vec::with_capacity(drive_count);
|
||||
let mut disks = Vec::with_capacity(drive_count);
|
||||
|
||||
for disk_idx in 0..drive_count {
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_idx);
|
||||
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("disk should be created");
|
||||
|
||||
let mut disk_format = format.clone();
|
||||
disk_format.erasure.this = format.erasure.sets[0][disk_idx];
|
||||
save_format_file(&Some(disk.clone()), &Some(disk_format))
|
||||
.await
|
||||
.expect("format should be saved");
|
||||
|
||||
mem::forget(dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
SetDisks::new(
|
||||
"ecstore-validation-blackbox".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
drive_count,
|
||||
parity_count,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
format,
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn first_shard_part_path(disk: &DiskStore, bucket: &str, object: &str) -> PathBuf {
|
||||
let object_dir = disk.path().join(bucket).join(object);
|
||||
let mut entries = fs::read_dir(&object_dir)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("object data dir should be readable at {object_dir:?}: {err}"));
|
||||
while let Some(entry) = entries.next_entry().await.expect("object data dir entry should be readable") {
|
||||
if entry
|
||||
.file_type()
|
||||
.await
|
||||
.expect("object data entry type should be readable")
|
||||
.is_dir()
|
||||
{
|
||||
let part_path = entry.path().join("part.1");
|
||||
if fs::metadata(&part_path).await.is_ok() {
|
||||
return part_path;
|
||||
}
|
||||
}
|
||||
}
|
||||
panic!("part.1 shard should exist under {object_dir:?}");
|
||||
}
|
||||
|
||||
async fn shard_part_paths(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) -> Vec<PathBuf> {
|
||||
let disks = set_disks.disks.read().await;
|
||||
let mut paths = Vec::new();
|
||||
for disk in disks.iter().flatten() {
|
||||
paths.push(first_shard_part_path(disk, bucket, object).await);
|
||||
}
|
||||
paths
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blackbox_put_unknown_actual_size_restores_body_and_records_written_size() {
|
||||
let set_disks = make_local_set_disks(4, 2).await;
|
||||
let bucket = "bb-unknown-actual-size";
|
||||
let object = "object.bin";
|
||||
let payload = (0..(BLOCK_SIZE_V2 + 123))
|
||||
.map(|idx| ((idx * 17) % 251) as u8)
|
||||
.collect::<Vec<_>>();
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let stream = HashReader::from_stream(Cursor::new(payload.clone()), payload.len() as i64, -1, None, None, false)
|
||||
.expect("hash reader should accept unknown actual size");
|
||||
let mut reader = PutObjReader::new(stream);
|
||||
let written = set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
assert_eq!(written.size, payload.len() as i64);
|
||||
assert_eq!(written.parts[0].actual_size, payload.len() as i64);
|
||||
|
||||
let mut get_reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("object reader should open");
|
||||
let mut restored = Vec::new();
|
||||
get_reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("object should stream");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blackbox_get_restores_body_after_one_shard_file_is_removed() {
|
||||
let set_disks = make_local_set_disks(4, 2).await;
|
||||
let bucket = "bb-missing-shard-file";
|
||||
let object = "object.bin";
|
||||
let payload = (0..(BLOCK_SIZE_V2 + 321))
|
||||
.map(|idx| ((idx * 19) % 251) as u8)
|
||||
.collect::<Vec<_>>();
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let disk = {
|
||||
let disks = set_disks.disks.read().await;
|
||||
disks[0].clone().expect("first disk should exist")
|
||||
};
|
||||
let shard_path = first_shard_part_path(&disk, bucket, object).await;
|
||||
fs::remove_file(&shard_path)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("test shard should be removable at {shard_path:?}: {err}"));
|
||||
|
||||
let mut get_reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("object should remain readable after one shard file is removed");
|
||||
let mut restored = Vec::new();
|
||||
get_reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("degraded object should stream");
|
||||
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blackbox_get_restores_body_and_enqueues_repair_after_one_corrupt_shard() {
|
||||
let set_disks = make_local_set_disks(4, 2).await;
|
||||
let bucket = "bb-corrupt-shard-repair";
|
||||
let object = "object.bin";
|
||||
let payload = (0..(BLOCK_SIZE_V2 + 777))
|
||||
.map(|idx| ((idx * 29) % 251) as u8)
|
||||
.collect::<Vec<_>>();
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let paths = shard_part_paths(&set_disks, bucket, object).await;
|
||||
fs::write(&paths[0], b"corrupt shard bytes")
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("test shard should be corruptible at {:?}: {err}", paths[0]));
|
||||
|
||||
let mut get_reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("object should remain readable after one corrupt shard");
|
||||
let mut restored = Vec::new();
|
||||
get_reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("corrupt-shard object should stream from parity");
|
||||
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blackbox_range_read_restores_exact_slice_with_one_offline_disk() {
|
||||
let set_disks = make_local_set_disks(4, 2).await;
|
||||
let bucket = "bb-range-offline-disk";
|
||||
let object = "object.bin";
|
||||
let payload = (0..(BLOCK_SIZE_V2 + 4096))
|
||||
.map(|idx| ((idx * 23) % 251) as u8)
|
||||
.collect::<Vec<_>>();
|
||||
let range_start = 513usize;
|
||||
let range_len = 8192usize;
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
{
|
||||
let mut disks = set_disks.disks.write().await;
|
||||
disks[2] = None;
|
||||
}
|
||||
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: range_start as i64,
|
||||
end: (range_start + range_len - 1) as i64,
|
||||
};
|
||||
let mut get_reader = set_disks
|
||||
.get_object_reader(bucket, object, Some(range), HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("range reader should open with one offline disk");
|
||||
let mut restored = Vec::new();
|
||||
get_reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("range body should stream");
|
||||
|
||||
assert_eq!(restored, payload[range_start..range_start + range_len]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_metadata() {
|
||||
let set_disks = make_local_set_disks(4, 2).await;
|
||||
let bucket = "bb-delete-marker-read-negative";
|
||||
let object = "object.bin";
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"body hidden by delete marker".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.await
|
||||
.expect("object should be written before delete marker");
|
||||
|
||||
let marker = set_disks
|
||||
.delete_object(bucket, object, opts.clone())
|
||||
.await
|
||||
.expect("version suspended delete should create a marker");
|
||||
assert!(marker.delete_marker);
|
||||
|
||||
let err = match set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("delete marker must hide the object body"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(
|
||||
matches!(err, Error::ObjectNotFound(_, _) | Error::MethodNotAllowed),
|
||||
"delete marker read must fail closed, got {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blackbox_local_disk_walk_dir_emits_metadata_entries_with_prefix_forward_and_limit() {
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let bucket = "bb-local-walk";
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("disk should be created");
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
|
||||
for object in ["prefix/a", "prefix/b", "prefix/c", "other/d"] {
|
||||
disk.write_all(bucket, &format!("{object}/xl.meta"), bytes::Bytes::from_static(b"meta"))
|
||||
.await
|
||||
.expect("metadata object should be written");
|
||||
}
|
||||
|
||||
let (reader, mut writer) = tokio::io::duplex(4096);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: "prefix/".to_string(),
|
||||
recursive: true,
|
||||
forward_to: Some("prefix/b".to_string()),
|
||||
limit: 2,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
disk.walk_dir(opts, &mut writer)
|
||||
.await
|
||||
.expect("walk_dir should stream metadata entries");
|
||||
MetacacheWriter::new(&mut writer)
|
||||
.close()
|
||||
.await
|
||||
.expect("metacache stream should close");
|
||||
drop(writer);
|
||||
|
||||
let mut reader = MetacacheReader::new(reader);
|
||||
let entries = reader.read_all().await.expect("metacache stream should decode");
|
||||
let names = entries
|
||||
.into_iter()
|
||||
.filter(|entry| !entry.metadata.is_empty())
|
||||
.map(|entry| entry.name)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(names, vec!["prefix/b".to_string(), "prefix/c".to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn blackbox_issue3031_diag_covers_put_success_cleanup_and_error_summary() {
|
||||
temp_env::async_with_vars([("RUSTFS_ISSUE3031_DIAG_ENABLE", Some("true"))], async {
|
||||
let set_disks = make_local_set_disks(4, 2).await;
|
||||
let bucket = "bb-issue3031-diag";
|
||||
let object = "object.bin";
|
||||
let first_payload = b"first diagnostic body".to_vec();
|
||||
let second_payload = b"second diagnostic body that replaces the first".to_vec();
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let mut first_reader = PutObjReader::from_vec(first_payload);
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut first_reader, &opts)
|
||||
.await
|
||||
.expect("first diagnostic write should succeed");
|
||||
|
||||
let mut second_reader = PutObjReader::from_vec(second_payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut second_reader, &opts)
|
||||
.await
|
||||
.expect("overwrite should succeed and report cleanup-present diagnostics");
|
||||
|
||||
let mut get_reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("overwritten object reader should open");
|
||||
let mut restored = Vec::new();
|
||||
get_reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("overwritten object should stream");
|
||||
assert_eq!(restored, second_payload);
|
||||
|
||||
let failed_set = make_local_set_disks(1, 0).await;
|
||||
let failed_bucket = "bb-issue3031-fail";
|
||||
failed_set
|
||||
.make_bucket(failed_bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("failure bucket should be created");
|
||||
{
|
||||
let mut disks = failed_set.disks.write().await;
|
||||
disks[0] = None;
|
||||
}
|
||||
let mut failed_reader = PutObjReader::from_vec(b"must fail closed".to_vec());
|
||||
let err = failed_set
|
||||
.put_object(failed_bucket, "object.bin", &mut failed_reader, &opts)
|
||||
.await
|
||||
.expect_err("offline only disk must fail writer setup under diagnostics");
|
||||
assert!(
|
||||
matches!(err, Error::InsufficientWriteQuorum(_, _) | Error::ErasureWriteQuorum),
|
||||
"diagnostic failure should still report a write-quorum style error, got {err:?}"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
Reference in New Issue
Block a user