mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-28 07:57:01 +00:00
fix(lifecycle): prevent eager date-expiry deletion on config update (#2708)
This commit is contained in:
houseme
@@ -13,6 +13,87 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
async fn collect_list_parts_results<F>(
|
||||
tasks: Vec<F>,
|
||||
read_quorum: usize,
|
||||
) -> disk::error::Result<(Vec<Option<DiskError>>, Vec<Vec<String>>)>
|
||||
where
|
||||
F: Future<Output = disk::error::Result<Vec<String>>> + Send + 'static,
|
||||
{
|
||||
let mut errs = vec![Some(DiskError::DiskNotFound); tasks.len()];
|
||||
let mut object_parts = vec![Vec::new(); tasks.len()];
|
||||
let mut successful_responses = 0usize;
|
||||
let mut pending = tasks.len();
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
join_set.spawn(async move { (index, task.await) });
|
||||
}
|
||||
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
pending = pending.saturating_sub(1);
|
||||
|
||||
match join_result {
|
||||
Ok((index, Ok(parts))) => {
|
||||
errs[index] = None;
|
||||
object_parts[index] = parts;
|
||||
successful_responses += 1;
|
||||
}
|
||||
Ok((index, Err(err))) => {
|
||||
errs[index] = Some(err);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
if successful_responses + pending < read_quorum {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
}
|
||||
|
||||
Ok((errs, object_parts))
|
||||
}
|
||||
|
||||
fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize) -> Vec<usize> {
|
||||
let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
for drive_parts in object_parts {
|
||||
let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
// part files can be either part.N or part.N.meta
|
||||
for part_path in drive_parts {
|
||||
if let Some(num_str) = part_path.strip_prefix("part.") {
|
||||
if let Some(meta_idx) = num_str.find(".meta") {
|
||||
if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
} else if let Ok(part_num) = num_str.parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Include only part.N.meta files with corresponding part.N
|
||||
for (&part_num, &cnt) in &parts_with_meta_count {
|
||||
if cnt >= 2 {
|
||||
*part_quorum_map.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
|
||||
for (part_num, count) in part_quorum_map {
|
||||
if count >= read_quorum {
|
||||
part_numbers.push(part_num);
|
||||
}
|
||||
}
|
||||
|
||||
part_numbers.sort();
|
||||
part_numbers
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn list_parts(
|
||||
@@ -21,10 +102,13 @@ impl SetDisks {
|
||||
read_quorum: usize,
|
||||
) -> disk::error::Result<Vec<usize>> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
let part_path = part_path.to_string();
|
||||
for disk in disks.iter() {
|
||||
let disk = disk.clone();
|
||||
let part_path = part_path.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
|
||||
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
@@ -35,60 +119,15 @@ impl SetDisks {
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
let mut object_parts = Vec::with_capacity(disks.len());
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
errs.push(None);
|
||||
object_parts.push(res);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
object_parts.push(vec![]);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (collected_errs, collected_parts) = collect_list_parts_results(futures, read_quorum).await?;
|
||||
errs.extend(collected_errs);
|
||||
object_parts.extend(collected_parts);
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
for drive_parts in object_parts {
|
||||
let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
// part files can be either part.N or part.N.meta
|
||||
for part_path in drive_parts {
|
||||
if let Some(num_str) = part_path.strip_prefix("part.") {
|
||||
if let Some(meta_idx) = num_str.find(".meta") {
|
||||
if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
} else if let Ok(part_num) = num_str.parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Include only part.N.meta files with corresponding part.N
|
||||
for (&part_num, &cnt) in &parts_with_meta_count {
|
||||
if cnt >= 2 {
|
||||
*part_quorum_map.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
|
||||
for (part_num, count) in part_quorum_map {
|
||||
if count >= read_quorum {
|
||||
part_numbers.push(part_num);
|
||||
}
|
||||
}
|
||||
|
||||
part_numbers.sort();
|
||||
|
||||
Ok(part_numbers)
|
||||
Ok(reduce_quorum_part_numbers(object_parts, read_quorum))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
@@ -145,3 +184,78 @@ impl SetDisks {
|
||||
Ok((fi, parts_metadata))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_list_parts_results_fails_early_when_quorum_is_impossible() {
|
||||
let started = std::time::Instant::now();
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Err(DiskError::DiskNotFound)),
|
||||
(15, Err(DiskError::DiskNotFound)),
|
||||
(250, Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let err = collect_list_parts_results(tasks, 2)
|
||||
.await
|
||||
.expect_err("quorum should become impossible before slow tail completes");
|
||||
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
assert!(started.elapsed() < Duration::from_millis(120));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_list_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
|
||||
let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, should_panic)| async move {
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
if should_panic {
|
||||
panic!("simulated task panic");
|
||||
}
|
||||
Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (errs, object_parts) = collect_list_parts_results(tasks, 2)
|
||||
.await
|
||||
.expect("quorum should still succeed");
|
||||
assert_eq!(errs.iter().filter(|err| err.is_none()).count(), 2);
|
||||
assert_eq!(object_parts.iter().filter(|parts| !parts.is_empty()).count(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduce_quorum_part_numbers_only_keeps_parts_present_on_quorum_of_drives() {
|
||||
let object_parts = vec![
|
||||
vec![
|
||||
"part.1".to_string(),
|
||||
"part.1.meta".to_string(),
|
||||
"part.2".to_string(),
|
||||
"part.2.meta".to_string(),
|
||||
],
|
||||
vec![
|
||||
"part.1".to_string(),
|
||||
"part.1.meta".to_string(),
|
||||
"part.3".to_string(),
|
||||
"part.3.meta".to_string(),
|
||||
],
|
||||
vec![
|
||||
"part.1".to_string(),
|
||||
"part.1.meta".to_string(),
|
||||
"part.2".to_string(),
|
||||
"part.2.meta".to_string(),
|
||||
],
|
||||
];
|
||||
|
||||
let parts = reduce_quorum_part_numbers(object_parts, 2);
|
||||
assert_eq!(parts, vec![1, 2]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,88 @@
|
||||
|
||||
use super::*;
|
||||
use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
|
||||
use std::future::Future;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
async fn collect_read_multiple_results<F>(
|
||||
tasks: Vec<F>,
|
||||
read_quorum: usize,
|
||||
) -> std::result::Result<(Vec<Option<Vec<ReadMultipleResp>>>, Vec<Option<DiskError>>), ()>
|
||||
where
|
||||
F: Future<Output = disk::error::Result<Vec<ReadMultipleResp>>> + Send + 'static,
|
||||
{
|
||||
let mut responses = vec![None; tasks.len()];
|
||||
let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
|
||||
let mut successful_responses = 0usize;
|
||||
let mut pending = tasks.len();
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
join_set.spawn(async move { (index, task.await) });
|
||||
}
|
||||
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
pending = pending.saturating_sub(1);
|
||||
|
||||
match join_result {
|
||||
Ok((index, Ok(resp))) => {
|
||||
responses[index] = Some(resp);
|
||||
errors[index] = None;
|
||||
successful_responses += 1;
|
||||
}
|
||||
Ok((index, Err(err))) => {
|
||||
errors[index] = Some(err);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
if successful_responses + pending < read_quorum {
|
||||
return Err(());
|
||||
}
|
||||
}
|
||||
|
||||
Ok((responses, errors))
|
||||
}
|
||||
|
||||
async fn collect_read_parts_results<F>(
|
||||
tasks: Vec<F>,
|
||||
read_quorum: usize,
|
||||
) -> std::result::Result<(Vec<Option<Vec<ObjectPartInfo>>>, Vec<Option<DiskError>>), ()>
|
||||
where
|
||||
F: Future<Output = disk::error::Result<Vec<ObjectPartInfo>>> + Send + 'static,
|
||||
{
|
||||
let mut responses = vec![None; tasks.len()];
|
||||
let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
|
||||
let mut successful_responses = 0usize;
|
||||
let mut pending = tasks.len();
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
join_set.spawn(async move { (index, task.await) });
|
||||
}
|
||||
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
pending = pending.saturating_sub(1);
|
||||
|
||||
match join_result {
|
||||
Ok((index, Ok(resp))) => {
|
||||
responses[index] = Some(resp);
|
||||
errors[index] = None;
|
||||
successful_responses += 1;
|
||||
}
|
||||
Ok((index, Err(err))) => {
|
||||
errors[index] = Some(err);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
if successful_responses + pending < read_quorum {
|
||||
return Err(());
|
||||
}
|
||||
}
|
||||
|
||||
Ok((responses, errors))
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn read_parts(
|
||||
@@ -25,9 +107,6 @@ impl SetDisks {
|
||||
) -> disk::error::Result<Vec<ObjectPartInfo>> {
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
let mut object_parts = Vec::with_capacity(disks.len());
|
||||
|
||||
// Use batch processor for better performance
|
||||
let processor = get_global_processors().read_processor();
|
||||
let bucket = bucket.to_string();
|
||||
let part_meta_paths = part_meta_paths.to_vec();
|
||||
|
||||
@@ -48,19 +127,13 @@ impl SetDisks {
|
||||
})
|
||||
.collect();
|
||||
|
||||
let results = processor.execute_batch(tasks).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
errs.push(None);
|
||||
object_parts.push(res);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
object_parts.push(vec![]);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (responses, collected_errors) = match collect_read_parts_results(tasks, read_quorum).await {
|
||||
Ok(collected) => collected,
|
||||
Err(()) => return Err(DiskError::ErasureReadQuorum),
|
||||
};
|
||||
|
||||
errs.extend(collected_errors);
|
||||
object_parts.extend(responses.into_iter().map(|resp| resp.unwrap_or_default()));
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
@@ -384,10 +457,23 @@ impl SetDisks {
|
||||
read_quorum: usize,
|
||||
) -> Vec<ReadMultipleResp> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let mut ress = Vec::with_capacity(disks.len());
|
||||
let mut errors = Vec::with_capacity(disks.len());
|
||||
let empty_quorum_result = || {
|
||||
req.files
|
||||
.iter()
|
||||
.map(|want| ReadMultipleResp {
|
||||
bucket: req.bucket.clone(),
|
||||
prefix: req.prefix.clone(),
|
||||
file: want.clone(),
|
||||
exists: false,
|
||||
error: Error::ErasureReadQuorum.to_string(),
|
||||
data: Vec::new(),
|
||||
mod_time: None,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
for disk in disks.iter() {
|
||||
let disk = disk.clone();
|
||||
let req = req.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
@@ -398,19 +484,10 @@ impl SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
ress.push(Some(res));
|
||||
errors.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
ress.push(None);
|
||||
errors.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
||||
Ok(collected) => collected,
|
||||
Err(()) => return empty_quorum_result(),
|
||||
};
|
||||
|
||||
// debug!("ReadMultipleResp ress {:?}", ress);
|
||||
// debug!("ReadMultipleResp errors {:?}", errors);
|
||||
@@ -839,3 +916,181 @@ impl SetDisks {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_multiple_results_fails_early_when_quorum_is_impossible() {
|
||||
let started = std::time::Instant::now();
|
||||
let resp = ReadMultipleResp {
|
||||
bucket: "bucket".to_string(),
|
||||
prefix: "prefix".to_string(),
|
||||
file: "file".to_string(),
|
||||
exists: true,
|
||||
error: String::new(),
|
||||
data: vec![1],
|
||||
mod_time: None,
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Err(DiskError::DiskNotFound)),
|
||||
(15, Err(DiskError::DiskNotFound)),
|
||||
(250, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let result = collect_read_multiple_results(tasks, 2).await;
|
||||
assert!(result.is_err(), "quorum should become impossible before slow tail completes");
|
||||
assert!(started.elapsed() < std::time::Duration::from_millis(120));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_multiple_results_returns_collected_responses_on_quorum() {
|
||||
let resp = ReadMultipleResp {
|
||||
bucket: "bucket".to_string(),
|
||||
prefix: "prefix".to_string(),
|
||||
file: "file".to_string(),
|
||||
exists: true,
|
||||
error: String::new(),
|
||||
data: vec![1, 2, 3],
|
||||
mod_time: None,
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
|
||||
(15, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
|
||||
(250, Err(DiskError::DiskNotFound)),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (responses, errors) = collect_read_multiple_results(tasks, 2).await.expect("quorum should succeed");
|
||||
|
||||
assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_multiple_results_tolerates_single_panicked_task_when_quorum_is_met() {
|
||||
let resp = ReadMultipleResp {
|
||||
bucket: "bucket".to_string(),
|
||||
prefix: "prefix".to_string(),
|
||||
file: "file".to_string(),
|
||||
exists: true,
|
||||
error: String::new(),
|
||||
data: vec![1, 2, 3],
|
||||
mod_time: None,
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, should_panic)| {
|
||||
let resp = resp.clone();
|
||||
async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
if should_panic {
|
||||
panic!("simulated task panic");
|
||||
}
|
||||
Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (responses, errors) = collect_read_multiple_results(tasks, 2)
|
||||
.await
|
||||
.expect("quorum should still succeed");
|
||||
assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_parts_results_fails_early_when_quorum_is_impossible() {
|
||||
let started = std::time::Instant::now();
|
||||
let part = ObjectPartInfo {
|
||||
number: 1,
|
||||
etag: "etag".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Err(DiskError::DiskNotFound)),
|
||||
(15, Err(DiskError::DiskNotFound)),
|
||||
(250, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let result = collect_read_parts_results(tasks, 2).await;
|
||||
assert!(result.is_err(), "quorum should become impossible before slow tail completes");
|
||||
assert!(started.elapsed() < std::time::Duration::from_millis(120));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_parts_results_returns_collected_responses_on_quorum() {
|
||||
let part = ObjectPartInfo {
|
||||
number: 1,
|
||||
etag: "etag".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
|
||||
(15, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
|
||||
(250, Err(DiskError::DiskNotFound)),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (responses, errors) = collect_read_parts_results(tasks, 2).await.expect("quorum should succeed");
|
||||
assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
|
||||
let part = ObjectPartInfo {
|
||||
number: 1,
|
||||
etag: "etag".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, should_panic)| {
|
||||
let part = part.clone();
|
||||
async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
if should_panic {
|
||||
panic!("simulated task panic");
|
||||
}
|
||||
Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (responses, errors) = collect_read_parts_results(tasks, 2)
|
||||
.await
|
||||
.expect("quorum should still succeed");
|
||||
assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user