gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-26 05:56:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): drain multipart uploads before decommission

Browse Source
This commit is contained in:
overtrue
2026-08-23 03:57:25 +08:00
parent 2d7120460b
commit e547820cb8
4 changed files with 234 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/ops/multipart.rs
+64 -47
View File
@@ -556,6 +556,69 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str) -> disk::
}
impl SetDisks {
async fn discover_multipart_upload_paths(
&self,
orig_bucket: &str,
error_path: &str,
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize)> {
let disks = self.disks.read().await.clone();
if disks.is_empty() {
return Err(Error::ErasureReadQuorum);
}
let discovery_quorum = if self.default_parity_count == 0 {
disks.len()
} else {
(disks.len() / 2).max(1)
};
let mut discovery_errors = (0..disks.len()).map(|_| Some(DiskError::DiskNotFound)).collect::<Vec<_>>();
let mut candidate_counts = HashMap::<String, usize>::new();
let mut discovery_tasks = JoinSet::new();
for (index, disk) in disks.iter().enumerate() {
let disk = disk.clone();
let orig_bucket = orig_bucket.to_string();
discovery_tasks.spawn(async move {
let result = match disk {
Some(disk) => multipart_upload_paths_on_disk(disk, &orig_bucket).await,
None => Err(DiskError::DiskNotFound),
};
(index, result)
});
}
while let Some(task_result) = discovery_tasks.join_next().await {
let Ok((index, result)) = task_result else {
continue;
};
match result {
Ok(paths) => {
discovery_errors[index] = None;
for path in paths {
*candidate_counts.entry(path).or_insert(0) += 1;
}
}
Err(err) => discovery_errors[index] = Some(err),
}
}
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
}
let mut candidate_paths = candidate_counts
.into_iter()
.filter_map(|(path, count)| (count >= discovery_quorum).then_some(path))
.collect::<Vec<_>>();
candidate_paths.sort_unstable();
Ok((disks, candidate_paths, discovery_quorum))
}
pub(crate) async fn first_multipart_upload_path_for_decommission(&self) -> Result<Option<String>> {
let (_, paths, _) = self
.discover_multipart_upload_paths(RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET)
.await?;
Ok(paths.into_iter().next())
}
async fn acquire_multipart_upload_read_lock(
&self,
op: &'static str,
@@ -747,53 +810,7 @@ impl SetDisks {
max_uploads: usize,
expected_incarnation_id: Option<Uuid>,
) -> Result<ListMultipartsInfo> {
let disks = self.disks.read().await.clone();
if disks.is_empty() {
return Err(Error::ErasureReadQuorum);
}
let discovery_quorum = if self.default_parity_count == 0 {
disks.len()
} else {
(disks.len() / 2).max(1)
};
let mut discovery_errors = (0..disks.len()).map(|_| Some(DiskError::DiskNotFound)).collect::<Vec<_>>();
let mut candidate_counts = HashMap::<String, usize>::new();
let mut discovery_tasks = JoinSet::new();
for (index, disk) in disks.iter().enumerate() {
let disk = disk.clone();
let bucket = bucket.to_string();
discovery_tasks.spawn(async move {
let result = match disk {
Some(disk) => multipart_upload_paths_on_disk(disk, &bucket).await,
None => Err(DiskError::DiskNotFound),
};
(index, result)
});
}
while let Some(task_result) = discovery_tasks.join_next().await {
let Ok((index, result)) = task_result else {
continue;
};
match result {
Ok(paths) => {
discovery_errors[index] = None;
for path in paths {
*candidate_counts.entry(path).or_insert(0) += 1;
}
}
Err(err) => discovery_errors[index] = Some(err),
}
}
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
return Err(to_object_err(err.into(), vec![bucket, prefix]));
}
let candidate_paths = candidate_counts
.into_iter()
.filter_map(|(path, count)| (count >= discovery_quorum).then_some(path))
.collect::<Vec<_>>();
let (disks, candidate_paths, discovery_quorum) = self.discover_multipart_upload_paths(bucket, prefix).await?;
let listed_uploads = stream::iter(candidate_paths)
.map(|upload_path| {
let disks = &disks;