mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-01 19:12:14 +00:00
cxymds
764 lines
26 KiB
Rust
764 lines
26 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
use super::*;
|
|
|
|
impl SetDisks {
|
|
pub(super) fn default_read_quorum(&self) -> usize {
|
|
self.set_drive_count - self.default_parity_count
|
|
}
|
|
|
|
pub(super) fn default_write_quorum(&self) -> usize {
|
|
let mut data_count = self.set_drive_count - self.default_parity_count;
|
|
if data_count == self.default_parity_count {
|
|
data_count += 1
|
|
}
|
|
|
|
data_count
|
|
}
|
|
|
|
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
|
#[allow(clippy::type_complexity)]
|
|
pub(super) async fn rename_data(
|
|
disks: &[Option<DiskStore>],
|
|
src_bucket: &str,
|
|
src_object: &str,
|
|
file_infos: &[FileInfo],
|
|
dst_bucket: &str,
|
|
dst_object: &str,
|
|
write_quorum: usize,
|
|
) -> disk::error::Result<(Vec<Option<DiskStore>>, Option<Vec<u8>>, Option<Uuid>, Vec<Option<DiskStore>>)> {
|
|
let mut futures = Vec::with_capacity(disks.len());
|
|
|
|
let mut errs = Vec::with_capacity(disks.len());
|
|
|
|
let src_bucket = Arc::new(src_bucket.to_string());
|
|
let src_object = Arc::new(src_object.to_string());
|
|
let dst_bucket = Arc::new(dst_bucket.to_string());
|
|
let dst_object = Arc::new(dst_object.to_string());
|
|
|
|
for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() {
|
|
let mut file_info = file_info.clone();
|
|
let disk = disk.clone();
|
|
let src_bucket = src_bucket.clone();
|
|
let src_object = src_object.clone();
|
|
let dst_object = dst_object.clone();
|
|
let dst_bucket = dst_bucket.clone();
|
|
|
|
futures.push(tokio::spawn(async move {
|
|
if file_info.erasure.index == 0 {
|
|
file_info.erasure.index = i + 1;
|
|
}
|
|
|
|
if !file_info.is_valid() {
|
|
return Err(DiskError::FileCorrupt);
|
|
}
|
|
|
|
if let Some(disk) = disk {
|
|
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
|
.await
|
|
} else {
|
|
Err(DiskError::DiskNotFound)
|
|
}
|
|
}));
|
|
}
|
|
|
|
let mut disk_versions = vec![None; disks.len()];
|
|
let mut data_dirs = vec![None; disks.len()];
|
|
|
|
let results = join_all(futures).await;
|
|
|
|
for (idx, result) in results.iter().enumerate() {
|
|
match result.as_ref().map_err(|_| DiskError::Unexpected)? {
|
|
Ok(res) => {
|
|
data_dirs[idx] = res.old_data_dir;
|
|
disk_versions[idx].clone_from(&res.sign);
|
|
errs.push(None);
|
|
}
|
|
Err(e) => {
|
|
errs.push(Some(e.clone()));
|
|
}
|
|
}
|
|
}
|
|
|
|
if issue3031_diag_enabled() {
|
|
let success_count = errs.iter().filter(|err| err.is_none()).count();
|
|
let failure_count = errs.len().saturating_sub(success_count);
|
|
let ignored_failure_count = errs
|
|
.iter()
|
|
.filter(|err| err.as_ref().is_some_and(|err| OBJECT_OP_IGNORED_ERRS.contains(err)))
|
|
.count();
|
|
let data_dir_vote_count = data_dirs.iter().filter(|data_dir| data_dir.is_some()).count();
|
|
let reduced_data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum);
|
|
warn!(
|
|
target: "rustfs_ecstore::set_disk",
|
|
src_bucket = %src_bucket,
|
|
src_object = %src_object,
|
|
dst_bucket = %dst_bucket,
|
|
dst_object = %dst_object,
|
|
write_quorum,
|
|
disk_count = errs.len(),
|
|
success_count,
|
|
failure_count,
|
|
ignored_failure_count,
|
|
data_dir_vote_count,
|
|
reduced_data_dir = ?reduced_data_dir,
|
|
errs = ?errs,
|
|
data_dirs = ?data_dirs,
|
|
"issue3031_rename_data_quorum_context"
|
|
);
|
|
}
|
|
|
|
let mut futures = Vec::with_capacity(disks.len());
|
|
if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
|
// TODO: add concurrency
|
|
for (i, err) in errs.iter().enumerate() {
|
|
if err.is_some() {
|
|
continue;
|
|
}
|
|
|
|
if let Some(disk) = disks[i].as_ref() {
|
|
let fi = file_infos[i].clone();
|
|
let old_data_dir = data_dirs[i];
|
|
let disk = disk.clone();
|
|
let dst_bucket = dst_bucket.clone();
|
|
let dst_object = dst_object.clone();
|
|
futures.push(tokio::spawn(async move {
|
|
disk.delete_version(
|
|
&dst_bucket,
|
|
&dst_object,
|
|
fi,
|
|
false,
|
|
DeleteOptions {
|
|
undo_write: true,
|
|
old_data_dir,
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await
|
|
}));
|
|
}
|
|
}
|
|
|
|
if issue3031_diag_enabled() {
|
|
warn!(
|
|
target: "rustfs_ecstore::set_disk",
|
|
src_bucket = %src_bucket,
|
|
src_object = %src_object,
|
|
dst_bucket = %dst_bucket,
|
|
dst_object = %dst_object,
|
|
write_quorum,
|
|
ret_err = %ret_err,
|
|
errs = ?errs,
|
|
data_dirs = ?data_dirs,
|
|
"issue3031_rename_data_quorum_failed"
|
|
);
|
|
}
|
|
|
|
let undo_results = join_all(futures).await;
|
|
let undo_error_count = undo_results
|
|
.iter()
|
|
.filter(|result| match result {
|
|
Err(_) | Ok(Err(_)) => true,
|
|
Ok(Ok(_)) => false,
|
|
})
|
|
.count();
|
|
if undo_error_count > 0 {
|
|
warn!(
|
|
target: "rustfs_ecstore::set_disk",
|
|
dst_bucket = %dst_bucket,
|
|
dst_object = %dst_object,
|
|
undo_error_count,
|
|
"rename_data quorum rollback reported errors"
|
|
);
|
|
}
|
|
return Err(ret_err);
|
|
}
|
|
|
|
let versions = None;
|
|
// TODO: reduceCommonVersions
|
|
|
|
let data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum);
|
|
let online_disks = Self::eval_disks(disks, &errs);
|
|
let cleanup_disks = if let Some(data_dir) = data_dir {
|
|
disks
|
|
.iter()
|
|
.zip(errs.iter())
|
|
.zip(data_dirs.iter())
|
|
.map(|((disk, err), old_data_dir)| {
|
|
if err.is_none() && *old_data_dir == Some(data_dir) {
|
|
disk.clone()
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
.collect()
|
|
} else {
|
|
vec![None; disks.len()]
|
|
};
|
|
|
|
// // TODO: reduce_common_data_dir
|
|
// if let Some(old_dir) = rename_ress
|
|
// .iter()
|
|
// .filter_map(|v| if v.is_some() { v.as_ref().unwrap().old_data_dir } else { None })
|
|
// .map(|v| v.to_string())
|
|
// .next()
|
|
// {
|
|
// let cm_errs = self.commit_rename_data_dir(&shuffle_disks, &bucket, &object, &old_dir).await;
|
|
// warn!("put_object commit_rename_data_dir:{:?}", &cm_errs);
|
|
// }
|
|
|
|
// self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
|
|
|
|
Ok((online_disks, versions, data_dir, cleanup_disks))
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
#[tracing::instrument(level = "debug", skip(self, disks))]
|
|
pub(super) async fn commit_rename_data_dir(
|
|
&self,
|
|
disks: &[Option<DiskStore>],
|
|
bucket: &str,
|
|
object: &str,
|
|
data_dir: &str,
|
|
write_quorum: usize,
|
|
) -> disk::error::Result<()> {
|
|
let file_path = Arc::new(format!("{object}/{data_dir}"));
|
|
let bucket = Arc::new(bucket.to_string());
|
|
let futures = disks.iter().map(|disk| {
|
|
let file_path = file_path.clone();
|
|
let bucket = bucket.clone();
|
|
let disk = disk.clone();
|
|
tokio::spawn(async move {
|
|
if let Some(disk) = disk {
|
|
(disk
|
|
.delete(
|
|
&bucket,
|
|
&file_path,
|
|
DeleteOptions {
|
|
recursive: true,
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await)
|
|
.err()
|
|
} else {
|
|
Some(DiskError::DiskNotFound)
|
|
}
|
|
})
|
|
});
|
|
let errs: Vec<Option<DiskError>> = join_all(futures)
|
|
.await
|
|
.into_iter()
|
|
.map(|e| e.unwrap_or(Some(DiskError::Unexpected)))
|
|
.collect();
|
|
|
|
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
|
return Err(err);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tracing::instrument(skip(self))]
|
|
pub(super) async fn cleanup_multipart_path(&self, paths: &[String]) {
|
|
if paths.is_empty() {
|
|
return;
|
|
}
|
|
let disks = self.get_disks_internal().await;
|
|
|
|
let mut errs = Vec::with_capacity(disks.len());
|
|
|
|
// Use improved simple batch processor instead of join_all for better performance
|
|
let processor = runtime_sources::batch_processors().write_processor();
|
|
|
|
let tasks: Vec<_> = disks
|
|
.iter()
|
|
.map(|disk| {
|
|
let disk = disk.clone();
|
|
let paths = paths.to_vec();
|
|
|
|
async move {
|
|
if let Some(disk) = disk {
|
|
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &paths).await
|
|
} else {
|
|
Err(DiskError::DiskNotFound)
|
|
}
|
|
}
|
|
})
|
|
.collect();
|
|
|
|
let results = processor.execute_batch(tasks).await;
|
|
for result in results {
|
|
match result {
|
|
Ok(_) => {
|
|
errs.push(None);
|
|
}
|
|
Err(e) => {
|
|
errs.push(Some(e));
|
|
}
|
|
}
|
|
}
|
|
|
|
if errs.iter().any(|e| e.is_some()) {
|
|
warn!("cleanup_multipart_path errs {:?}", &errs);
|
|
}
|
|
}
|
|
|
|
#[tracing::instrument(skip(disks, meta))]
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(super) async fn rename_part(
|
|
&self,
|
|
disks: &[Option<DiskStore>],
|
|
src_bucket: &str,
|
|
src_object: &str,
|
|
dst_bucket: &str,
|
|
dst_object: &str,
|
|
meta: Bytes,
|
|
write_quorum: usize,
|
|
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
|
|
) -> disk::error::Result<Vec<Option<DiskStore>>> {
|
|
let src_bucket = Arc::new(src_bucket.to_string());
|
|
let src_object = Arc::new(src_object.to_string());
|
|
let dst_bucket = Arc::new(dst_bucket.to_string());
|
|
let dst_object = Arc::new(dst_object.to_string());
|
|
|
|
// Match MinIO's multipart overwrite semantics: clear any stale destination
|
|
// part payload and metadata before the new per-disk rename fan-out begins.
|
|
self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
|
|
.await;
|
|
|
|
let mut errs = Vec::with_capacity(disks.len());
|
|
|
|
let futures = disks.iter().map(|disk| {
|
|
let disk = disk.clone();
|
|
let meta = meta.clone();
|
|
let src_bucket = src_bucket.clone();
|
|
let src_object = src_object.clone();
|
|
let dst_bucket = dst_bucket.clone();
|
|
let dst_object = dst_object.clone();
|
|
tokio::spawn(async move {
|
|
if let Some(disk) = disk {
|
|
disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
|
|
.await
|
|
} else {
|
|
Err(DiskError::DiskNotFound)
|
|
}
|
|
})
|
|
});
|
|
|
|
let results = join_all(futures).await;
|
|
for result in results {
|
|
match result? {
|
|
Ok(_) => {
|
|
errs.push(None);
|
|
}
|
|
Err(e) => {
|
|
errs.push(Some(e));
|
|
}
|
|
}
|
|
}
|
|
|
|
if issue3031_diag_enabled() {
|
|
let success_count = errs.iter().filter(|err| err.is_none()).count();
|
|
let error_count = errs.len().saturating_sub(success_count);
|
|
let disk_not_found_count = errs.iter().filter(|err| matches!(err, Some(DiskError::DiskNotFound))).count();
|
|
let file_not_found_count = errs.iter().filter(|err| matches!(err, Some(DiskError::FileNotFound))).count();
|
|
warn!(
|
|
target: "rustfs_ecstore::set_disk",
|
|
src_bucket = %src_bucket,
|
|
src_object = %src_object,
|
|
dst_bucket = %dst_bucket,
|
|
dst_object = %dst_object,
|
|
write_quorum = write_quorum,
|
|
disk_count = errs.len(),
|
|
success_count = success_count,
|
|
error_count = error_count,
|
|
disk_not_found_count = disk_not_found_count,
|
|
file_not_found_count = file_not_found_count,
|
|
"issue3031_rename_part_context"
|
|
);
|
|
}
|
|
|
|
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
|
if let Some(context) = quorum_context {
|
|
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
|
|
} else {
|
|
warn!("rename_part errs {:?}", &errs);
|
|
}
|
|
self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
|
|
.await;
|
|
return Err(err);
|
|
}
|
|
|
|
let disks = Self::eval_disks(disks, &errs);
|
|
Ok(disks)
|
|
}
|
|
|
|
pub(super) fn eval_disks(disks: &[Option<DiskStore>], errs: &[Option<DiskError>]) -> Vec<Option<DiskStore>> {
|
|
if disks.len() != errs.len() {
|
|
return Vec::new();
|
|
}
|
|
|
|
let mut online_disks = vec![None; disks.len()];
|
|
|
|
for (i, err_op) in errs.iter().enumerate() {
|
|
if err_op.is_none() {
|
|
online_disks[i].clone_from(&disks[i]);
|
|
}
|
|
}
|
|
|
|
online_disks
|
|
}
|
|
|
|
#[tracing::instrument(skip(disks, files))]
|
|
pub(super) async fn write_unique_file_info(
|
|
disks: &[Option<DiskStore>],
|
|
org_bucket: &str,
|
|
bucket: &str,
|
|
prefix: &str,
|
|
files: &[FileInfo],
|
|
write_quorum: usize,
|
|
) -> disk::error::Result<()> {
|
|
let mut futures = Vec::with_capacity(disks.len());
|
|
let mut errs = Vec::with_capacity(disks.len());
|
|
|
|
for (i, disk) in disks.iter().enumerate() {
|
|
let mut file_info = files[i].clone();
|
|
file_info.erasure.index = i + 1;
|
|
futures.push(async move {
|
|
if let Some(disk) = disk {
|
|
disk.write_metadata(org_bucket, bucket, prefix, file_info).await
|
|
} else {
|
|
Err(DiskError::DiskNotFound)
|
|
}
|
|
});
|
|
}
|
|
|
|
let results = join_all(futures).await;
|
|
for result in results {
|
|
match result {
|
|
Ok(_) => {
|
|
errs.push(None);
|
|
}
|
|
Err(e) => {
|
|
errs.push(Some(e));
|
|
}
|
|
}
|
|
}
|
|
|
|
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
|
let mut revert_futures = Vec::with_capacity(disks.len());
|
|
for (i, err) in errs.iter().enumerate() {
|
|
if err.is_some() {
|
|
continue;
|
|
}
|
|
|
|
if let Some(disk) = disks[i].as_ref() {
|
|
let disk = disk.clone();
|
|
let bucket = bucket.to_string();
|
|
let path = path_join_buf(&[prefix, STORAGE_FORMAT_FILE]);
|
|
revert_futures.push(async move {
|
|
if let Err(err) = disk
|
|
.delete(
|
|
&bucket,
|
|
&path,
|
|
DeleteOptions {
|
|
recursive: true,
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
warn!("write meta revert err {:?}", err);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
join_all(revert_futures).await;
|
|
return Err(err);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(super) async fn update_object_meta(
|
|
&self,
|
|
bucket: &str,
|
|
object: &str,
|
|
fi: FileInfo,
|
|
disks: &[Option<DiskStore>],
|
|
) -> disk::error::Result<()> {
|
|
self.update_object_meta_with_opts(bucket, object, fi, disks, &UpdateMetadataOpts::default())
|
|
.await
|
|
}
|
|
|
|
pub(super) async fn update_object_meta_with_opts(
|
|
&self,
|
|
bucket: &str,
|
|
object: &str,
|
|
fi: FileInfo,
|
|
disks: &[Option<DiskStore>],
|
|
opts: &UpdateMetadataOpts,
|
|
) -> disk::error::Result<()> {
|
|
if fi.metadata.is_empty() && !opts.replace_user_metadata {
|
|
return Ok(());
|
|
}
|
|
|
|
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
|
|
|
let mut futures = Vec::with_capacity(disks.len());
|
|
|
|
let mut errs = Vec::with_capacity(disks.len());
|
|
|
|
for disk in disks.iter() {
|
|
let fi = fi.clone();
|
|
futures.push(async move {
|
|
if let Some(disk) = disk {
|
|
disk.update_metadata(bucket, object, fi, opts).await
|
|
} else {
|
|
Err(DiskError::DiskNotFound)
|
|
}
|
|
})
|
|
}
|
|
|
|
let results = join_all(futures).await;
|
|
for result in results {
|
|
match result {
|
|
Ok(_) => {
|
|
errs.push(None);
|
|
}
|
|
Err(e) => {
|
|
errs.push(Some(e));
|
|
}
|
|
}
|
|
}
|
|
|
|
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, fi.write_quorum(self.default_write_quorum())) {
|
|
return Err(err);
|
|
}
|
|
|
|
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
pub(super) async fn delete_if_dangling(
|
|
&self,
|
|
bucket: &str,
|
|
object: &str,
|
|
meta_arr: &[FileInfo],
|
|
errs: &[Option<DiskError>],
|
|
data_errs_by_part: &HashMap<usize, Vec<usize>>,
|
|
opts: ObjectOptions,
|
|
) -> disk::error::Result<FileInfo> {
|
|
let (m, can_heal) = is_object_dangling(meta_arr, errs, data_errs_by_part);
|
|
|
|
if !can_heal {
|
|
return Err(DiskError::ErasureReadQuorum);
|
|
}
|
|
|
|
let mut tags: HashMap<String, String> = HashMap::new();
|
|
tags.insert("set".to_string(), self.set_index.to_string());
|
|
tags.insert("pool".to_string(), self.pool_index.to_string());
|
|
tags.insert("merrs".to_string(), join_errs(errs));
|
|
tags.insert("derrs".to_string(), format!("{data_errs_by_part:?}"));
|
|
if m.is_valid() {
|
|
tags.insert("sz".to_string(), m.size.to_string());
|
|
tags.insert(
|
|
"mt".to_string(),
|
|
m.mod_time
|
|
.as_ref()
|
|
.map_or(String::new(), |mod_time| mod_time.unix_timestamp().to_string()),
|
|
);
|
|
tags.insert("d:p".to_string(), format!("{}:{}", m.erasure.data_blocks, m.erasure.parity_blocks));
|
|
} else {
|
|
tags.insert("invalid".to_string(), "1".to_string());
|
|
tags.insert(
|
|
"d:p".to_string(),
|
|
format!("{}:{}", self.set_drive_count - self.default_parity_count, self.default_parity_count),
|
|
);
|
|
}
|
|
let mut offline = 0;
|
|
for (i, err) in errs.iter().enumerate() {
|
|
let mut found = false;
|
|
if let Some(err) = err
|
|
&& err == &DiskError::DiskNotFound
|
|
{
|
|
found = true;
|
|
}
|
|
for p in data_errs_by_part {
|
|
if let Some(v) = p.1.get(i)
|
|
&& *v == CHECK_PART_DISK_NOT_FOUND
|
|
{
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if found {
|
|
offline += 1;
|
|
}
|
|
}
|
|
|
|
if offline > 0 {
|
|
tags.insert("offline".to_string(), offline.to_string());
|
|
}
|
|
|
|
let mut fi = FileInfo::default();
|
|
if let Some(ref version_id) = opts.version_id {
|
|
fi.version_id = Uuid::parse_str(version_id).ok();
|
|
}
|
|
|
|
fi.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
|
|
|
let disks = self.get_disks_internal().await;
|
|
|
|
let mut futures = Vec::with_capacity(disks.len());
|
|
for disk_op in disks.iter() {
|
|
let bucket = bucket.to_string();
|
|
let object = object.to_string();
|
|
let fi = fi.clone();
|
|
futures.push(async move {
|
|
if let Some(disk) = disk_op {
|
|
disk.delete_version(&bucket, &object, fi, false, DeleteOptions::default())
|
|
.await
|
|
} else {
|
|
Err(DiskError::DiskNotFound)
|
|
}
|
|
});
|
|
}
|
|
|
|
let results = join_all(futures).await;
|
|
let mut delete_errs = Vec::with_capacity(results.len());
|
|
for (index, result) in results.into_iter().enumerate() {
|
|
let key = format!("ddisk-{index}");
|
|
let already_absent = matches!(
|
|
errs.get(index).and_then(Option::as_ref),
|
|
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
|
);
|
|
match result {
|
|
Ok(_) => {
|
|
tags.insert(key, "<nil>".to_string());
|
|
delete_errs.push(None);
|
|
}
|
|
Err(e) => {
|
|
tags.insert(key, e.to_string());
|
|
if already_absent || matches!(&e, DiskError::FileNotFound | DiskError::FileVersionNotFound) {
|
|
delete_errs.push(None);
|
|
} else {
|
|
delete_errs.push(Some(e));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let write_quorum = if m.is_valid() {
|
|
m.write_quorum(self.default_write_quorum())
|
|
} else {
|
|
self.default_write_quorum()
|
|
};
|
|
if let Some(err) = reduce_write_quorum_errs(&delete_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
|
return Err(err);
|
|
}
|
|
|
|
Ok(m)
|
|
}
|
|
|
|
pub(super) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
|
|
let disks = self.get_disks_internal().await;
|
|
let write_quorum = disks.len() / 2 + 1;
|
|
|
|
let mut futures = Vec::with_capacity(disks.len());
|
|
|
|
for disk_op in disks.iter() {
|
|
let bucket = bucket.to_string();
|
|
let prefix = prefix.to_string();
|
|
futures.push(async move {
|
|
if let Some(disk) = disk_op {
|
|
disk.delete(
|
|
&bucket,
|
|
&prefix,
|
|
DeleteOptions {
|
|
recursive: true,
|
|
immediate: true,
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await
|
|
} else {
|
|
Ok(())
|
|
}
|
|
});
|
|
}
|
|
|
|
let errs = join_all(futures).await.into_iter().map(|v| v.err()).collect::<Vec<_>>();
|
|
|
|
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
|
return Err(err);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
pub(super) async fn check_write_precondition(
|
|
&self,
|
|
bucket: &str,
|
|
object: &str,
|
|
opts: &ObjectOptions,
|
|
) -> Option<StorageError> {
|
|
let mut opts = opts.clone();
|
|
|
|
let http_preconditions = opts.http_preconditions?;
|
|
opts.http_preconditions = None;
|
|
|
|
// Never claim a lock here, to avoid deadlock
|
|
// - If no_lock is false, we must have obtained the lock out side of this function
|
|
// - If no_lock is true, we should not obtain locks
|
|
opts.no_lock = true;
|
|
let oi = self.get_object_info(bucket, object, &opts).await;
|
|
|
|
match oi {
|
|
Ok(oi) => {
|
|
// If top level is a delete marker proceed to upload.
|
|
if oi.delete_marker {
|
|
return None;
|
|
}
|
|
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
|
|
let if_match = http_preconditions.if_match_value().map(str::to_owned);
|
|
if should_prevent_write(&oi, if_none_match, if_match) {
|
|
return Some(StorageError::PreconditionFailed);
|
|
}
|
|
}
|
|
|
|
Err(StorageError::VersionNotFound(_, _, _))
|
|
| Err(StorageError::ObjectNotFound(_, _))
|
|
| Err(StorageError::ErasureReadQuorum) => {
|
|
// When the object is not found,
|
|
// - if If-Match is set, we should return 404 NotFound
|
|
// - if If-None-Match is set, we should be able to proceed with the request
|
|
if http_preconditions.if_match_value().is_some() {
|
|
return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()));
|
|
}
|
|
}
|
|
|
|
Err(e) => {
|
|
return Some(e);
|
|
}
|
|
}
|
|
|
|
None
|
|
}
|
|
}
|