mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-28 07:57:01 +00:00
refactor(ecstore): split set_disk.rs into submodules (#1945)
This commit is contained in:
安正超
@@ -0,0 +1,629 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) fn default_read_quorum(&self) -> usize {
|
||||
self.set_drive_count - self.default_parity_count
|
||||
}
|
||||
|
||||
pub(super) fn default_write_quorum(&self) -> usize {
|
||||
let mut data_count = self.set_drive_count - self.default_parity_count;
|
||||
if data_count == self.default_parity_count {
|
||||
data_count += 1
|
||||
}
|
||||
|
||||
data_count
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub(super) async fn rename_data(
|
||||
disks: &[Option<DiskStore>],
|
||||
src_bucket: &str,
|
||||
src_object: &str,
|
||||
file_infos: &[FileInfo],
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<(Vec<Option<DiskStore>>, Option<Vec<u8>>, Option<Uuid>)> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
let dst_object = Arc::new(dst_object.to_string());
|
||||
|
||||
for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() {
|
||||
let mut file_info = file_info.clone();
|
||||
let disk = disk.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
|
||||
futures.push(tokio::spawn(async move {
|
||||
if file_info.erasure.index == 0 {
|
||||
file_info.erasure.index = i + 1;
|
||||
}
|
||||
|
||||
if !file_info.is_valid() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
if let Some(disk) = disk {
|
||||
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
let mut disk_versions = vec![None; disks.len()];
|
||||
let mut data_dirs = vec![None; disks.len()];
|
||||
|
||||
let results = join_all(futures).await;
|
||||
|
||||
for (idx, result) in results.iter().enumerate() {
|
||||
match result.as_ref().map_err(|_| DiskError::Unexpected)? {
|
||||
Ok(res) => {
|
||||
data_dirs[idx] = res.old_data_dir;
|
||||
disk_versions[idx].clone_from(&res.sign);
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
// TODO: add concurrency
|
||||
for (i, err) in errs.iter().enumerate() {
|
||||
if err.is_some() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(disk) = disks[i].as_ref() {
|
||||
let fi = file_infos[i].clone();
|
||||
let old_data_dir = data_dirs[i];
|
||||
let disk = disk.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
futures.push(tokio::spawn(async move {
|
||||
let _ = disk
|
||||
.delete_version(
|
||||
&src_bucket,
|
||||
&src_object,
|
||||
fi,
|
||||
false,
|
||||
DeleteOptions {
|
||||
undo_write: true,
|
||||
old_data_dir,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
debug!("rename_data delete_version err {:?}", e);
|
||||
e
|
||||
});
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
let _ = join_all(futures).await;
|
||||
return Err(ret_err);
|
||||
}
|
||||
|
||||
let versions = None;
|
||||
// TODO: reduceCommonVersions
|
||||
|
||||
let data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum);
|
||||
|
||||
// // TODO: reduce_common_data_dir
|
||||
// if let Some(old_dir) = rename_ress
|
||||
// .iter()
|
||||
// .filter_map(|v| if v.is_some() { v.as_ref().unwrap().old_data_dir } else { None })
|
||||
// .map(|v| v.to_string())
|
||||
// .next()
|
||||
// {
|
||||
// let cm_errs = self.commit_rename_data_dir(&shuffle_disks, &bucket, &object, &old_dir).await;
|
||||
// warn!("put_object commit_rename_data_dir:{:?}", &cm_errs);
|
||||
// }
|
||||
|
||||
// self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
|
||||
|
||||
Ok((Self::eval_disks(disks, &errs), versions, data_dir))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[tracing::instrument(level = "debug", skip(self, disks))]
|
||||
pub(super) async fn commit_rename_data_dir(
|
||||
&self,
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data_dir: &str,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let file_path = Arc::new(format!("{object}/{data_dir}"));
|
||||
let bucket = Arc::new(bucket.to_string());
|
||||
let futures = disks.iter().map(|disk| {
|
||||
let file_path = file_path.clone();
|
||||
let bucket = bucket.clone();
|
||||
let disk = disk.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Some(disk) = disk {
|
||||
(disk
|
||||
.delete(
|
||||
&bucket,
|
||||
&file_path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await)
|
||||
.err()
|
||||
} else {
|
||||
Some(DiskError::DiskNotFound)
|
||||
}
|
||||
})
|
||||
});
|
||||
let errs: Vec<Option<DiskError>> = join_all(futures)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|e| e.unwrap_or(Some(DiskError::Unexpected)))
|
||||
.collect();
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub(super) async fn cleanup_multipart_path(&self, paths: &[String]) {
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
// Use improved simple batch processor instead of join_all for better performance
|
||||
let processor = get_global_processors().write_processor();
|
||||
|
||||
let tasks: Vec<_> = disks
|
||||
.iter()
|
||||
.map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let paths = paths.to_vec();
|
||||
|
||||
async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &paths).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let results = processor.execute_batch(tasks).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => {
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if errs.iter().any(|e| e.is_some()) {
|
||||
warn!("cleanup_multipart_path errs {:?}", &errs);
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(disks, meta))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn rename_part(
|
||||
&self,
|
||||
disks: &[Option<DiskStore>],
|
||||
src_bucket: &str,
|
||||
src_object: &str,
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
meta: Bytes,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<Vec<Option<DiskStore>>> {
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
let dst_object = Arc::new(dst_object.to_string());
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
let futures = disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let meta = meta.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result? {
|
||||
Ok(_) => {
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
warn!("rename_part errs {:?}", &errs);
|
||||
self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let disks = Self::eval_disks(disks, &errs);
|
||||
Ok(disks)
|
||||
}
|
||||
|
||||
pub(super) fn eval_disks(disks: &[Option<DiskStore>], errs: &[Option<DiskError>]) -> Vec<Option<DiskStore>> {
|
||||
if disks.len() != errs.len() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut online_disks = vec![None; disks.len()];
|
||||
|
||||
for (i, err_op) in errs.iter().enumerate() {
|
||||
if err_op.is_none() {
|
||||
online_disks[i].clone_from(&disks[i]);
|
||||
}
|
||||
}
|
||||
|
||||
online_disks
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(disks, files))]
|
||||
pub(super) async fn write_unique_file_info(
|
||||
disks: &[Option<DiskStore>],
|
||||
org_bucket: &str,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
files: &[FileInfo],
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
let mut file_info = files[i].clone();
|
||||
file_info.erasure.index = i + 1;
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.write_metadata(org_bucket, bucket, prefix, file_info).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => {
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
// TODO: add concurrency
|
||||
for (i, err) in errs.iter().enumerate() {
|
||||
if err.is_some() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(disk) = disks[i].as_ref() {
|
||||
let _ = disk
|
||||
.delete(
|
||||
bucket,
|
||||
&path_join_buf(&[prefix, STORAGE_FORMAT_FILE]),
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!("write meta revert err {:?}", e);
|
||||
e
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn update_object_meta(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
fi: FileInfo,
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> disk::error::Result<()> {
|
||||
self.update_object_meta_with_opts(bucket, object, fi, disks, &UpdateMetadataOpts::default())
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn update_object_meta_with_opts(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
fi: FileInfo,
|
||||
disks: &[Option<DiskStore>],
|
||||
opts: &UpdateMetadataOpts,
|
||||
) -> disk::error::Result<()> {
|
||||
if fi.metadata.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk in disks.iter() {
|
||||
let fi = fi.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.update_metadata(bucket, object, fi, opts).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => {
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, fi.write_quorum(self.default_write_quorum())) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn delete_if_dang_ling(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
meta_arr: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
data_errs_by_part: &HashMap<usize, Vec<usize>>,
|
||||
opts: ObjectOptions,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
let (m, can_heal) = is_object_dang_ling(meta_arr, errs, data_errs_by_part);
|
||||
|
||||
if !can_heal {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let mut tags: HashMap<String, String> = HashMap::new();
|
||||
tags.insert("set".to_string(), self.set_index.to_string());
|
||||
tags.insert("pool".to_string(), self.pool_index.to_string());
|
||||
tags.insert("merrs".to_string(), join_errs(errs));
|
||||
tags.insert("derrs".to_string(), format!("{data_errs_by_part:?}"));
|
||||
if m.is_valid() {
|
||||
tags.insert("sz".to_string(), m.size.to_string());
|
||||
tags.insert(
|
||||
"mt".to_string(),
|
||||
m.mod_time
|
||||
.as_ref()
|
||||
.map_or(String::new(), |mod_time| mod_time.unix_timestamp().to_string()),
|
||||
);
|
||||
tags.insert("d:p".to_string(), format!("{}:{}", m.erasure.data_blocks, m.erasure.parity_blocks));
|
||||
} else {
|
||||
tags.insert("invalid".to_string(), "1".to_string());
|
||||
tags.insert(
|
||||
"d:p".to_string(),
|
||||
format!("{}:{}", self.set_drive_count - self.default_parity_count, self.default_parity_count),
|
||||
);
|
||||
}
|
||||
let mut offline = 0;
|
||||
for (i, err) in errs.iter().enumerate() {
|
||||
let mut found = false;
|
||||
if let Some(err) = err
|
||||
&& err == &DiskError::DiskNotFound
|
||||
{
|
||||
found = true;
|
||||
}
|
||||
for p in data_errs_by_part {
|
||||
if let Some(v) = p.1.get(i)
|
||||
&& *v == CHECK_PART_DISK_NOT_FOUND
|
||||
{
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if found {
|
||||
offline += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if offline > 0 {
|
||||
tags.insert("offline".to_string(), offline.to_string());
|
||||
}
|
||||
|
||||
let mut fi = FileInfo::default();
|
||||
if let Some(ref version_id) = opts.version_id {
|
||||
fi.version_id = Uuid::parse_str(version_id).ok();
|
||||
}
|
||||
|
||||
fi.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
for disk_op in disks.iter() {
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.to_string();
|
||||
let fi = fi.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk_op {
|
||||
disk.delete_version(&bucket, &object, fi, false, DeleteOptions::default())
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for (index, result) in results.into_iter().enumerate() {
|
||||
let key = format!("ddisk-{index}");
|
||||
match result {
|
||||
Ok(_) => {
|
||||
tags.insert(key, "<nil>".to_string());
|
||||
}
|
||||
Err(e) => {
|
||||
tags.insert(key, e.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: audit
|
||||
|
||||
Ok(m)
|
||||
}
|
||||
|
||||
pub(super) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let write_quorum = disks.len() / 2 + 1;
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk_op in disks.iter() {
|
||||
let bucket = bucket.to_string();
|
||||
let prefix = prefix.to_string();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk_op {
|
||||
disk.delete(
|
||||
&bucket,
|
||||
&prefix,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let errs = join_all(futures).await.into_iter().map(|v| v.err()).collect::<Vec<_>>();
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn check_write_precondition(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Option<StorageError> {
|
||||
let mut opts = opts.clone();
|
||||
|
||||
let http_preconditions = opts.http_preconditions?;
|
||||
opts.http_preconditions = None;
|
||||
|
||||
// Never claim a lock here, to avoid deadlock
|
||||
// - If no_lock is false, we must have obtained the lock out side of this function
|
||||
// - If no_lock is true, we should not obtain locks
|
||||
opts.no_lock = true;
|
||||
let oi = self.get_object_info(bucket, object, &opts).await;
|
||||
|
||||
match oi {
|
||||
Ok(oi) => {
|
||||
// If top level is a delete marker proceed to upload.
|
||||
if oi.delete_marker {
|
||||
return None;
|
||||
}
|
||||
if should_prevent_write(&oi, http_preconditions.if_none_match, http_preconditions.if_match) {
|
||||
return Some(StorageError::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
|
||||
Err(StorageError::VersionNotFound(_, _, _))
|
||||
| Err(StorageError::ObjectNotFound(_, _))
|
||||
| Err(StorageError::ErasureReadQuorum) => {
|
||||
// When the object is not found,
|
||||
// - if If-Match is set, we should return 404 NotFound
|
||||
// - if If-None-Match is set, we should be able to proceed with the request
|
||||
if http_preconditions.if_match.is_some() {
|
||||
return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
Err(e) => {
|
||||
return Some(e);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user