gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into dada/fix-entry

Browse Source
This commit is contained in:
weisd
2025-05-09 14:33:30 +08:00
committed by GitHub
parent 79d58a98f4 e90b531567
commit d7cac5c727
76 changed files with 1384 additions and 550 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/set_disk.rs
+62 -54
View File
@@ -284,10 +284,20 @@ impl SetDisks {
// let mut ress = Vec::with_capacity(disks.len());
let mut errs = Vec::with_capacity(disks.len());
for (i, disk) in disks.iter().enumerate() {
let mut file_info = file_infos[i].clone();
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
futures.push(async move {
for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() {
let mut file_info = file_info.clone();
let disk = disk.clone();
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
let dst_object = dst_object.clone();
let dst_bucket = dst_bucket.clone();
futures.push(tokio::spawn(async move {
if file_info.erasure.index == 0 {
file_info.erasure.index = i + 1;
}
@@ -297,12 +307,12 @@ impl SetDisks {
}
if let Some(disk) = disk {
disk.rename_data(src_bucket, src_object, file_info, dst_bucket, dst_object)
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.await
} else {
Err(Error::new(DiskError::DiskNotFound))
}
})
}));
}
let mut disk_versions = vec![None; disks.len()];
@@ -311,15 +321,13 @@ impl SetDisks {
let results = join_all(futures).await;
for (idx, result) in results.iter().enumerate() {
match result {
match result.as_ref().map_err(|_| Error::new(DiskError::Unexpected))? {
Ok(res) => {
data_dirs[idx] = res.old_data_dir;
disk_versions[idx].clone_from(&res.sign);
// ress.push(Some(res));
errs.push(None);
}
Err(e) => {
// ress.push(None);
errs.push(Some(clone_err(e)));
}
}
@@ -336,11 +344,14 @@ impl SetDisks {
if let Some(disk) = disks[i].as_ref() {
let fi = file_infos[i].clone();
let old_data_dir = data_dirs[i];
futures.push(async move {
let disk = disk.clone();
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
futures.push(tokio::spawn(async move {
let _ = disk
.delete_version(
src_bucket,
src_object,
&src_bucket,
&src_object,
fi,
false,
DeleteOptions {
@@ -354,7 +365,7 @@ impl SetDisks {
debug!("rename_data delete_version err {:?}", e);
e
});
});
}));
}
}
@@ -416,41 +427,41 @@ impl SetDisks {
data_dir: &str,
write_quorum: usize,
) -> Result<()> {
let file_path = format!("{}/{}", object, data_dir);
let mut futures = Vec::with_capacity(disks.len());
let mut errs = Vec::with_capacity(disks.len());
for disk in disks.iter() {
let file_path = Arc::new(format!("{}/{}", object, data_dir));
let bucket = Arc::new(bucket.to_string());
let futures = disks.iter().map(|disk| {
let file_path = file_path.clone();
futures.push(async move {
let bucket = bucket.clone();
let disk = disk.clone();
tokio::spawn(async move {
if let Some(disk) = disk {
disk.delete(
bucket,
&file_path,
DeleteOptions {
recursive: true,
..Default::default()
},
)
.await
match disk
.delete(
&bucket,
&file_path,
DeleteOptions {
recursive: true,
..Default::default()
},
)
.await
{
Ok(_) => None,
Err(e) => Some(e),
}
} else {
Err(Error::new(DiskError::DiskNotFound))
Some(Error::new(DiskError::DiskNotFound))
}
});
}
let results = join_all(futures).await;
for result in results {
match result {
Ok(_) => {
errs.push(None);
}
Err(e) => {
errs.push(Some(e));
}
}
}
})
});
let errs: Vec<Option<Error>> = join_all(futures)
.await
.into_iter()
.map(|e| match e {
Ok(e) => e,
Err(_) => Some(Error::new(DiskError::Unexpected)),
})
.collect();
if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) {
return Err(err);
@@ -3759,7 +3770,7 @@ impl ObjectIO for SetDisks {
let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap());
let mut erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let is_inline_buffer = {
if let Some(sc) = GLOBAL_StorageClass.get() {
@@ -3798,19 +3809,18 @@ impl ObjectIO for SetDisks {
}
let stream = replace(&mut data.stream, Box::new(empty()));
let mut etag_stream = EtagReader::new(stream);
let etag_stream = EtagReader::new(stream);
// TODO: etag from header
let w_size = erasure
.encode(&mut etag_stream, &mut writers, data.content_length, write_quorum)
let (w_size, etag) = Arc::new(erasure)
.encode(etag_stream, &mut writers, data.content_length, write_quorum)
.await?; // TODO: 出错，删除临时目录
if let Err(err) = close_bitrot_writers(&mut writers).await {
error!("close_bitrot_writers err {:?}", err);
}
let etag = etag_stream.etag().await;
//TODO: userDefined
user_defined.insert("etag".to_owned(), etag.clone());
@@ -4408,21 +4418,19 @@ impl StorageAPI for SetDisks {
}
}
let mut erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let stream = replace(&mut data.stream, Box::new(empty()));
let mut etag_stream = EtagReader::new(stream);
let etag_stream = EtagReader::new(stream);
let w_size = erasure
.encode(&mut etag_stream, &mut writers, data.content_length, write_quorum)
let (w_size, mut etag) = Arc::new(erasure)
.encode(etag_stream, &mut writers, data.content_length, write_quorum)
.await?;
if let Err(err) = close_bitrot_writers(&mut writers).await {
error!("close_bitrot_writers err {:?}", err);
}
let mut etag = etag_stream.etag().await;
if let Some(ref tag) = opts.preserve_etag {
etag = tag.clone();
}