mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-28 09:08:58 +00:00
refactor(ecstore): split set_disk.rs into submodules (#1945)
This commit is contained in:
安正超
+10
-3231
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,770 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl SetDisks {
|
||||
#[tracing::instrument(skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
|
||||
pub(super) async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
info!(?opts, "Starting heal_object");
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
let mut result = HealResultItem {
|
||||
heal_item_type: HealItemType::Object.to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: version_id.to_string(),
|
||||
disk_count: disks.len(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let write_lock_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
|
||||
StorageError::other(format!(
|
||||
"Failed to acquire write lock: {}",
|
||||
self.format_lock_error_from_error(bucket, object, "write", &e)
|
||||
))
|
||||
})?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let version_id_op = {
|
||||
if version_id.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(version_id.to_string())
|
||||
}
|
||||
};
|
||||
|
||||
let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await?;
|
||||
|
||||
info!(
|
||||
parts_count = parts_metadata.len(),
|
||||
bucket = bucket,
|
||||
object = object,
|
||||
version_id = version_id,
|
||||
?errs,
|
||||
"File info read complete"
|
||||
);
|
||||
if DiskError::is_all_not_found(&errs) {
|
||||
warn!(
|
||||
"heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}",
|
||||
bucket, object, version_id
|
||||
);
|
||||
let err = if !version_id.is_empty() {
|
||||
DiskError::FileVersionNotFound
|
||||
} else {
|
||||
DiskError::FileNotFound
|
||||
};
|
||||
// Nothing to do, file is already gone.
|
||||
return Ok((
|
||||
self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
|
||||
.await,
|
||||
Some(err),
|
||||
));
|
||||
}
|
||||
|
||||
info!(parts_count = parts_metadata.len(), "heal_object Initiating quorum check");
|
||||
match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) {
|
||||
Ok((read_quorum, _)) => {
|
||||
result.parity_blocks = result.disk_count - read_quorum as usize;
|
||||
result.data_blocks = read_quorum as usize;
|
||||
|
||||
let ((mut online_disks, quorum_mod_time, quorum_etag), disk_len) = {
|
||||
let disks = self.disks.read().await;
|
||||
let disk_len = disks.len();
|
||||
(Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize), disk_len)
|
||||
};
|
||||
|
||||
info!(?parts_metadata, ?errs, ?read_quorum, ?disk_len, "heal_object List disks metadata");
|
||||
|
||||
info!(?online_disks, ?quorum_mod_time, ?quorum_etag, "heal_object List online disks");
|
||||
|
||||
let filter_by_etag = quorum_etag.is_some();
|
||||
match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) {
|
||||
Ok(latest_meta) => {
|
||||
info!("heal_object latest_meta: {:?}", latest_meta);
|
||||
|
||||
let (data_errs_by_disk, data_errs_by_part) = disks_with_all_parts(
|
||||
&mut online_disks,
|
||||
&mut parts_metadata,
|
||||
&errs,
|
||||
&latest_meta,
|
||||
filter_by_etag,
|
||||
bucket,
|
||||
object,
|
||||
opts.scan_mode,
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!(
|
||||
"disks_with_all_parts heal_object results: available_disks count={}, total_disks={}",
|
||||
online_disks.iter().filter(|d| d.is_some()).count(),
|
||||
online_disks.len()
|
||||
);
|
||||
|
||||
let erasure = if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
// Initialize erasure coding
|
||||
erasure_coding::Erasure::new(
|
||||
latest_meta.erasure.data_blocks,
|
||||
latest_meta.erasure.parity_blocks,
|
||||
latest_meta.erasure.block_size,
|
||||
)
|
||||
} else {
|
||||
erasure_coding::Erasure::default()
|
||||
};
|
||||
|
||||
result.object_size =
|
||||
ObjectInfo::from_file_info(&latest_meta, bucket, object, true).get_actual_size()? as usize;
|
||||
// Loop to find number of disks with valid data, per-drive
|
||||
// data state and a list of outdated disks on which data needs
|
||||
// to be healed.
|
||||
let mut out_dated_disks = vec![None; disk_len];
|
||||
let mut disks_to_heal_count = 0;
|
||||
let mut meta_to_heal_count = 0;
|
||||
|
||||
for index in 0..online_disks.len() {
|
||||
let (yes, is_meta, reason) = should_heal_object_on_disk(
|
||||
&errs[index],
|
||||
&data_errs_by_disk[&index],
|
||||
&parts_metadata[index],
|
||||
&latest_meta,
|
||||
);
|
||||
|
||||
if yes {
|
||||
out_dated_disks[index] = disks[index].clone();
|
||||
disks_to_heal_count += 1;
|
||||
if is_meta {
|
||||
meta_to_heal_count += 1;
|
||||
}
|
||||
debug!("heal_object Disk {} marked for healing (endpoint={})", index, self.set_endpoints[index]);
|
||||
}
|
||||
|
||||
let drive_state = match reason {
|
||||
Some(err) => match err {
|
||||
DiskError::DiskNotFound => DriveState::Offline.to_string(),
|
||||
DiskError::FileNotFound
|
||||
| DiskError::FileVersionNotFound
|
||||
| DiskError::VolumeNotFound
|
||||
| DiskError::PartMissingOrCorrupt
|
||||
| DiskError::OutdatedXLMeta => DriveState::Missing.to_string(),
|
||||
DiskError::FileCorrupt => DriveState::Corrupt.to_string(),
|
||||
_ => DriveState::Unknown(err.to_string()).to_string(),
|
||||
},
|
||||
None => DriveState::Ok.to_string(),
|
||||
};
|
||||
result.before.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: self.set_endpoints[index].to_string(),
|
||||
state: drive_state.to_string(),
|
||||
});
|
||||
|
||||
result.after.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: self.set_endpoints[index].to_string(),
|
||||
state: drive_state.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if disks_to_heal_count == 0 {
|
||||
return Ok((result, None));
|
||||
}
|
||||
|
||||
if opts.dry_run {
|
||||
return Ok((result, None));
|
||||
}
|
||||
|
||||
let mut cannot_heal = !latest_meta.deleted && meta_to_heal_count > latest_meta.erasure.parity_blocks;
|
||||
if cannot_heal && quorum_etag.is_some() {
|
||||
cannot_heal = false;
|
||||
}
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
for (_, part_errs) in data_errs_by_part.iter() {
|
||||
if count_part_not_success(part_errs) > latest_meta.erasure.parity_blocks {
|
||||
cannot_heal = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if cannot_heal {
|
||||
let total_disks = parts_metadata.len();
|
||||
let healthy_count = total_disks.saturating_sub(disks_to_heal_count);
|
||||
let required_data = total_disks.saturating_sub(latest_meta.erasure.parity_blocks);
|
||||
|
||||
error!(
|
||||
"Data corruption detected for {}/{}: Insufficient healthy shards. Need at least {} data shards, but found only {} healthy disks. (Missing/Corrupt: {}, Parity: {})",
|
||||
bucket,
|
||||
object,
|
||||
required_data,
|
||||
healthy_count,
|
||||
disks_to_heal_count,
|
||||
latest_meta.erasure.parity_blocks
|
||||
);
|
||||
|
||||
// Allow for dangling deletes, on versions that have DataDir missing etc.
|
||||
// this would end up restoring the correct readable versions.
|
||||
return match self
|
||||
.delete_if_dang_ling(
|
||||
bucket,
|
||||
object,
|
||||
&parts_metadata,
|
||||
&errs,
|
||||
&data_errs_by_part,
|
||||
ObjectOptions {
|
||||
version_id: version_id_op.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(m) => {
|
||||
let derr = if !version_id.is_empty() {
|
||||
DiskError::FileVersionNotFound
|
||||
} else {
|
||||
DiskError::FileNotFound
|
||||
};
|
||||
let mut t_errs = Vec::with_capacity(errs.len());
|
||||
for _ in 0..errs.len() {
|
||||
t_errs.push(None);
|
||||
}
|
||||
Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, Some(derr)))
|
||||
}
|
||||
Err(err) => {
|
||||
// t_errs = vec![Some(err.clone()]; errs.len());
|
||||
let mut t_errs = Vec::with_capacity(errs.len());
|
||||
for _ in 0..errs.len() {
|
||||
t_errs.push(Some(err.clone()));
|
||||
}
|
||||
|
||||
Ok((
|
||||
self.default_heal_result(FileInfo::default(), &t_errs, bucket, object, version_id)
|
||||
.await,
|
||||
Some(err),
|
||||
))
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
if !latest_meta.deleted && latest_meta.erasure.distribution.len() != online_disks.len() {
|
||||
let err_str = format!(
|
||||
"unexpected file distribution ({:?}) from available disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
|
||||
latest_meta.erasure.distribution, online_disks, bucket, object, version_id
|
||||
);
|
||||
warn!(err_str);
|
||||
let err = DiskError::other(err_str);
|
||||
return Ok((
|
||||
self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await,
|
||||
Some(err),
|
||||
));
|
||||
}
|
||||
|
||||
let latest_disks = Self::shuffle_disks(&online_disks, &latest_meta.erasure.distribution);
|
||||
if !latest_meta.deleted && latest_meta.erasure.distribution.len() != out_dated_disks.len() {
|
||||
let err_str = format!(
|
||||
"unexpected file distribution ({:?}) from outdated disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
|
||||
latest_meta.erasure.distribution, out_dated_disks, bucket, object, version_id
|
||||
);
|
||||
warn!(err_str);
|
||||
let err = DiskError::other(err_str);
|
||||
return Ok((
|
||||
self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await,
|
||||
Some(err),
|
||||
));
|
||||
}
|
||||
|
||||
if !latest_meta.deleted && latest_meta.erasure.distribution.len() != parts_metadata.len() {
|
||||
let err_str = format!(
|
||||
"unexpected file distribution ({:?}) from metadata entries ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
|
||||
latest_meta.erasure.distribution,
|
||||
parts_metadata.len(),
|
||||
bucket,
|
||||
object,
|
||||
version_id
|
||||
);
|
||||
warn!(err_str);
|
||||
let err = DiskError::other(err_str);
|
||||
return Ok((
|
||||
self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await,
|
||||
Some(err),
|
||||
));
|
||||
}
|
||||
|
||||
out_dated_disks = Self::shuffle_disks(&out_dated_disks, &latest_meta.erasure.distribution);
|
||||
let mut parts_metadata = Self::shuffle_parts_metadata(&parts_metadata, &latest_meta.erasure.distribution);
|
||||
let mut copy_parts_metadata = vec![None; parts_metadata.len()];
|
||||
for (index, disk) in latest_disks.iter().enumerate() {
|
||||
if disk.is_some() {
|
||||
copy_parts_metadata[index] = Some(parts_metadata[index].clone());
|
||||
}
|
||||
}
|
||||
|
||||
let clean_file_info = |fi: &FileInfo| -> FileInfo {
|
||||
let mut nfi = fi.clone();
|
||||
if !nfi.is_remote() {
|
||||
nfi.data = None;
|
||||
nfi.erasure.index = 0;
|
||||
nfi.erasure.checksums = Vec::new();
|
||||
}
|
||||
nfi
|
||||
};
|
||||
for (index, disk) in out_dated_disks.iter().enumerate() {
|
||||
if disk.is_some() {
|
||||
// Make sure to write the FileInfo information
|
||||
// that is expected to be in quorum.
|
||||
parts_metadata[index] = clean_file_info(&latest_meta);
|
||||
}
|
||||
}
|
||||
|
||||
// We write at temporary location and then rename to final location.
|
||||
let tmp_id = Uuid::new_v4().to_string();
|
||||
let src_data_dir = latest_meta.data_dir.unwrap().to_string();
|
||||
let dst_data_dir = latest_meta.data_dir.unwrap();
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
let erasure_info = latest_meta.erasure.clone();
|
||||
|
||||
for (part_index, part) in latest_meta.parts.iter().enumerate() {
|
||||
let till_offset = erasure.shard_file_offset(0, part.size, part.size);
|
||||
let checksum_algo = erasure_info.get_checksum_info(part.number).algorithm;
|
||||
let mut readers = Vec::with_capacity(latest_disks.len());
|
||||
let mut writers = Vec::with_capacity(out_dated_disks.len());
|
||||
// let mut errors = Vec::with_capacity(out_dated_disks.len());
|
||||
|
||||
let mut prefer = vec![false; latest_disks.len()];
|
||||
for (index, disk) in latest_disks.iter().enumerate() {
|
||||
let this_part_errs =
|
||||
Self::shuffle_check_parts(&data_errs_by_part[&part_index], &erasure_info.distribution);
|
||||
if this_part_errs[index] != CHECK_PART_SUCCESS {
|
||||
info!(
|
||||
"reading part {}: index={}, part_errs={:?}, skipping",
|
||||
part.number, index, this_part_errs[index]
|
||||
);
|
||||
readers.push(None);
|
||||
continue;
|
||||
}
|
||||
|
||||
if let (Some(disk), Some(metadata)) = (disk, ©_parts_metadata[index]) {
|
||||
match create_bitrot_reader(
|
||||
metadata.data.as_deref(),
|
||||
Some(disk),
|
||||
bucket,
|
||||
&path_join_buf(&[object, &src_data_dir, &format!("part.{}", part.number)]),
|
||||
0,
|
||||
till_offset,
|
||||
erasure.shard_size(),
|
||||
checksum_algo.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(reader)) => {
|
||||
readers.push(Some(reader));
|
||||
}
|
||||
Ok(None) => {
|
||||
readers.push(None);
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
readers.push(None);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
prefer[index] = disk.host_name().is_empty();
|
||||
} else {
|
||||
readers.push(None);
|
||||
// errors.push(Some(DiskError::DiskNotFound));
|
||||
}
|
||||
}
|
||||
|
||||
let is_inline_buffer = {
|
||||
if let Some(sc) = GLOBAL_STORAGE_CLASS.get() {
|
||||
sc.should_inline(erasure.shard_file_size(latest_meta.size), false)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
// create writers for all disk positions, but only for outdated disks
|
||||
for (index, disk_op) in out_dated_disks.iter().enumerate() {
|
||||
if let Some(outdated_disk) = disk_op {
|
||||
let writer = match create_bitrot_writer(
|
||||
is_inline_buffer,
|
||||
Some(outdated_disk),
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&path_join_buf(&[
|
||||
&tmp_id.to_string(),
|
||||
&dst_data_dir.to_string(),
|
||||
&format!("part.{}", part.number),
|
||||
]),
|
||||
erasure.shard_file_size(part.size as i64),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::HighwayHash256,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(writer) => writer,
|
||||
Err(err) => {
|
||||
info!(
|
||||
"create_bitrot_writer disk {}, err {:?}, skipping operation",
|
||||
outdated_disk.to_string(),
|
||||
err
|
||||
);
|
||||
writers.push(None);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
writers.push(Some(writer));
|
||||
} else {
|
||||
writers.push(None);
|
||||
}
|
||||
}
|
||||
|
||||
// Heal each part. erasure.Heal() will write the healed
|
||||
// part to .rustfs/tmp/uuid/ which needs to be renamed
|
||||
// later to the final location.
|
||||
erasure.heal(&mut writers, readers, part.size, &prefer).await?;
|
||||
// close_bitrot_writers(&mut writers).await?;
|
||||
|
||||
for (index, disk_op) in out_dated_disks.iter_mut().enumerate() {
|
||||
if disk_op.is_none() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if writers[index].is_none() {
|
||||
*disk_op = None;
|
||||
disks_to_heal_count -= 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
parts_metadata[index].data_dir = Some(dst_data_dir);
|
||||
parts_metadata[index].add_object_part(
|
||||
part.number,
|
||||
part.etag.clone(),
|
||||
part.size,
|
||||
part.mod_time,
|
||||
part.actual_size,
|
||||
part.index.clone(),
|
||||
part.checksums.clone(),
|
||||
);
|
||||
if is_inline_buffer {
|
||||
if let Some(writer) = writers[index].take() {
|
||||
// if let Some(w) = writer.as_any().downcast_ref::<BitrotFileWriter>() {
|
||||
// parts_metadata[index].data = Some(w.inline_data().to_vec());
|
||||
// }
|
||||
parts_metadata[index].data =
|
||||
Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default());
|
||||
}
|
||||
parts_metadata[index].set_inline_data();
|
||||
} else {
|
||||
parts_metadata[index].data = None;
|
||||
}
|
||||
}
|
||||
|
||||
if disks_to_heal_count == 0 {
|
||||
return Ok((
|
||||
result,
|
||||
Some(DiskError::other(format!(
|
||||
"all drives had write errors, unable to heal {bucket}/{object}"
|
||||
))),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Rename from tmp location to the actual location.
|
||||
for (index, outdated_disk) in out_dated_disks.iter().enumerate() {
|
||||
if let Some(disk) = outdated_disk {
|
||||
// record the index of the updated disks
|
||||
parts_metadata[index].erasure.index = index + 1;
|
||||
// Attempt a rename now from healed data to final location.
|
||||
parts_metadata[index].set_healing();
|
||||
|
||||
let rename_result = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object)
|
||||
.await;
|
||||
|
||||
if let Err(err) = &rename_result {
|
||||
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id)
|
||||
.await
|
||||
.map_err(DiskError::other)?;
|
||||
} else {
|
||||
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id)
|
||||
.await
|
||||
.map_err(DiskError::other)?;
|
||||
|
||||
if parts_metadata[index].is_remote() {
|
||||
let rm_data_dir = parts_metadata[index].data_dir.unwrap().to_string();
|
||||
|
||||
let d_path = Path::new(&encode_dir_object(object)).join(rm_data_dir);
|
||||
|
||||
disk.delete(
|
||||
bucket,
|
||||
d_path.to_str().unwrap(),
|
||||
DeleteOptions {
|
||||
immediate: true,
|
||||
|
||||
recursive: true,
|
||||
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
for (i, v) in result.before.drives.iter().enumerate() {
|
||||
if v.endpoint == disk.endpoint().to_string() {
|
||||
result.after.drives[i].state = DriveState::Ok.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((result, None))
|
||||
}
|
||||
Err(err) => Ok((result, Some(err))),
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let data_errs_by_part = HashMap::new();
|
||||
match self
|
||||
.delete_if_dang_ling(
|
||||
bucket,
|
||||
object,
|
||||
&parts_metadata,
|
||||
&errs,
|
||||
&data_errs_by_part,
|
||||
ObjectOptions {
|
||||
version_id: version_id_op.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(m) => {
|
||||
let err = if !version_id.is_empty() {
|
||||
DiskError::FileVersionNotFound
|
||||
} else {
|
||||
DiskError::FileNotFound
|
||||
};
|
||||
Ok((self.default_heal_result(m, &errs, bucket, object, version_id).await, Some(err)))
|
||||
}
|
||||
Err(_) => Ok((
|
||||
self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
|
||||
.await,
|
||||
Some(err),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn heal_object_dir_locked(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
dry_run: bool,
|
||||
remove: bool,
|
||||
) -> Result<(HealResultItem, Option<DiskError>)> {
|
||||
let disks = {
|
||||
let disks = self.disks.read().await;
|
||||
disks.clone()
|
||||
};
|
||||
let mut result = HealResultItem {
|
||||
heal_item_type: HealItemType::Object.to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
disk_count: self.disks.read().await.len(),
|
||||
parity_blocks: self.default_parity_count,
|
||||
data_blocks: disks.len() - self.default_parity_count,
|
||||
object_size: 0,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
result.before.drives = vec![HealDriveInfo::default(); disks.len()];
|
||||
result.after.drives = vec![HealDriveInfo::default(); disks.len()];
|
||||
|
||||
let errs = stat_all_dirs(&disks, bucket, object).await;
|
||||
let dang_ling_object = is_object_dir_dang_ling(&errs);
|
||||
if dang_ling_object && !dry_run && remove {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
for disk in disks.iter().flatten() {
|
||||
let disk = disk.clone();
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.to_string();
|
||||
futures.push(tokio::spawn(async move {
|
||||
let _ = disk
|
||||
.delete(
|
||||
&bucket,
|
||||
&object,
|
||||
DeleteOptions {
|
||||
recursive: false,
|
||||
immediate: false,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}));
|
||||
}
|
||||
|
||||
// ignore errors
|
||||
let _ = join_all(futures).await;
|
||||
}
|
||||
|
||||
for (err, drive) in errs.iter().zip(self.set_endpoints.iter()) {
|
||||
let endpoint = drive.to_string();
|
||||
let drive_state = match err {
|
||||
Some(err) => match err {
|
||||
DiskError::DiskNotFound => DriveState::Offline.to_string(),
|
||||
DiskError::FileNotFound | DiskError::VolumeNotFound => DriveState::Missing.to_string(),
|
||||
_ => DriveState::Corrupt.to_string(),
|
||||
},
|
||||
None => DriveState::Ok.to_string(),
|
||||
};
|
||||
result.before.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: endpoint.clone(),
|
||||
state: drive_state.to_string(),
|
||||
});
|
||||
|
||||
result.after.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint,
|
||||
state: drive_state.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if dang_ling_object || DiskError::is_all_not_found(&errs) {
|
||||
return Ok((result, Some(DiskError::FileNotFound)));
|
||||
}
|
||||
|
||||
if dry_run {
|
||||
// Quit without try to heal the object dir
|
||||
return Ok((result, None));
|
||||
}
|
||||
for (index, (err, disk)) in errs.iter().zip(disks.iter()).enumerate() {
|
||||
if let (Some(DiskError::VolumeNotFound | DiskError::FileNotFound), Some(disk)) = (err, disk) {
|
||||
let vol_path = Path::new(bucket).join(object);
|
||||
let drive_state = match disk.make_volume(vol_path.to_str().unwrap()).await {
|
||||
Ok(_) => DriveState::Ok.to_string(),
|
||||
Err(merr) => match merr {
|
||||
DiskError::VolumeExists => DriveState::Ok.to_string(),
|
||||
DiskError::DiskNotFound => DriveState::Offline.to_string(),
|
||||
_ => DriveState::Corrupt.to_string(),
|
||||
},
|
||||
};
|
||||
result.after.drives[index].state = drive_state.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
Ok((result, None))
|
||||
}
|
||||
|
||||
pub(super) async fn heal_object_dir(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
dry_run: bool,
|
||||
remove: bool,
|
||||
) -> Result<(HealResultItem, Option<DiskError>)> {
|
||||
let _write_lock_guard = self
|
||||
.new_ns_lock(bucket, object)
|
||||
.await?
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
let message = format!(
|
||||
"Failed to acquire write lock: {}",
|
||||
self.format_lock_error_from_error(bucket, object, "write", &e)
|
||||
);
|
||||
DiskError::other(message)
|
||||
})?;
|
||||
|
||||
self.heal_object_dir_locked(bucket, object, dry_run, remove).await
|
||||
}
|
||||
|
||||
pub(super) async fn default_heal_result(
|
||||
&self,
|
||||
lfi: FileInfo,
|
||||
errs: &[Option<DiskError>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
) -> HealResultItem {
|
||||
let disk_len = { self.disks.read().await.len() };
|
||||
let mut result = HealResultItem {
|
||||
heal_item_type: HealItemType::Object.to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
object_size: lfi.size as usize,
|
||||
version_id: version_id.to_string(),
|
||||
disk_count: disk_len,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if lfi.is_valid() {
|
||||
result.parity_blocks = lfi.erasure.parity_blocks;
|
||||
} else {
|
||||
result.parity_blocks = self.default_parity_count;
|
||||
}
|
||||
|
||||
result.data_blocks = disk_len - result.parity_blocks;
|
||||
|
||||
for (index, disk) in self.disks.read().await.iter().enumerate() {
|
||||
if disk.is_none() {
|
||||
result.before.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: self.set_endpoints[index].to_string(),
|
||||
state: DriveState::Offline.to_string(),
|
||||
});
|
||||
|
||||
result.after.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: self.set_endpoints[index].to_string(),
|
||||
state: DriveState::Offline.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut drive_state = DriveState::Corrupt;
|
||||
if let Some(err) = &errs[index] {
|
||||
if err == &DiskError::FileNotFound || err == &DiskError::VolumeNotFound {
|
||||
drive_state = DriveState::Missing;
|
||||
}
|
||||
} else {
|
||||
drive_state = DriveState::Ok;
|
||||
}
|
||||
|
||||
result.before.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: self.set_endpoints[index].to_string(),
|
||||
state: drive_state.to_string(),
|
||||
});
|
||||
result.after.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: self.set_endpoints[index].to_string(),
|
||||
state: drive_state.to_string(),
|
||||
});
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl SetDisks {
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
let disks = self.disks.read().await;
|
||||
|
||||
let disks = disks.clone();
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let mut errors = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk in disks.iter() {
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.delete(
|
||||
bucket,
|
||||
prefix,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => {
|
||||
errors.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errors.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
||||
match err {
|
||||
LockResult::Timeout => {
|
||||
format!("{mode} lock acquisition timed out on {bucket}/{object} (owner={})", self.locker_owner)
|
||||
}
|
||||
LockResult::Conflict {
|
||||
current_owner,
|
||||
current_mode,
|
||||
} => format!("{mode} lock conflicted on {bucket}/{object}: held by {current_owner} as {current_mode:?}"),
|
||||
LockResult::Acquired => format!("unexpected lock state while acquiring {mode} lock on {bucket}/{object}"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn format_lock_error_from_error(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
mode: &str,
|
||||
err: &rustfs_lock::error::LockError,
|
||||
) -> String {
|
||||
match err {
|
||||
rustfs_lock::error::LockError::Timeout { .. } => {
|
||||
format!(
|
||||
"ns_loc: {mode} lock acquisition timed out on {bucket}/{object} (owner={})",
|
||||
self.locker_owner
|
||||
)
|
||||
}
|
||||
rustfs_lock::error::LockError::AlreadyLocked { owner, .. } => {
|
||||
format!("ns_loc: {mode} lock conflicted on {bucket}/{object}: held by {owner}")
|
||||
}
|
||||
_ => format!("ns_loc: {mode} lock acquisition failed on {bucket}/{object}: {}", err),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn get_disks_internal(&self) -> Vec<Option<DiskStore>> {
|
||||
let rl = self.disks.read().await;
|
||||
|
||||
rl.clone()
|
||||
}
|
||||
|
||||
pub async fn get_local_disks(&self) -> Vec<Option<DiskStore>> {
|
||||
let rl = self.disks.read().await;
|
||||
|
||||
let mut disks: Vec<Option<DiskStore>> = rl
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter(|v| v.as_ref().is_some_and(|d| d.is_local()))
|
||||
.collect();
|
||||
|
||||
let mut rng = rand::rng();
|
||||
|
||||
disks.shuffle(&mut rng);
|
||||
|
||||
disks
|
||||
}
|
||||
|
||||
pub(super) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> {
|
||||
let mut disks = self.get_disks_internal().await;
|
||||
|
||||
// TODO: diskinfo filter online
|
||||
|
||||
let mut new_disk = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk in disks.iter() {
|
||||
if let Some(d) = disk
|
||||
&& d.is_online().await
|
||||
{
|
||||
new_disk.push(disk.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let mut rng = rand::rng();
|
||||
|
||||
disks.shuffle(&mut rng);
|
||||
|
||||
new_disk
|
||||
// let disks = self.get_disks_internal().await;
|
||||
// let (filtered, _) = self.filter_online_disks(disks).await;
|
||||
// filtered.into_iter().filter(|disk| disk.is_some()).collect()
|
||||
}
|
||||
|
||||
pub(super) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> {
|
||||
let mut disks = self.get_online_disks().await;
|
||||
|
||||
let mut rng = rand::rng();
|
||||
|
||||
disks.shuffle(&mut rng);
|
||||
|
||||
disks
|
||||
.into_iter()
|
||||
.filter(|v| v.as_ref().is_some_and(|d| d.is_local()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn get_online_disks_with_healing(&self, incl_healing: bool) -> (Vec<DiskStore>, bool) {
|
||||
let (disks, _, healing) = self.get_online_disks_with_healing_and_info(incl_healing).await;
|
||||
(disks, healing > 0)
|
||||
}
|
||||
|
||||
pub async fn get_online_disks_with_healing_and_info(&self, incl_healing: bool) -> (Vec<DiskStore>, Vec<DiskInfo>, usize) {
|
||||
let mut disks = self.get_disks_internal().await;
|
||||
|
||||
let mut infos = Vec::with_capacity(disks.len());
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let mut numbers: Vec<usize> = (0..disks.len()).collect();
|
||||
{
|
||||
let mut rng = rand::rng();
|
||||
disks.shuffle(&mut rng);
|
||||
|
||||
numbers.shuffle(&mut rng);
|
||||
}
|
||||
|
||||
for &i in numbers.iter() {
|
||||
let disk = disks[i].clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.disk_info(&DiskInfoOptions::default()).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Use optimized batch processor for disk info retrieval
|
||||
let processor = get_global_processors().metadata_processor();
|
||||
let results = processor.execute_batch(futures).await;
|
||||
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
infos.push(res);
|
||||
}
|
||||
Err(err) => {
|
||||
infos.push(DiskInfo {
|
||||
error: err.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut healing: usize = 0;
|
||||
|
||||
let mut scanning_disks = Vec::new();
|
||||
let mut healing_disks = Vec::new();
|
||||
let mut scanning_infos = Vec::new();
|
||||
let mut healing_infos = Vec::new();
|
||||
|
||||
let mut new_disks = Vec::new();
|
||||
let mut new_infos = Vec::new();
|
||||
|
||||
for &i in numbers.iter() {
|
||||
let (info, disk) = (infos[i].clone(), disks[i].clone());
|
||||
if !info.error.is_empty() || disk.is_none() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if info.healing {
|
||||
healing += 1;
|
||||
if incl_healing {
|
||||
healing_disks.push(disk.unwrap());
|
||||
healing_infos.push(info);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if !info.healing {
|
||||
new_disks.push(disk.unwrap());
|
||||
new_infos.push(info);
|
||||
} else {
|
||||
scanning_disks.push(disk.unwrap());
|
||||
scanning_infos.push(info);
|
||||
}
|
||||
}
|
||||
|
||||
new_disks.extend(scanning_disks);
|
||||
new_infos.extend(scanning_infos);
|
||||
new_disks.extend(healing_disks);
|
||||
new_infos.extend(healing_infos);
|
||||
|
||||
(new_disks, new_infos, healing)
|
||||
}
|
||||
|
||||
pub(super) async fn _get_local_disks(&self) -> Vec<Option<DiskStore>> {
|
||||
let mut disks = self.get_disks_internal().await;
|
||||
|
||||
let mut rng = rand::rng();
|
||||
|
||||
disks.shuffle(&mut rng);
|
||||
|
||||
disks
|
||||
.into_iter()
|
||||
.filter(|v| v.as_ref().is_some_and(|d| d.is_local()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn connect_disks(&self) {
|
||||
let rl = self.disks.read().await;
|
||||
|
||||
let disks = rl.clone();
|
||||
|
||||
// Explicitly release the lock
|
||||
drop(rl);
|
||||
|
||||
for (i, opdisk) in disks.iter().enumerate() {
|
||||
if let Some(disk) = opdisk {
|
||||
if disk.is_online().await && disk.get_disk_location().set_idx.is_some() {
|
||||
info!("Disk {:?} is online", disk.to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
|
||||
if let Some(endpoint) = self.set_endpoints.get(i) {
|
||||
info!("will renew disk, opdisk: {:?}", opdisk);
|
||||
self.renew_disk(endpoint).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn renew_disk(&self, ep: &Endpoint) {
|
||||
debug!("renew_disk: start {:?}", ep);
|
||||
|
||||
let (new_disk, fm) = match Self::connect_endpoint(ep).await {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
warn!("renew_disk: connect_endpoint err {:?}", &e);
|
||||
if ep.is_local && e == DiskError::UnformattedDisk {
|
||||
info!("renew_disk unformatteddisk will trigger heal_disk, {:?}", ep);
|
||||
let set_disk_id = format!("pool_{}_set_{}", ep.pool_idx, ep.set_idx);
|
||||
let _ = send_heal_disk(set_disk_id, Some(HealChannelPriority::Normal)).await;
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let (set_idx, disk_idx) = match self.find_disk_index(&fm) {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
warn!("renew_disk: find_disk_index err {:?}", e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Check that the endpoint matches
|
||||
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
|
||||
if new_disk.is_local() {
|
||||
let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await;
|
||||
let path = new_disk.endpoint().to_string();
|
||||
global_local_disk_map.insert(path, Some(new_disk.clone()));
|
||||
|
||||
if is_dist_erasure().await {
|
||||
let mut local_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.write().await;
|
||||
local_set_drives[self.pool_index][set_idx][disk_idx] = Some(new_disk.clone());
|
||||
}
|
||||
}
|
||||
|
||||
debug!("renew_disk: update {:?}", fm.erasure.this);
|
||||
|
||||
let mut disk_lock = self.disks.write().await;
|
||||
disk_lock[disk_idx] = Some(new_disk);
|
||||
}
|
||||
|
||||
pub(super) fn find_disk_index(&self, fm: &FormatV3) -> Result<(usize, usize)> {
|
||||
self.format.check_other(fm)?;
|
||||
|
||||
if fm.erasure.this.is_nil() {
|
||||
return Err(Error::other("DriveID: offline"));
|
||||
}
|
||||
|
||||
for i in 0..self.format.erasure.sets.len() {
|
||||
for j in 0..self.format.erasure.sets[0].len() {
|
||||
if fm.erasure.this == self.format.erasure.sets[i][j] {
|
||||
return Ok((i, j));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::other("DriveID: not found"))
|
||||
}
|
||||
|
||||
pub(super) async fn connect_endpoint(ep: &Endpoint) -> disk::error::Result<(DiskStore, FormatV3)> {
|
||||
let disk = new_disk(ep, &DiskOption::default()).await?;
|
||||
|
||||
let fm = load_format_erasure(&disk, false).await?;
|
||||
|
||||
Ok((disk, fm))
|
||||
}
|
||||
|
||||
pub(super) async fn get_online_disk_with_healing(&self, incl_healing: bool) -> Result<(Vec<Option<DiskStore>>, bool)> {
|
||||
let (new_disks, _, healing) = self.get_online_disk_with_healing_and_info(incl_healing).await?;
|
||||
Ok((new_disks, healing > 0))
|
||||
}
|
||||
|
||||
pub(super) async fn get_online_disk_with_healing_and_info(
|
||||
&self,
|
||||
incl_healing: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<DiskInfo>, usize)> {
|
||||
let mut infos = vec![DiskInfo::default(); self.disks.read().await.len()];
|
||||
for (idx, disk) in self.disks.write().await.iter().enumerate() {
|
||||
if let Some(disk) = disk {
|
||||
match disk.disk_info(&DiskInfoOptions::default()).await {
|
||||
Ok(disk_info) => infos[idx] = disk_info,
|
||||
Err(err) => infos[idx].error = err.to_string(),
|
||||
}
|
||||
} else {
|
||||
infos[idx].error = "disk not found".to_string();
|
||||
}
|
||||
}
|
||||
|
||||
let mut new_disks = Vec::new();
|
||||
let mut healing_disks = Vec::new();
|
||||
let mut scanning_disks = Vec::new();
|
||||
let mut new_infos = Vec::new();
|
||||
let mut healing_infos = Vec::new();
|
||||
let mut scanning_infos = Vec::new();
|
||||
let mut healing = 0;
|
||||
|
||||
infos.iter().zip(self.disks.write().await.iter()).for_each(|(info, disk)| {
|
||||
if info.error.is_empty() {
|
||||
if info.healing {
|
||||
healing += 1;
|
||||
if incl_healing {
|
||||
healing_disks.push(disk.clone());
|
||||
healing_infos.push(info.clone());
|
||||
}
|
||||
} else if !info.scanning {
|
||||
new_disks.push(disk.clone());
|
||||
new_infos.push(info.clone());
|
||||
} else {
|
||||
scanning_disks.push(disk.clone());
|
||||
scanning_infos.push(info.clone());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Prefer non-scanning disks over disks which are currently being scanned.
|
||||
new_disks.extend(scanning_disks);
|
||||
new_infos.extend(scanning_infos);
|
||||
|
||||
// Then add healing disks.
|
||||
new_disks.extend(healing_disks);
|
||||
new_infos.extend(healing_infos);
|
||||
|
||||
Ok((new_disks, new_infos, healing))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,551 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) fn reduce_common_data_dir(data_dirs: &Vec<Option<Uuid>>, write_quorum: usize) -> Option<Uuid> {
|
||||
let mut data_dirs_count = HashMap::new();
|
||||
|
||||
for ddir in data_dirs {
|
||||
*data_dirs_count.entry(ddir).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
let mut max = 0;
|
||||
let mut data_dir = None;
|
||||
for (ddir, count) in data_dirs_count {
|
||||
if count > max {
|
||||
max = count;
|
||||
data_dir = *ddir;
|
||||
}
|
||||
}
|
||||
|
||||
if max >= write_quorum { data_dir } else { None }
|
||||
}
|
||||
|
||||
pub(super) fn get_upload_id_dir(bucket: &str, object: &str, upload_id: &str) -> String {
|
||||
let upload_uuid = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(upload_id.as_bytes())
|
||||
.and_then(|v| {
|
||||
String::from_utf8(v).map_or(Ok(upload_id.to_owned()), |v| {
|
||||
let parts: Vec<_> = v.splitn(2, '.').collect();
|
||||
if parts.len() == 2 {
|
||||
Ok(parts[1].to_string())
|
||||
} else {
|
||||
Ok(upload_id.to_string())
|
||||
}
|
||||
})
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
format!("{}/{}", Self::get_multipart_sha_dir(bucket, object), upload_uuid)
|
||||
}
|
||||
|
||||
pub(super) fn get_multipart_sha_dir(bucket: &str, object: &str) -> String {
|
||||
let path = format!("{bucket}/{object}");
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(path);
|
||||
hex(hasher.finalize())
|
||||
}
|
||||
|
||||
pub(super) fn common_parity(parities: &[i32], default_parity_count: i32) -> i32 {
|
||||
let n = parities.len() as i32;
|
||||
|
||||
let mut occ_map: HashMap<i32, i32> = HashMap::new();
|
||||
for &p in parities {
|
||||
*occ_map.entry(p).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
let mut max_occ = 0;
|
||||
let mut cparity = 0;
|
||||
for (&parity, &occ) in &occ_map {
|
||||
if parity == -1 {
|
||||
// Ignore non defined parity
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut read_quorum = n - parity;
|
||||
if default_parity_count > 0 && parity == 0 {
|
||||
// In this case, parity == 0 implies that this object version is a
|
||||
// delete marker
|
||||
read_quorum = n / 2 + 1;
|
||||
}
|
||||
if occ < read_quorum {
|
||||
// Ignore this parity since we don't have enough shards for read quorum
|
||||
continue;
|
||||
}
|
||||
|
||||
if occ > max_occ {
|
||||
max_occ = occ;
|
||||
cparity = parity;
|
||||
}
|
||||
}
|
||||
|
||||
if max_occ == 0 {
|
||||
// Did not find anything useful
|
||||
return -1;
|
||||
}
|
||||
cparity
|
||||
}
|
||||
|
||||
pub(super) fn list_object_modtimes(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> Vec<Option<OffsetDateTime>> {
|
||||
let mut times = vec![None; parts_metadata.len()];
|
||||
|
||||
for (i, metadata) in parts_metadata.iter().enumerate() {
|
||||
if errs[i].is_some() {
|
||||
continue;
|
||||
}
|
||||
|
||||
times[i] = metadata.mod_time
|
||||
}
|
||||
|
||||
times
|
||||
}
|
||||
|
||||
pub(super) fn common_time(times: &[Option<OffsetDateTime>], quorum: usize) -> Option<OffsetDateTime> {
|
||||
let (time, count) = Self::common_time_and_occurrence(times);
|
||||
if count >= quorum { time } else { None }
|
||||
}
|
||||
|
||||
pub(super) fn common_time_and_occurrence(times: &[Option<OffsetDateTime>]) -> (Option<OffsetDateTime>, usize) {
|
||||
let mut time_occurrence_map = HashMap::new();
|
||||
|
||||
// Ignore the uuid sentinel and count the rest.
|
||||
for time in times.iter().flatten() {
|
||||
*time_occurrence_map.entry(time.unix_timestamp_nanos()).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
let mut maxima = 0; // Counter for remembering max occurrence of elements.
|
||||
let mut latest = 0;
|
||||
|
||||
// Find the common cardinality from previously collected
|
||||
// occurrences of elements.
|
||||
for (&nano, &count) in &time_occurrence_map {
|
||||
if count < maxima {
|
||||
continue;
|
||||
}
|
||||
|
||||
// We are at or above maxima
|
||||
if count > maxima || nano > latest {
|
||||
maxima = count;
|
||||
latest = nano;
|
||||
}
|
||||
}
|
||||
|
||||
if latest == 0 {
|
||||
return (None, maxima);
|
||||
}
|
||||
|
||||
if let Ok(time) = OffsetDateTime::from_unix_timestamp_nanos(latest) {
|
||||
(Some(time), maxima)
|
||||
} else {
|
||||
(None, maxima)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn common_etag(etags: &[Option<String>], quorum: usize) -> Option<String> {
|
||||
let (etag, count) = Self::common_etags(etags);
|
||||
if count >= quorum { etag } else { None }
|
||||
}
|
||||
|
||||
pub(super) fn common_etags(etags: &[Option<String>]) -> (Option<String>, usize) {
|
||||
let mut etags_map = HashMap::new();
|
||||
|
||||
for etag in etags.iter().flatten() {
|
||||
*etags_map.entry(etag).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
let mut maxima = 0; // Counter for remembering max occurrence of elements.
|
||||
let mut latest = None;
|
||||
|
||||
for (&etag, &count) in &etags_map {
|
||||
if count < maxima {
|
||||
continue;
|
||||
}
|
||||
|
||||
// We are at or above maxima
|
||||
if count > maxima {
|
||||
maxima = count;
|
||||
latest = Some(etag.clone());
|
||||
}
|
||||
}
|
||||
|
||||
(latest, maxima)
|
||||
}
|
||||
|
||||
pub(super) fn list_object_etags(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> Vec<Option<String>> {
|
||||
let mut etags = vec![None; parts_metadata.len()];
|
||||
|
||||
for (i, metadata) in parts_metadata.iter().enumerate() {
|
||||
if errs[i].is_some() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(etag) = metadata.metadata.get("etag") {
|
||||
etags[i] = Some(etag.clone())
|
||||
}
|
||||
}
|
||||
|
||||
etags
|
||||
}
|
||||
|
||||
pub(super) fn list_object_parities(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> Vec<i32> {
|
||||
let total_shards = parts_metadata.len();
|
||||
let half = total_shards as i32 / 2;
|
||||
let mut parities: Vec<i32> = vec![-1; total_shards];
|
||||
|
||||
for (index, metadata) in parts_metadata.iter().enumerate() {
|
||||
if errs[index].is_some() {
|
||||
parities[index] = -1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if !metadata.is_valid() {
|
||||
parities[index] = -1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if metadata.deleted || metadata.size == 0 {
|
||||
parities[index] = half;
|
||||
// } else if metadata.transition_status == "TransitionComplete" {
|
||||
// TODO: metadata.transition_status
|
||||
// parities[index] = total_shards - (total_shards / 2 + 1);
|
||||
} else {
|
||||
parities[index] = metadata.erasure.parity_blocks as i32;
|
||||
}
|
||||
}
|
||||
parities
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(parts_metadata))]
|
||||
pub(super) fn object_quorum_from_meta(
|
||||
parts_metadata: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
default_parity_count: usize,
|
||||
) -> disk::error::Result<(i32, i32)> {
|
||||
let expected_rquorum = if default_parity_count == 0 {
|
||||
parts_metadata.len()
|
||||
} else {
|
||||
parts_metadata.len() / 2
|
||||
};
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(errs, OBJECT_OP_IGNORED_ERRS, expected_rquorum) {
|
||||
// let object = parts_metadata.first().map(|v| v.name.clone()).unwrap_or_default();
|
||||
// error!("object_quorum_from_meta: {:?}, errs={:?}, object={:?}", err, errs, object);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if default_parity_count == 0 {
|
||||
return Ok((parts_metadata.len() as i32, parts_metadata.len() as i32));
|
||||
}
|
||||
|
||||
let parities = Self::list_object_parities(parts_metadata, errs);
|
||||
|
||||
let parity_blocks = Self::common_parity(&parities, default_parity_count as i32);
|
||||
|
||||
if parity_blocks < 0 {
|
||||
error!("object_quorum_from_meta: parity_blocks < 0, errs={:?}", errs);
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let data_blocks = parts_metadata.len() as i32 - parity_blocks;
|
||||
let write_quorum = if data_blocks == parity_blocks {
|
||||
data_blocks + 1
|
||||
} else {
|
||||
data_blocks
|
||||
};
|
||||
|
||||
Ok((data_blocks, write_quorum))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, parts_metadata))]
|
||||
pub(super) fn list_online_disks(
|
||||
disks: &[Option<DiskStore>],
|
||||
parts_metadata: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
quorum: usize,
|
||||
) -> (Vec<Option<DiskStore>>, Option<OffsetDateTime>, Option<String>) {
|
||||
let mod_times = Self::list_object_modtimes(parts_metadata, errs);
|
||||
let etags = Self::list_object_etags(parts_metadata, errs);
|
||||
|
||||
let mod_time = Self::common_time(&mod_times, quorum);
|
||||
let etag = Self::common_etag(&etags, quorum);
|
||||
|
||||
let mut new_disk = vec![None; disks.len()];
|
||||
|
||||
for (i, &t) in mod_times.iter().enumerate() {
|
||||
if parts_metadata[i].is_valid() && mod_time == t {
|
||||
new_disk[i].clone_from(&disks[i]);
|
||||
}
|
||||
}
|
||||
|
||||
(new_disk, mod_time, etag)
|
||||
}
|
||||
|
||||
pub(super) fn pick_valid_fileinfo(
|
||||
metas: &[FileInfo],
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
etag: Option<String>,
|
||||
quorum: usize,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum)
|
||||
}
|
||||
|
||||
pub(super) fn find_file_info_in_quorum(
|
||||
metas: &[FileInfo],
|
||||
mod_time: &Option<OffsetDateTime>,
|
||||
etag: &Option<String>,
|
||||
quorum: usize,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
if quorum < 1 {
|
||||
warn!("find_file_info_in_quorum: quorum < 1");
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let mut meta_hashes = vec![None; metas.len()];
|
||||
let mut hasher = Sha256::new();
|
||||
|
||||
for (i, meta) in metas.iter().enumerate() {
|
||||
if !meta.is_valid() {
|
||||
debug!(
|
||||
index = i,
|
||||
valid = false,
|
||||
version_id = ?meta.version_id,
|
||||
mod_time = ?meta.mod_time,
|
||||
"find_file_info_in_quorum: skipping invalid meta"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
debug!(
|
||||
index = i,
|
||||
valid = true,
|
||||
version_id = ?meta.version_id,
|
||||
mod_time = ?meta.mod_time,
|
||||
deleted = meta.deleted,
|
||||
size = meta.size,
|
||||
"find_file_info_in_quorum: inspecting meta"
|
||||
);
|
||||
|
||||
let etag_only = mod_time.is_none() && etag.is_some() && meta.get_etag().is_some_and(|v| &v == etag.as_ref().unwrap());
|
||||
let mod_valid = mod_time == &meta.mod_time;
|
||||
|
||||
if etag_only || mod_valid {
|
||||
for part in meta.parts.iter() {
|
||||
hasher.update(format!("part.{}", part.number).as_bytes());
|
||||
hasher.update(format!("part.{}", part.size).as_bytes());
|
||||
}
|
||||
|
||||
if !meta.deleted && meta.size != 0 {
|
||||
hasher.update(format!("{}+{}", meta.erasure.data_blocks, meta.erasure.parity_blocks).as_bytes());
|
||||
hasher.update(format!("{:?}", meta.erasure.distribution).as_bytes());
|
||||
}
|
||||
|
||||
if meta.is_remote() {
|
||||
// TODO:
|
||||
}
|
||||
|
||||
// TODO: IsEncrypted
|
||||
|
||||
// TODO: IsCompressed
|
||||
|
||||
meta_hashes[i] = Some(hex(hasher.clone().finalize().as_slice()));
|
||||
|
||||
hasher.reset();
|
||||
} else {
|
||||
debug!(
|
||||
index = i,
|
||||
etag_only_match = etag_only,
|
||||
mod_valid_match = mod_valid,
|
||||
"find_file_info_in_quorum: meta does not match common etag or mod_time, skipping hash calculation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut count_map = HashMap::new();
|
||||
|
||||
for hash in meta_hashes.iter().flatten() {
|
||||
*count_map.entry(hash).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
let mut max_val = None;
|
||||
let mut max_count = 0;
|
||||
|
||||
for (&val, &count) in &count_map {
|
||||
if count > max_count {
|
||||
max_val = Some(val);
|
||||
max_count = count;
|
||||
}
|
||||
}
|
||||
|
||||
if max_count < quorum {
|
||||
warn!("find_file_info_in_quorum: max_count < quorum, max_val={:?}", max_val);
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let mut found_fi = None;
|
||||
let mut found = false;
|
||||
|
||||
let mut valid_obj_map = HashMap::new();
|
||||
|
||||
for (i, op_hash) in meta_hashes.iter().enumerate() {
|
||||
if let Some(hash) = op_hash
|
||||
&& let Some(max_hash) = max_val
|
||||
&& hash == max_hash
|
||||
{
|
||||
if metas[i].is_valid() && !found {
|
||||
found_fi = Some(metas[i].clone());
|
||||
found = true;
|
||||
}
|
||||
|
||||
let props = ObjProps {
|
||||
mod_time: metas[i].mod_time,
|
||||
num_versions: metas[i].num_versions,
|
||||
};
|
||||
|
||||
*valid_obj_map.entry(props).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if found {
|
||||
let mut fi = found_fi.unwrap();
|
||||
|
||||
for (val, &count) in &valid_obj_map {
|
||||
if count > quorum {
|
||||
fi.mod_time = val.mod_time;
|
||||
fi.num_versions = val.num_versions;
|
||||
fi.is_latest = val.mod_time.is_none();
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(fi);
|
||||
}
|
||||
|
||||
warn!("find_file_info_in_quorum: fileinfo not found");
|
||||
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
}
|
||||
|
||||
pub(super) fn shuffle_disks_and_parts_metadata_by_index(
|
||||
disks: &[Option<DiskStore>],
|
||||
parts_metadata: &[FileInfo],
|
||||
fi: &FileInfo,
|
||||
) -> (Vec<Option<DiskStore>>, Vec<FileInfo>) {
|
||||
let mut shuffled_disks = vec![None; disks.len()];
|
||||
let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()];
|
||||
let distribution = &fi.erasure.distribution;
|
||||
|
||||
let mut inconsistent = 0;
|
||||
for (k, v) in parts_metadata.iter().enumerate() {
|
||||
if disks[k].is_none() {
|
||||
inconsistent += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if !v.is_valid() {
|
||||
inconsistent += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if distribution[k] != v.erasure.index {
|
||||
inconsistent += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let block_idx = distribution[k];
|
||||
shuffled_parts_metadata[block_idx - 1] = parts_metadata[k].clone();
|
||||
shuffled_disks[block_idx - 1].clone_from(&disks[k]);
|
||||
}
|
||||
|
||||
if inconsistent < fi.erasure.parity_blocks {
|
||||
return (shuffled_disks, shuffled_parts_metadata);
|
||||
}
|
||||
|
||||
Self::shuffle_disks_and_parts_metadata(disks, parts_metadata, fi)
|
||||
}
|
||||
|
||||
pub(super) fn shuffle_disks_and_parts_metadata(
|
||||
disks: &[Option<DiskStore>],
|
||||
parts_metadata: &[FileInfo],
|
||||
fi: &FileInfo,
|
||||
) -> (Vec<Option<DiskStore>>, Vec<FileInfo>) {
|
||||
let init = fi.mod_time.is_none();
|
||||
|
||||
let mut shuffled_disks = vec![None; disks.len()];
|
||||
let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()];
|
||||
let distribution = &fi.erasure.distribution;
|
||||
|
||||
for (k, v) in disks.iter().enumerate() {
|
||||
if v.is_none() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !init && !parts_metadata[k].is_valid() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// if !init && fi.xlv1 != parts_metadata[k].xlv1 {
|
||||
// continue;
|
||||
// }
|
||||
|
||||
let block_idx = distribution[k];
|
||||
shuffled_parts_metadata[block_idx - 1] = parts_metadata[k].clone();
|
||||
shuffled_disks[block_idx - 1].clone_from(&disks[k]);
|
||||
}
|
||||
|
||||
(shuffled_disks, shuffled_parts_metadata)
|
||||
}
|
||||
|
||||
pub(super) fn shuffle_parts_metadata(parts_metadata: &[FileInfo], distribution: &[usize]) -> Vec<FileInfo> {
|
||||
if distribution.is_empty() {
|
||||
return parts_metadata.to_vec();
|
||||
}
|
||||
let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()];
|
||||
// Shuffle slice xl metadata for expected distribution.
|
||||
for index in 0..parts_metadata.len() {
|
||||
let block_index = distribution[index];
|
||||
shuffled_parts_metadata[block_index - 1] = parts_metadata[index].clone();
|
||||
}
|
||||
shuffled_parts_metadata
|
||||
}
|
||||
|
||||
pub(super) fn shuffle_disks(disks: &[Option<DiskStore>], distribution: &[usize]) -> Vec<Option<DiskStore>> {
|
||||
if distribution.is_empty() {
|
||||
return disks.to_vec();
|
||||
}
|
||||
|
||||
let mut shuffled_disks = vec![None; disks.len()];
|
||||
|
||||
for (i, v) in disks.iter().enumerate() {
|
||||
let idx = distribution[i];
|
||||
shuffled_disks[idx - 1].clone_from(v);
|
||||
}
|
||||
|
||||
shuffled_disks
|
||||
}
|
||||
|
||||
pub(super) fn shuffle_check_parts(parts_errs: &[usize], distribution: &[usize]) -> Vec<usize> {
|
||||
if distribution.is_empty() {
|
||||
return parts_errs.to_vec();
|
||||
}
|
||||
let mut shuffled_parts_errs = vec![0; parts_errs.len()];
|
||||
for (i, v) in parts_errs.iter().enumerate() {
|
||||
let idx = distribution[i];
|
||||
shuffled_parts_errs[idx - 1] = *v;
|
||||
}
|
||||
shuffled_parts_errs
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn list_parts(
|
||||
disks: &[Option<DiskStore>],
|
||||
part_path: &str,
|
||||
read_quorum: usize,
|
||||
) -> disk::error::Result<Vec<usize>> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
let mut object_parts = Vec::with_capacity(disks.len());
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
errs.push(None);
|
||||
object_parts.push(res);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
object_parts.push(vec![]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
for drive_parts in object_parts {
|
||||
let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
// part files can be either part.N or part.N.meta
|
||||
for part_path in drive_parts {
|
||||
if let Some(num_str) = part_path.strip_prefix("part.") {
|
||||
if let Some(meta_idx) = num_str.find(".meta") {
|
||||
if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
} else if let Ok(part_num) = num_str.parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Include only part.N.meta files with corresponding part.N
|
||||
for (&part_num, &cnt) in &parts_with_meta_count {
|
||||
if cnt >= 2 {
|
||||
*part_quorum_map.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
|
||||
for (part_num, count) in part_quorum_map {
|
||||
if count >= read_quorum {
|
||||
part_numbers.push(part_num);
|
||||
}
|
||||
}
|
||||
|
||||
part_numbers.sort();
|
||||
|
||||
Ok(part_numbers)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
pub(super) async fn check_upload_id_exists(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
write: bool,
|
||||
) -> Result<(FileInfo, Vec<FileInfo>)> {
|
||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
let disks = self.disks.read().await;
|
||||
|
||||
let disks = disks.clone();
|
||||
|
||||
let (parts_metadata, errs) =
|
||||
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await?;
|
||||
|
||||
let map_err_notfound = |err: DiskError| {
|
||||
if err == DiskError::FileNotFound {
|
||||
return StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
|
||||
}
|
||||
err.into()
|
||||
};
|
||||
|
||||
let (read_quorum, write_quorum) =
|
||||
Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count).map_err(map_err_notfound)?;
|
||||
|
||||
if read_quorum < 0 {
|
||||
error!("check_upload_id_exists: read_quorum < 0, errs={:?}", errs);
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
if write_quorum < 0 {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
let mut quorum = read_quorum as usize;
|
||||
if write {
|
||||
quorum = write_quorum as usize;
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
|
||||
return Err(map_err_notfound(err));
|
||||
}
|
||||
} else if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
|
||||
return Err(map_err_notfound(err));
|
||||
}
|
||||
|
||||
let (_, mod_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, quorum);
|
||||
|
||||
let fi = Self::pick_valid_fileinfo(&parts_metadata, mod_time, etag, quorum)?;
|
||||
|
||||
Ok((fi, parts_metadata))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,808 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn read_parts(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
part_meta_paths: &[String],
|
||||
part_numbers: &[usize],
|
||||
read_quorum: usize,
|
||||
) -> disk::error::Result<Vec<ObjectPartInfo>> {
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
let mut object_parts = Vec::with_capacity(disks.len());
|
||||
|
||||
// Use batch processor for better performance
|
||||
let processor = get_global_processors().read_processor();
|
||||
let bucket = bucket.to_string();
|
||||
let part_meta_paths = part_meta_paths.to_vec();
|
||||
|
||||
let tasks: Vec<_> = disks
|
||||
.iter()
|
||||
.map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let bucket = bucket.clone();
|
||||
let part_meta_paths = part_meta_paths.clone();
|
||||
|
||||
async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.read_parts(&bucket, &part_meta_paths).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let results = processor.execute_batch(tasks).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
errs.push(None);
|
||||
object_parts.push(res);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
object_parts.push(vec![]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut ret = vec![ObjectPartInfo::default(); part_meta_paths.len()];
|
||||
|
||||
for (part_idx, part_info) in part_meta_paths.iter().enumerate() {
|
||||
let mut part_meta_quorum = HashMap::new();
|
||||
let mut part_infos = Vec::new();
|
||||
for (j, parts) in object_parts.iter().enumerate() {
|
||||
if parts.len() != part_meta_paths.len() {
|
||||
*part_meta_quorum.entry(part_info.clone()).or_insert(0) += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if !parts[part_idx].etag.is_empty() {
|
||||
*part_meta_quorum.entry(parts[part_idx].etag.clone()).or_insert(0) += 1;
|
||||
part_infos.push(parts[part_idx].clone());
|
||||
continue;
|
||||
}
|
||||
|
||||
*part_meta_quorum.entry(part_info.clone()).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
let mut max_quorum = 0;
|
||||
let mut max_etag = None;
|
||||
let mut max_part_meta = None;
|
||||
for (etag, quorum) in part_meta_quorum.iter() {
|
||||
if quorum > &max_quorum {
|
||||
max_quorum = *quorum;
|
||||
max_etag = Some(etag);
|
||||
max_part_meta = Some(etag);
|
||||
}
|
||||
}
|
||||
|
||||
let mut found = None;
|
||||
for info in part_infos.iter() {
|
||||
if let Some(etag) = max_etag
|
||||
&& info.etag == *etag
|
||||
{
|
||||
found = Some(info.clone());
|
||||
break;
|
||||
}
|
||||
|
||||
if let Some(part_meta) = max_part_meta
|
||||
&& info.etag.is_empty()
|
||||
&& part_meta.ends_with(format!("part.{0}.meta", info.number).as_str())
|
||||
{
|
||||
found = Some(info.clone());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if let (Some(found), Some(max_etag)) = (found, max_etag)
|
||||
&& !found.etag.is_empty()
|
||||
&& part_meta_quorum.get(max_etag).unwrap_or(&0) >= &read_quorum
|
||||
{
|
||||
ret[part_idx] = found.clone();
|
||||
} else {
|
||||
ret[part_idx] = ObjectPartInfo {
|
||||
number: part_numbers[part_idx],
|
||||
error: Some(format!("part.{} not found", part_numbers[part_idx])),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks))]
|
||||
pub(super) async fn read_all_fileinfo(
|
||||
disks: &[Option<DiskStore>],
|
||||
org_bucket: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
read_data: bool,
|
||||
healing: bool,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>)> {
|
||||
let mut ress = Vec::with_capacity(disks.len());
|
||||
let mut errors = Vec::with_capacity(disks.len());
|
||||
let opts = Arc::new(ReadOptions {
|
||||
read_data,
|
||||
healing,
|
||||
..Default::default()
|
||||
});
|
||||
let org_bucket = Arc::new(org_bucket.to_string());
|
||||
let bucket = Arc::new(bucket.to_string());
|
||||
let object = Arc::new(object.to_string());
|
||||
let version_id = Arc::new(version_id.to_string());
|
||||
let futures = disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let opts = opts.clone();
|
||||
let org_bucket = org_bucket.clone();
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
// Wait for all tasks to complete
|
||||
let results = join_all(futures).await;
|
||||
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => match res {
|
||||
Ok(file_info) => {
|
||||
ress.push(file_info);
|
||||
errors.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
ress.push(FileInfo::default());
|
||||
errors.push(Some(e));
|
||||
}
|
||||
},
|
||||
Err(_) => {
|
||||
ress.push(FileInfo::default());
|
||||
errors.push(Some(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((ress, errors))
|
||||
}
|
||||
|
||||
pub async fn read_version_optimized(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<Vec<FileInfo>> {
|
||||
// Use existing disk selection logic
|
||||
let disks = self.disks.read().await;
|
||||
let required_reads = self.format.erasure.sets.len();
|
||||
|
||||
// Clone parameters outside the closure to avoid lifetime issues
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.to_string();
|
||||
let version_id = version_id.to_string();
|
||||
let opts = opts.clone();
|
||||
|
||||
let processor = get_global_processors().read_processor();
|
||||
let tasks: Vec<_> = disks
|
||||
.iter()
|
||||
.take(required_reads + 2) // Read a few extra for reliability
|
||||
.filter_map(|disk| {
|
||||
disk.as_ref().map(|d| {
|
||||
let disk = d.clone();
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let opts = opts.clone();
|
||||
|
||||
async move { disk.read_version(&bucket, &bucket, &object, &version_id, &opts).await }
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
match processor.execute_batch_with_quorum(tasks, required_reads).await {
|
||||
Ok(results) => Ok(results),
|
||||
Err(_) => Err(DiskError::FileNotFound.into()), // Use existing error type
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn read_all_xl(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
read_data: bool,
|
||||
incl_free_vers: bool,
|
||||
) -> (Vec<FileInfo>, Vec<Option<DiskError>>) {
|
||||
let (fileinfos, errs) = Self::read_all_raw_file_info(disks, bucket, object, read_data).await;
|
||||
|
||||
Self::pick_latest_quorum_files_info(fileinfos, errs, bucket, object, read_data, incl_free_vers).await
|
||||
}
|
||||
|
||||
pub(super) async fn read_all_raw_file_info(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
read_data: bool,
|
||||
) -> (Vec<Option<RawFileInfo>>, Vec<Option<DiskError>>) {
|
||||
let mut ress = Vec::with_capacity(disks.len());
|
||||
let mut errors = Vec::with_capacity(disks.len());
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk in disks.iter() {
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.read_xl(bucket, object, read_data).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
ress.push(Some(res));
|
||||
errors.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
ress.push(None);
|
||||
errors.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(ress, errors)
|
||||
}
|
||||
|
||||
pub(super) async fn pick_latest_quorum_files_info(
|
||||
fileinfos: Vec<Option<RawFileInfo>>,
|
||||
errs: Vec<Option<DiskError>>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
read_data: bool,
|
||||
incl_free_vers: bool,
|
||||
) -> (Vec<FileInfo>, Vec<Option<DiskError>>) {
|
||||
let mut metadata_array = vec![None; fileinfos.len()];
|
||||
let mut meta_file_infos = vec![FileInfo::default(); fileinfos.len()];
|
||||
let mut metadata_shallow_versions = vec![None; fileinfos.len()];
|
||||
|
||||
let mut v2_bufs = {
|
||||
if !read_data {
|
||||
vec![Vec::new(); fileinfos.len()]
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
|
||||
let mut errs = errs;
|
||||
|
||||
for (idx, info_op) in fileinfos.iter().enumerate() {
|
||||
if let Some(info) = info_op {
|
||||
if !read_data {
|
||||
v2_bufs[idx] = info.buf.clone();
|
||||
}
|
||||
|
||||
let xlmeta = match FileMeta::load(&info.buf) {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
errs[idx] = Some(err.into());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
metadata_array[idx] = Some(xlmeta);
|
||||
meta_file_infos[idx] = FileInfo::default();
|
||||
}
|
||||
}
|
||||
|
||||
for (idx, info_op) in metadata_array.iter().enumerate() {
|
||||
if let Some(info) = info_op {
|
||||
metadata_shallow_versions[idx] = Some(info.versions.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let shallow_versions: Vec<Vec<FileMetaShallowVersion>> = metadata_shallow_versions.iter().flatten().cloned().collect();
|
||||
|
||||
let read_quorum = fileinfos.len().div_ceil(2);
|
||||
let versions = merge_file_meta_versions(read_quorum, false, 1, &shallow_versions);
|
||||
let meta = FileMeta {
|
||||
versions,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let finfo = match meta.into_fileinfo(bucket, object, "", true, incl_free_vers, true) {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
for item in errs.iter_mut() {
|
||||
if item.is_none() {
|
||||
*item = Some(err.clone().into());
|
||||
}
|
||||
}
|
||||
|
||||
return (meta_file_infos, errs);
|
||||
}
|
||||
};
|
||||
|
||||
if !finfo.is_valid() {
|
||||
for item in errs.iter_mut() {
|
||||
if item.is_none() {
|
||||
*item = Some(DiskError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
|
||||
return (meta_file_infos, errs);
|
||||
}
|
||||
|
||||
let vid = finfo.version_id.unwrap_or(Uuid::nil());
|
||||
|
||||
for (idx, meta_op) in metadata_array.iter().enumerate() {
|
||||
if let Some(meta) = meta_op {
|
||||
match meta.into_fileinfo(bucket, object, vid.to_string().as_str(), read_data, incl_free_vers, true) {
|
||||
Ok(res) => meta_file_infos[idx] = res,
|
||||
Err(err) => errs[idx] = Some(err.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(meta_file_infos, errs)
|
||||
}
|
||||
|
||||
pub(super) async fn read_multiple_files(
|
||||
disks: &[Option<DiskStore>],
|
||||
req: ReadMultipleReq,
|
||||
read_quorum: usize,
|
||||
) -> Vec<ReadMultipleResp> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let mut ress = Vec::with_capacity(disks.len());
|
||||
let mut errors = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk in disks.iter() {
|
||||
let req = req.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.read_multiple(req).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
ress.push(Some(res));
|
||||
errors.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
ress.push(None);
|
||||
errors.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// debug!("ReadMultipleResp ress {:?}", ress);
|
||||
// debug!("ReadMultipleResp errors {:?}", errors);
|
||||
|
||||
let mut ret = Vec::with_capacity(req.files.len());
|
||||
|
||||
for want in req.files.iter() {
|
||||
let mut quorum = 0;
|
||||
|
||||
let mut get_res = ReadMultipleResp::default();
|
||||
|
||||
for res in ress.iter() {
|
||||
if res.is_none() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let disk_res = res.as_ref().unwrap();
|
||||
|
||||
for resp in disk_res.iter() {
|
||||
if !resp.error.is_empty() || !resp.exists {
|
||||
continue;
|
||||
}
|
||||
|
||||
if &resp.file != want || resp.bucket != req.bucket || resp.prefix != req.prefix {
|
||||
continue;
|
||||
}
|
||||
quorum += 1;
|
||||
|
||||
if get_res.mod_time > resp.mod_time || get_res.data.len() > resp.data.len() {
|
||||
continue;
|
||||
}
|
||||
|
||||
get_res = resp.clone();
|
||||
}
|
||||
}
|
||||
|
||||
if quorum < read_quorum {
|
||||
// debug!("quorum < read_quorum: {} < {}", quorum, read_quorum);
|
||||
get_res.exists = false;
|
||||
get_res.error = Error::ErasureReadQuorum.to_string();
|
||||
get_res.data = Vec::new();
|
||||
}
|
||||
|
||||
ret.push(get_res);
|
||||
}
|
||||
|
||||
// log err
|
||||
|
||||
ret
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
pub(super) async fn get_object_fileinfo(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
|
||||
let disks = self.disks.read().await;
|
||||
|
||||
let disks = disks.clone();
|
||||
|
||||
let vid = opts.version_id.clone().unwrap_or_default();
|
||||
|
||||
// TODO: optimize concurrency and break once enough slots are available
|
||||
let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await?;
|
||||
// warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
|
||||
// warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
|
||||
|
||||
let _min_disks = self.set_drive_count - self.default_parity_count;
|
||||
|
||||
let (read_quorum, _) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
// error!("Self::object_quorum_from_meta: {:?}, bucket: {}, object: {}", &e, bucket, object);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum as usize) {
|
||||
error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object);
|
||||
return Err(to_object_err(err.into(), vec![bucket, object]));
|
||||
}
|
||||
|
||||
let (op_online_disks, mot_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize);
|
||||
|
||||
let fi = Self::pick_valid_fileinfo(&parts_metadata, mot_time, etag, read_quorum as usize)?;
|
||||
if errs.iter().any(|err| err.is_some()) {
|
||||
let _ =
|
||||
rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
fi.volume.to_string(), // bucket
|
||||
Some(fi.name.to_string()), // object_prefix
|
||||
false, // force_start
|
||||
Some(HealChannelPriority::Normal), // priority
|
||||
Some(self.pool_index), // pool_index
|
||||
Some(self.set_index), // set_index
|
||||
))
|
||||
.await;
|
||||
}
|
||||
// debug!("get_object_fileinfo pick fi {:?}", &fi);
|
||||
|
||||
// let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
|
||||
|
||||
Ok((fi, parts_metadata, op_online_disks))
|
||||
}
|
||||
|
||||
pub(super) async fn get_object_info_and_quorum(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> (ObjectInfo, usize, Option<StorageError>) {
|
||||
let fi = match self.get_object_fileinfo(bucket, object, opts, false).await {
|
||||
Ok((fi, _, _)) => fi,
|
||||
Err(e) => return (ObjectInfo::default(), 0, Some(e)),
|
||||
};
|
||||
|
||||
let write_quorum = fi.write_quorum(self.default_write_quorum());
|
||||
|
||||
let oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
|
||||
if !fi.version_purge_status().is_empty() && opts.version_id.is_some() {
|
||||
return (
|
||||
oi,
|
||||
write_quorum,
|
||||
Some(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])),
|
||||
);
|
||||
}
|
||||
|
||||
if fi.deleted {
|
||||
return if opts.version_id.is_none() || opts.delete_marker {
|
||||
(oi, write_quorum, Some(to_object_err(StorageError::FileNotFound, vec![bucket, object])))
|
||||
} else {
|
||||
(
|
||||
oi,
|
||||
write_quorum,
|
||||
Some(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])),
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
(oi, write_quorum, None)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn get_object_with_fileinfo<W>(
|
||||
// &self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
offset: usize,
|
||||
length: i64,
|
||||
writer: &mut W,
|
||||
fi: FileInfo,
|
||||
files: Vec<FileInfo>,
|
||||
disks: &[Option<DiskStore>],
|
||||
set_index: usize,
|
||||
pool_index: usize,
|
||||
) -> Result<()>
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
debug!(bucket, object, requested_length = length, offset, "get_object_with_fileinfo start");
|
||||
let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, &files, &fi);
|
||||
|
||||
let total_size = fi.size as usize;
|
||||
|
||||
let length = if length < 0 {
|
||||
fi.size as usize - offset
|
||||
} else {
|
||||
length as usize
|
||||
};
|
||||
|
||||
if offset > total_size || offset + length > total_size {
|
||||
error!("get_object_with_fileinfo offset out of range: {}, total_size: {}", offset, total_size);
|
||||
return Err(Error::other("offset out of range"));
|
||||
}
|
||||
|
||||
let (part_index, mut part_offset) = fi.to_part_offset(offset)?;
|
||||
|
||||
let mut end_offset = offset;
|
||||
if length > 0 {
|
||||
end_offset += length - 1
|
||||
}
|
||||
|
||||
let (last_part_index, last_part_relative_offset) = fi.to_part_offset(end_offset)?;
|
||||
|
||||
debug!(
|
||||
bucket,
|
||||
object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds"
|
||||
);
|
||||
|
||||
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
|
||||
|
||||
let part_indices: Vec<usize> = (part_index..=last_part_index).collect();
|
||||
debug!(bucket, object, ?part_indices, "Multipart part indices to stream");
|
||||
|
||||
let mut total_read = 0;
|
||||
for current_part in part_indices {
|
||||
if total_read == length {
|
||||
debug!(
|
||||
bucket,
|
||||
object,
|
||||
total_read,
|
||||
requested_length = length,
|
||||
part_index = current_part,
|
||||
"Stopping multipart stream early because accumulated bytes match request"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
let part_number = fi.parts[current_part].number;
|
||||
let part_size = fi.parts[current_part].size;
|
||||
let mut part_length = part_size - part_offset;
|
||||
if part_length > (length - total_read) {
|
||||
part_length = length - total_read
|
||||
}
|
||||
|
||||
let till_offset = erasure.shard_file_offset(part_offset, part_length, part_size);
|
||||
|
||||
let read_offset = (part_offset / erasure.block_size) * erasure.shard_size();
|
||||
|
||||
debug!(
|
||||
bucket,
|
||||
object,
|
||||
part_index = current_part,
|
||||
part_number,
|
||||
part_offset,
|
||||
part_size,
|
||||
part_length,
|
||||
read_offset,
|
||||
till_offset,
|
||||
total_read_before = total_read,
|
||||
requested_length = length,
|
||||
"Streaming multipart part"
|
||||
);
|
||||
|
||||
let mut readers = Vec::with_capacity(disks.len());
|
||||
let mut errors = Vec::with_capacity(disks.len());
|
||||
for (idx, disk_op) in disks.iter().enumerate() {
|
||||
match create_bitrot_reader(
|
||||
files[idx].data.as_deref(),
|
||||
disk_op.as_ref(),
|
||||
bucket,
|
||||
&format!("{}/{}/part.{}", object, files[idx].data_dir.unwrap_or_default(), part_number),
|
||||
read_offset,
|
||||
till_offset,
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::HighwayHash256,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(reader)) => {
|
||||
readers.push(Some(reader));
|
||||
errors.push(None);
|
||||
}
|
||||
Ok(None) => {
|
||||
readers.push(None);
|
||||
errors.push(Some(DiskError::DiskNotFound));
|
||||
}
|
||||
Err(e) => {
|
||||
readers.push(None);
|
||||
errors.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
|
||||
if nil_count < erasure.data_shards {
|
||||
if let Some(read_err) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) {
|
||||
error!("create_bitrot_reader reduce_read_quorum_errs {:?}", &errors);
|
||||
return Err(to_object_err(read_err.into(), vec![bucket, object]));
|
||||
}
|
||||
error!("create_bitrot_reader not enough disks to read: {:?}", &errors);
|
||||
return Err(Error::other(format!("not enough disks to read: {errors:?}")));
|
||||
}
|
||||
|
||||
// Check if we have missing shards even though we can read successfully
|
||||
// This happens when a node was offline during write and comes back online
|
||||
let total_shards = erasure.data_shards + erasure.parity_shards;
|
||||
let available_shards = nil_count;
|
||||
let missing_shards = total_shards - available_shards;
|
||||
|
||||
info!(
|
||||
bucket,
|
||||
object,
|
||||
part_number,
|
||||
total_shards,
|
||||
available_shards,
|
||||
missing_shards,
|
||||
data_shards = erasure.data_shards,
|
||||
parity_shards = erasure.parity_shards,
|
||||
"Shard availability check"
|
||||
);
|
||||
|
||||
if missing_shards > 0 && available_shards >= erasure.data_shards {
|
||||
// We have missing shards but enough to read - trigger background heal
|
||||
info!(
|
||||
bucket,
|
||||
object,
|
||||
part_number,
|
||||
missing_shards,
|
||||
available_shards,
|
||||
pool_index,
|
||||
set_index,
|
||||
"Detected missing shards during read, triggering background heal"
|
||||
);
|
||||
if let Err(e) =
|
||||
rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(pool_index),
|
||||
Some(set_index),
|
||||
))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket,
|
||||
object,
|
||||
part_number,
|
||||
error = %e,
|
||||
"Failed to enqueue heal request for missing shards"
|
||||
);
|
||||
} else {
|
||||
warn!(bucket, object, part_number, "Successfully enqueued heal request for missing shards");
|
||||
}
|
||||
}
|
||||
|
||||
// debug!(
|
||||
// "read part {} part_offset {},part_length {},part_size {} ",
|
||||
// part_number, part_offset, part_length, part_size
|
||||
// );
|
||||
let (written, err) = erasure.decode(writer, readers, part_offset, part_length, part_size).await;
|
||||
debug!(
|
||||
bucket,
|
||||
object,
|
||||
part_index = current_part,
|
||||
part_number,
|
||||
part_length,
|
||||
bytes_written = written,
|
||||
"Finished decoding multipart part"
|
||||
);
|
||||
if let Some(e) = err {
|
||||
let de_err: DiskError = e.into();
|
||||
let mut has_err = true;
|
||||
if written == part_length {
|
||||
match de_err {
|
||||
DiskError::FileNotFound | DiskError::FileCorrupt => {
|
||||
error!("erasure.decode err 111 {:?}", &de_err);
|
||||
if let Err(e) = rustfs_common::heal_channel::send_heal_request(
|
||||
rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(pool_index),
|
||||
Some(set_index),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket,
|
||||
object,
|
||||
part_number,
|
||||
error = %e,
|
||||
"Failed to enqueue heal request after decode error"
|
||||
);
|
||||
}
|
||||
has_err = false;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if has_err {
|
||||
error!("erasure.decode err {} {:?}", written, &de_err);
|
||||
return Err(de_err.into());
|
||||
}
|
||||
}
|
||||
|
||||
// debug!("ec decode {} written size {}", part_number, n);
|
||||
|
||||
total_read += part_length;
|
||||
part_offset = 0;
|
||||
}
|
||||
|
||||
// debug!("read end");
|
||||
|
||||
debug!(bucket, object, total_read, expected_length = length, "Multipart read finished");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl SetDisks {
|
||||
pub async fn update_restore_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
obj_info: &ObjectInfo,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
let mut oi = obj_info.clone();
|
||||
oi.metadata_only = true;
|
||||
|
||||
oi.user_defined.remove(X_AMZ_RESTORE.as_str());
|
||||
|
||||
let version_id = oi.version_id.map(|v| v.to_string());
|
||||
let _obj = self
|
||||
.copy_object(
|
||||
bucket,
|
||||
object,
|
||||
bucket,
|
||||
object,
|
||||
&mut oi,
|
||||
&ObjectOptions {
|
||||
version_id: version_id.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
&ObjectOptions {
|
||||
version_id,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,629 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) fn default_read_quorum(&self) -> usize {
|
||||
self.set_drive_count - self.default_parity_count
|
||||
}
|
||||
|
||||
pub(super) fn default_write_quorum(&self) -> usize {
|
||||
let mut data_count = self.set_drive_count - self.default_parity_count;
|
||||
if data_count == self.default_parity_count {
|
||||
data_count += 1
|
||||
}
|
||||
|
||||
data_count
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub(super) async fn rename_data(
|
||||
disks: &[Option<DiskStore>],
|
||||
src_bucket: &str,
|
||||
src_object: &str,
|
||||
file_infos: &[FileInfo],
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<(Vec<Option<DiskStore>>, Option<Vec<u8>>, Option<Uuid>)> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
let dst_object = Arc::new(dst_object.to_string());
|
||||
|
||||
for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() {
|
||||
let mut file_info = file_info.clone();
|
||||
let disk = disk.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
|
||||
futures.push(tokio::spawn(async move {
|
||||
if file_info.erasure.index == 0 {
|
||||
file_info.erasure.index = i + 1;
|
||||
}
|
||||
|
||||
if !file_info.is_valid() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
if let Some(disk) = disk {
|
||||
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
let mut disk_versions = vec![None; disks.len()];
|
||||
let mut data_dirs = vec![None; disks.len()];
|
||||
|
||||
let results = join_all(futures).await;
|
||||
|
||||
for (idx, result) in results.iter().enumerate() {
|
||||
match result.as_ref().map_err(|_| DiskError::Unexpected)? {
|
||||
Ok(res) => {
|
||||
data_dirs[idx] = res.old_data_dir;
|
||||
disk_versions[idx].clone_from(&res.sign);
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
// TODO: add concurrency
|
||||
for (i, err) in errs.iter().enumerate() {
|
||||
if err.is_some() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(disk) = disks[i].as_ref() {
|
||||
let fi = file_infos[i].clone();
|
||||
let old_data_dir = data_dirs[i];
|
||||
let disk = disk.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
futures.push(tokio::spawn(async move {
|
||||
let _ = disk
|
||||
.delete_version(
|
||||
&src_bucket,
|
||||
&src_object,
|
||||
fi,
|
||||
false,
|
||||
DeleteOptions {
|
||||
undo_write: true,
|
||||
old_data_dir,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
debug!("rename_data delete_version err {:?}", e);
|
||||
e
|
||||
});
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
let _ = join_all(futures).await;
|
||||
return Err(ret_err);
|
||||
}
|
||||
|
||||
let versions = None;
|
||||
// TODO: reduceCommonVersions
|
||||
|
||||
let data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum);
|
||||
|
||||
// // TODO: reduce_common_data_dir
|
||||
// if let Some(old_dir) = rename_ress
|
||||
// .iter()
|
||||
// .filter_map(|v| if v.is_some() { v.as_ref().unwrap().old_data_dir } else { None })
|
||||
// .map(|v| v.to_string())
|
||||
// .next()
|
||||
// {
|
||||
// let cm_errs = self.commit_rename_data_dir(&shuffle_disks, &bucket, &object, &old_dir).await;
|
||||
// warn!("put_object commit_rename_data_dir:{:?}", &cm_errs);
|
||||
// }
|
||||
|
||||
// self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
|
||||
|
||||
Ok((Self::eval_disks(disks, &errs), versions, data_dir))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[tracing::instrument(level = "debug", skip(self, disks))]
|
||||
pub(super) async fn commit_rename_data_dir(
|
||||
&self,
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data_dir: &str,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let file_path = Arc::new(format!("{object}/{data_dir}"));
|
||||
let bucket = Arc::new(bucket.to_string());
|
||||
let futures = disks.iter().map(|disk| {
|
||||
let file_path = file_path.clone();
|
||||
let bucket = bucket.clone();
|
||||
let disk = disk.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Some(disk) = disk {
|
||||
(disk
|
||||
.delete(
|
||||
&bucket,
|
||||
&file_path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await)
|
||||
.err()
|
||||
} else {
|
||||
Some(DiskError::DiskNotFound)
|
||||
}
|
||||
})
|
||||
});
|
||||
let errs: Vec<Option<DiskError>> = join_all(futures)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|e| e.unwrap_or(Some(DiskError::Unexpected)))
|
||||
.collect();
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub(super) async fn cleanup_multipart_path(&self, paths: &[String]) {
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
// Use improved simple batch processor instead of join_all for better performance
|
||||
let processor = get_global_processors().write_processor();
|
||||
|
||||
let tasks: Vec<_> = disks
|
||||
.iter()
|
||||
.map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let paths = paths.to_vec();
|
||||
|
||||
async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &paths).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let results = processor.execute_batch(tasks).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => {
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if errs.iter().any(|e| e.is_some()) {
|
||||
warn!("cleanup_multipart_path errs {:?}", &errs);
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(disks, meta))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn rename_part(
|
||||
&self,
|
||||
disks: &[Option<DiskStore>],
|
||||
src_bucket: &str,
|
||||
src_object: &str,
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
meta: Bytes,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<Vec<Option<DiskStore>>> {
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
let dst_object = Arc::new(dst_object.to_string());
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
let futures = disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let meta = meta.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result? {
|
||||
Ok(_) => {
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
warn!("rename_part errs {:?}", &errs);
|
||||
self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let disks = Self::eval_disks(disks, &errs);
|
||||
Ok(disks)
|
||||
}
|
||||
|
||||
pub(super) fn eval_disks(disks: &[Option<DiskStore>], errs: &[Option<DiskError>]) -> Vec<Option<DiskStore>> {
|
||||
if disks.len() != errs.len() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut online_disks = vec![None; disks.len()];
|
||||
|
||||
for (i, err_op) in errs.iter().enumerate() {
|
||||
if err_op.is_none() {
|
||||
online_disks[i].clone_from(&disks[i]);
|
||||
}
|
||||
}
|
||||
|
||||
online_disks
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(disks, files))]
|
||||
pub(super) async fn write_unique_file_info(
|
||||
disks: &[Option<DiskStore>],
|
||||
org_bucket: &str,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
files: &[FileInfo],
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
let mut file_info = files[i].clone();
|
||||
file_info.erasure.index = i + 1;
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.write_metadata(org_bucket, bucket, prefix, file_info).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => {
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
// TODO: add concurrency
|
||||
for (i, err) in errs.iter().enumerate() {
|
||||
if err.is_some() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(disk) = disks[i].as_ref() {
|
||||
let _ = disk
|
||||
.delete(
|
||||
bucket,
|
||||
&path_join_buf(&[prefix, STORAGE_FORMAT_FILE]),
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!("write meta revert err {:?}", e);
|
||||
e
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn update_object_meta(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
fi: FileInfo,
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> disk::error::Result<()> {
|
||||
self.update_object_meta_with_opts(bucket, object, fi, disks, &UpdateMetadataOpts::default())
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn update_object_meta_with_opts(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
fi: FileInfo,
|
||||
disks: &[Option<DiskStore>],
|
||||
opts: &UpdateMetadataOpts,
|
||||
) -> disk::error::Result<()> {
|
||||
if fi.metadata.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk in disks.iter() {
|
||||
let fi = fi.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.update_metadata(bucket, object, fi, opts).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => {
|
||||
errs.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, fi.write_quorum(self.default_write_quorum())) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn delete_if_dang_ling(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
meta_arr: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
data_errs_by_part: &HashMap<usize, Vec<usize>>,
|
||||
opts: ObjectOptions,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
let (m, can_heal) = is_object_dang_ling(meta_arr, errs, data_errs_by_part);
|
||||
|
||||
if !can_heal {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let mut tags: HashMap<String, String> = HashMap::new();
|
||||
tags.insert("set".to_string(), self.set_index.to_string());
|
||||
tags.insert("pool".to_string(), self.pool_index.to_string());
|
||||
tags.insert("merrs".to_string(), join_errs(errs));
|
||||
tags.insert("derrs".to_string(), format!("{data_errs_by_part:?}"));
|
||||
if m.is_valid() {
|
||||
tags.insert("sz".to_string(), m.size.to_string());
|
||||
tags.insert(
|
||||
"mt".to_string(),
|
||||
m.mod_time
|
||||
.as_ref()
|
||||
.map_or(String::new(), |mod_time| mod_time.unix_timestamp().to_string()),
|
||||
);
|
||||
tags.insert("d:p".to_string(), format!("{}:{}", m.erasure.data_blocks, m.erasure.parity_blocks));
|
||||
} else {
|
||||
tags.insert("invalid".to_string(), "1".to_string());
|
||||
tags.insert(
|
||||
"d:p".to_string(),
|
||||
format!("{}:{}", self.set_drive_count - self.default_parity_count, self.default_parity_count),
|
||||
);
|
||||
}
|
||||
let mut offline = 0;
|
||||
for (i, err) in errs.iter().enumerate() {
|
||||
let mut found = false;
|
||||
if let Some(err) = err
|
||||
&& err == &DiskError::DiskNotFound
|
||||
{
|
||||
found = true;
|
||||
}
|
||||
for p in data_errs_by_part {
|
||||
if let Some(v) = p.1.get(i)
|
||||
&& *v == CHECK_PART_DISK_NOT_FOUND
|
||||
{
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if found {
|
||||
offline += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if offline > 0 {
|
||||
tags.insert("offline".to_string(), offline.to_string());
|
||||
}
|
||||
|
||||
let mut fi = FileInfo::default();
|
||||
if let Some(ref version_id) = opts.version_id {
|
||||
fi.version_id = Uuid::parse_str(version_id).ok();
|
||||
}
|
||||
|
||||
fi.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
for disk_op in disks.iter() {
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.to_string();
|
||||
let fi = fi.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk_op {
|
||||
disk.delete_version(&bucket, &object, fi, false, DeleteOptions::default())
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for (index, result) in results.into_iter().enumerate() {
|
||||
let key = format!("ddisk-{index}");
|
||||
match result {
|
||||
Ok(_) => {
|
||||
tags.insert(key, "<nil>".to_string());
|
||||
}
|
||||
Err(e) => {
|
||||
tags.insert(key, e.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: audit
|
||||
|
||||
Ok(m)
|
||||
}
|
||||
|
||||
pub(super) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let write_quorum = disks.len() / 2 + 1;
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk_op in disks.iter() {
|
||||
let bucket = bucket.to_string();
|
||||
let prefix = prefix.to_string();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk_op {
|
||||
disk.delete(
|
||||
&bucket,
|
||||
&prefix,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let errs = join_all(futures).await.into_iter().map(|v| v.err()).collect::<Vec<_>>();
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn check_write_precondition(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Option<StorageError> {
|
||||
let mut opts = opts.clone();
|
||||
|
||||
let http_preconditions = opts.http_preconditions?;
|
||||
opts.http_preconditions = None;
|
||||
|
||||
// Never claim a lock here, to avoid deadlock
|
||||
// - If no_lock is false, we must have obtained the lock out side of this function
|
||||
// - If no_lock is true, we should not obtain locks
|
||||
opts.no_lock = true;
|
||||
let oi = self.get_object_info(bucket, object, &opts).await;
|
||||
|
||||
match oi {
|
||||
Ok(oi) => {
|
||||
// If top level is a delete marker proceed to upload.
|
||||
if oi.delete_marker {
|
||||
return None;
|
||||
}
|
||||
if should_prevent_write(&oi, http_preconditions.if_none_match, http_preconditions.if_match) {
|
||||
return Some(StorageError::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
|
||||
Err(StorageError::VersionNotFound(_, _, _))
|
||||
| Err(StorageError::ObjectNotFound(_, _))
|
||||
| Err(StorageError::ErasureReadQuorum) => {
|
||||
// When the object is not found,
|
||||
// - if If-Match is set, we should return 404 NotFound
|
||||
// - if If-None-Match is set, we should be able to proceed with the request
|
||||
if http_preconditions.if_match.is_some() {
|
||||
return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
Err(e) => {
|
||||
return Some(e);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user