gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): harden rebalance data movement (#3234)

Browse Source 
* fix(ecstore): harden rebalance data movement

* fix(ecstore): preserve failed rebalance status

* fix(ecstore): avoid rebalance walkdir total timeout

* fix(ecstore): retry rebalance listing timeouts

* fix(ecstore): restrict data movement resume target

* refactor(ecstore): simplify multipart movement target

* fix(ecstore): restore resume target checks

* perf(ecstore): speed up rebalance bucket merges

* fix: keep rebalance listing alive on transient failures

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-06-08 18:40:26 +08:00
committed by GitHub
parent 2fc1cf90e0
commit 9504dff595
11 changed files with 2673 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/cache_value/metacache_set.rs
+48 -1
View File
@@ -56,6 +56,7 @@ async fn peek_with_timeout<R: AsyncRead + Unpin>(reader: &mut MetacacheReader<R>
pub(crate) enum TestReaderBehavior {
Eof,
Stall,
IgnoreCancel,
ProducerError(DiskError),
PartialThenTimeout(Vec<MetaCacheEntry>),
}
@@ -72,6 +73,7 @@ pub struct ListPathRawOptions {
pub min_disks: usize,
pub report_not_found: bool,
pub per_disk_limit: i32,
pub skip_walkdir_total_timeout: bool,
pub agreed: Option<AgreedFn>,
pub partial: Option<PartialFn>,
pub finished: Option<FinishedFn>,
@@ -97,6 +99,7 @@ impl Clone for ListPathRawOptions {
min_disks: self.min_disks,
report_not_found: self.report_not_found,
per_disk_limit: self.per_disk_limit,
skip_walkdir_total_timeout: self.skip_walkdir_total_timeout,
#[cfg(test)]
test_reader_behaviors: self.test_reader_behaviors.clone(),
#[cfg(test)]
@@ -137,6 +140,11 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
cancel_rx_clone.cancelled().await;
return Ok(());
}
TestReaderBehavior::IgnoreCancel => {
let _held_writer = wr;
std::future::pending::<()>().await;
return Ok(());
}
TestReaderBehavior::ProducerError(err) => {
record_producer_error(&producer_errs_clone, disk_idx, &err);
return Err(err);
@@ -162,6 +170,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
filter_prefix: opts_clone.filter_prefix.clone(),
forward_to: opts_clone.forward_to.clone(),
limit: opts_clone.per_disk_limit,
skip_total_timeout: opts_clone.skip_walkdir_total_timeout,
..Default::default()
};
@@ -213,6 +222,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
filter_prefix: opts_clone.filter_prefix.clone(),
forward_to: opts_clone.forward_to.clone(),
limit: opts_clone.per_disk_limit,
skip_total_timeout: opts_clone.skip_walkdir_total_timeout,
..Default::default()
},
&mut wr,
@@ -485,6 +495,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
if let Err(err) = revjob.await.map_err(std::io::Error::other)? {
error!("list_path_raw: revjob err {:?}", err);
cancel_rx.cancel();
for job in jobs {
job.abort();
}
return Err(err);
}
@@ -492,9 +505,15 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
// The merge consumer can finish successfully before every producer finishes
// (for example after reaching EOF quorum while a tolerated drive is stalled,
// or after the requested listing limit is satisfied). Cancel remaining walk
// jobs before joining them so list calls do not wait for slow remote streams.
// jobs before aborting them so list calls do not wait for slow remote streams.
cancel_rx.cancel();
for job in jobs.iter() {
if !job.is_finished() {
job.abort();
}
}
let results = join_all(jobs).await;
let mut job_errs = Vec::new();
for result in results {
@@ -509,6 +528,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
job_errs.push(err);
}
Err(err) => {
if err.is_cancelled() {
continue;
}
error!("list_path_raw join err {:?}", err);
job_errs.push(err.into());
}
@@ -582,6 +604,31 @@ mod tests {
.expect("listing should complete when healthy quorum reached EOF and only a tolerated drive stalled");
}
#[tokio::test]
async fn list_path_raw_aborts_unresponsive_producer_after_quorum_eof() {
let result = timeout(
Duration::from_millis(200),
list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None, None, None],
min_disks: 2,
test_reader_behaviors: vec![
TestReaderBehavior::Eof,
TestReaderBehavior::Eof,
TestReaderBehavior::IgnoreCancel,
],
peek_timeout: Some(Duration::from_millis(20)),
..Default::default()
},
),
)
.await;
let listing = result.expect("list_path_raw should abort unresponsive producer instead of hanging");
assert!(listing.is_ok());
}
#[tokio::test]
async fn list_path_raw_returns_timeout_when_producer_fails_after_partial_entry() {
let seen = Arc::new(Mutex::new(Vec::new()));
crates/ecstore/src/data_movement.rs
+354 -16
View File
@@ -12,11 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::error::{Error, Result};
use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found};
use crate::store::ECStore;
use crate::store_api::{CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store_api::{
CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader,
};
use bytes::Bytes;
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use rustfs_utils::path::encode_dir_object;
use std::io::Cursor;
use std::pin::Pin;
use std::sync::{
@@ -102,13 +105,14 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz
}
}
fn data_movement_complete_multipart_opts(object_info: &ObjectInfo) -> ObjectOptions {
fn data_movement_complete_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions {
ObjectOptions {
versioned: object_info.version_id.is_some(),
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
data_movement: true,
mod_time: object_info.mod_time,
preserve_etag: object_info.etag.clone(),
src_pool_idx,
..Default::default()
}
}
@@ -139,6 +143,111 @@ fn resolve_data_movement_abort_result(
))
}
fn data_movement_stage_error(op_label: &str, stage: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error {
Error::other(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"))
}
fn should_check_data_movement_overwrite_resume(err: &Error) -> bool {
is_err_data_movement_overwrite(err)
}
fn effective_actual_size(info: &ObjectInfo) -> Option<i64> {
info.get_actual_size().ok()
}
fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
source.version_id == target.version_id
&& source.delete_marker == target.delete_marker
&& source.size == target.size
&& effective_actual_size(source) == effective_actual_size(target)
&& source.etag == target.etag
&& source.checksum == target.checksum
&& source.mod_time == target.mod_time
}
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
target_pool_idx != src_pool_idx
}
async fn find_data_movement_target_info(
store: &ECStore,
target_pool_idx: usize,
bucket: &str,
object_info: &ObjectInfo,
) -> Result<Option<ObjectInfo>> {
let opts = ObjectOptions {
versioned: object_info.version_id.is_some(),
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
no_lock: true,
..Default::default()
};
let object = encode_dir_object(object_info.name.as_str());
let Some(pool) = store.pools.get(target_pool_idx) else {
return Err(Error::other(format!(
"data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
)));
};
match pool.get_object_info(bucket, object.as_str(), &opts).await {
Ok(target_info) => Ok(Some(target_info)),
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
Err(err) => Err(err),
}
}
fn resolve_data_movement_overwrite_resume_result(
err: &Error,
target_result: Result<Option<ObjectInfo>>,
source: &ObjectInfo,
src_pool_idx: usize,
target_pool_idx: usize,
) -> Result<bool> {
if !should_check_data_movement_overwrite_resume(err)
|| !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx)
{
return Ok(false);
}
let Some(target) = target_result? else {
return Ok(false);
};
Ok(is_equivalent_data_movement_object(source, &target))
}
async fn should_treat_data_movement_overwrite_as_complete(
store: &ECStore,
src_pool_idx: usize,
target_pool_idx: usize,
bucket: &str,
object_info: &ObjectInfo,
err: &Error,
) -> Result<bool> {
if !should_check_data_movement_overwrite_resume(err) {
return Ok(false);
}
resolve_data_movement_overwrite_resume_result(
err,
find_data_movement_target_info(store, target_pool_idx, bucket, object_info).await,
object_info,
src_pool_idx,
target_pool_idx,
)
}
fn data_movement_part_stage_error(
op_label: &str,
stage: &str,
bucket: &str,
object: &str,
part_number: usize,
err: impl std::fmt::Display,
) -> Error {
Error::other(format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}"))
}
pub(crate) async fn migrate_object(
store: Arc<ECStore>,
pool_idx: usize,
@@ -149,14 +258,24 @@ pub(crate) async fn migrate_object(
let object_info = rd.object_info.clone();
if object_info.is_multipart() {
let res = match store
.new_multipart_upload(&bucket, &object_info.name, &data_movement_new_multipart_opts(&object_info, pool_idx))
let (res, target_pool_idx) = match store
.handle_new_multipart_upload_with_pool_idx(
&bucket,
&object_info.name,
&data_movement_new_multipart_opts(&object_info, pool_idx),
)
.await
{
Ok(res) => res,
Err(err) => {
error!("{op_label}: new_multipart_upload err {:?}", &err);
return Err(err);
return Err(data_movement_stage_error(
op_label,
"new_multipart_upload",
bucket.as_str(),
object_info.name.as_str(),
err,
));
}
};
@@ -167,12 +286,39 @@ pub(crate) async fn migrate_object(
for (i, part) in object_info.parts.iter().enumerate() {
let mut chunk = vec![0u8; part.size];
reader.read_exact(&mut chunk).await?;
reader.read_exact(&mut chunk).await.map_err(|err| {
data_movement_part_stage_error(
op_label,
"read_part",
bucket.as_str(),
object_info.name.as_str(),
part.number,
Error::other(err.to_string()),
)
})?;
let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?;
let part_size = i64::try_from(part.size).map_err(|_| {
data_movement_part_stage_error(
op_label,
"prepare_part",
bucket.as_str(),
object_info.name.as_str(),
part.number,
Error::other("part size overflow"),
)
})?;
let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size };
let index = decode_part_index(part.index.as_ref());
let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?;
let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index).map_err(|err| {
data_movement_part_stage_error(
op_label,
"prepare_part",
bucket.as_str(),
object_info.name.as_str(),
part.number,
err,
)
})?;
let pi = match store
.put_object_part(
@@ -191,7 +337,14 @@ pub(crate) async fn migrate_object(
Ok(pi) => pi,
Err(err) => {
error!("{op_label}: put_object_part {i} err {:?}", &err);
return Err(err);
return Err(data_movement_part_stage_error(
op_label,
"put_object_part",
bucket.as_str(),
object_info.name.as_str(),
part.number,
err,
));
}
};
@@ -209,12 +362,32 @@ pub(crate) async fn migrate_object(
&object_info.name,
&res.upload_id,
parts,
&data_movement_complete_multipart_opts(&object_info),
&data_movement_complete_multipart_opts(&object_info, pool_idx),
)
.await
{
if should_treat_data_movement_overwrite_as_complete(
store.as_ref(),
pool_idx,
target_pool_idx,
bucket.as_str(),
&object_info,
&err,
)
.await?
{
mark_multipart_upload_completed(&abort_multipart_flag);
return Ok(());
}
error!("{op_label}: complete_multipart_upload err {:?}", &err);
return Err(err);
return Err(data_movement_stage_error(
op_label,
"complete_multipart_upload",
bucket.as_str(),
object_info.name.as_str(),
err,
));
}
mark_multipart_upload_completed(&abort_multipart_flag);
@@ -248,13 +421,18 @@ pub(crate) async fn migrate_object(
return Ok(());
}
let actual_size = object_info.get_actual_size()?;
let actual_size = object_info.get_actual_size().map_err(|err| {
data_movement_stage_error(op_label, "prepare_put_object", bucket.as_str(), object_info.name.as_str(), err)
})?;
let index = object_info
.parts
.first()
.and_then(|part| decode_part_index(part.index.as_ref()));
let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index);
let hrd = HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
let hrd =
HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false).map_err(|err| {
data_movement_stage_error(op_label, "prepare_put_object", bucket.as_str(), object_info.name.as_str(), err)
})?;
let mut data = PutObjReader::new(hrd);
if let Err(err) = store
@@ -267,7 +445,13 @@ pub(crate) async fn migrate_object(
.await
{
error!("{op_label}: put_object err {:?}", &err);
return Err(err);
return Err(data_movement_stage_error(
op_label,
"put_object",
bucket.as_str(),
object_info.name.as_str(),
err,
));
}
Ok(())
@@ -309,6 +493,33 @@ mod tests {
assert!(message.contains(Error::SlowDown.to_string().as_str()));
}
#[test]
fn test_data_movement_stage_error_includes_stage_and_object() {
let err = data_movement_stage_error("rebalance_object", "put_object", "bucket-a", "object-a", Error::SlowDown);
let message = err.to_string();
assert!(message.contains("rebalance_object: put_object failed for bucket-a/object-a"));
assert!(message.contains(Error::SlowDown.to_string().as_str()));
}
#[test]
fn test_data_movement_part_stage_error_includes_stage_object_and_part() {
let err =
data_movement_part_stage_error("rebalance_object", "put_object_part", "bucket-a", "object-a", 7, Error::SlowDown);
let message = err.to_string();
assert!(message.contains("rebalance_object: put_object_part failed for bucket-a/object-a part 7"));
assert!(message.contains(Error::SlowDown.to_string().as_str()));
}
#[test]
fn test_should_check_data_movement_overwrite_resume_only_for_overwrite_error() {
assert!(should_check_data_movement_overwrite_resume(&Error::DataMovementOverwriteErr(
"bucket-a".to_string(),
"object-a".to_string(),
"version-a".to_string(),
)));
assert!(!should_check_data_movement_overwrite_resume(&Error::SlowDown));
}
#[test]
fn test_decode_part_index_returns_none_when_absent() {
assert!(decode_part_index(None).is_none());
@@ -366,13 +577,14 @@ mod tests {
..Default::default()
};
let opts = data_movement_complete_multipart_opts(&object_info);
let opts = data_movement_complete_multipart_opts(&object_info, 7);
assert!(opts.versioned);
assert!(opts.data_movement);
assert_eq!(opts.mod_time, Some(mod_time));
assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str()));
assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value"));
assert_eq!(opts.src_pool_idx, 7);
}
#[test]
@@ -396,4 +608,130 @@ mod tests {
assert!(opts.data_movement);
assert_eq!(opts.mod_time, object_info.mod_time);
}
#[test]
fn test_is_equivalent_data_movement_object_accepts_matching_metadata() {
let version_id = Uuid::nil();
let info = ObjectInfo {
version_id: Some(version_id),
size: 128,
actual_size: 96,
etag: Some("etag-value".to_string()),
checksum: Some(Bytes::from_static(b"checksum")),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
assert!(is_equivalent_data_movement_object(&info, &info.clone()));
}
#[test]
fn test_is_equivalent_data_movement_object_rejects_content_mismatch() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
size: 128,
actual_size: 96,
etag: Some("etag-source".to_string()),
checksum: Some(Bytes::from_static(b"checksum-source")),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let target = ObjectInfo {
etag: Some("etag-target".to_string()),
checksum: Some(Bytes::from_static(b"checksum-target")),
..source.clone()
};
assert!(!is_equivalent_data_movement_object(&source, &target));
}
#[test]
fn test_is_equivalent_data_movement_object_uses_effective_actual_size() {
let source = ObjectInfo {
size: 128,
actual_size: 0,
etag: Some("etag-value".to_string()),
..Default::default()
};
let target = ObjectInfo {
size: 128,
actual_size: 128,
etag: Some("etag-value".to_string()),
..Default::default()
};
assert!(is_equivalent_data_movement_object(&source, &target));
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_accepts_equivalent_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
size: 128,
etag: Some("etag-value".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(source.clone())), &source, 0, 1)
.expect("equivalent overwrite target should be evaluated");
assert!(should_resume);
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_rejects_source_pool_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
size: 128,
etag: Some("etag-value".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(source.clone())), &source, 0, 0)
.expect("source-pool target should be rejected before target lookup");
assert!(!should_resume);
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_rejects_non_equivalent_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
size: 128,
etag: Some("etag-source".to_string()),
..Default::default()
};
let target = ObjectInfo {
etag: Some("etag-target".to_string()),
..source.clone()
};
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &source, 0, 1)
.expect("non-equivalent overwrite target should be evaluated");
assert!(!should_resume);
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_propagates_target_lookup_error() {
let source = ObjectInfo::default();
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
let result = resolve_data_movement_overwrite_resume_result(&err, Err(Error::SlowDown), &source, 0, 1);
assert!(matches!(result, Err(Error::SlowDown)));
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_ignores_non_overwrite_error() {
let source = ObjectInfo::default();
let result = resolve_data_movement_overwrite_resume_result(&Error::SlowDown, Err(Error::FileAccessDenied), &source, 0, 1)
.expect("non-overwrite errors should not query target equivalence");
assert!(!result);
}
}
crates/ecstore/src/disk/disk_store.rs
+53 -1
View File
@@ -1136,10 +1136,16 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn walk_dir<W: tokio::io::AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
let timeout_duration = if opts.skip_total_timeout {
Duration::ZERO
} else {
get_drive_walkdir_timeout()
};
self.track_disk_health_with_op_and_timeout_action(
"walk_dir",
|| async { self.disk.walk_dir(opts, wr).await },
get_drive_walkdir_timeout(),
timeout_duration,
// Listing/scanner backpressure should fail only the current walk, not poison drive health.
TimeoutHealthAction::IgnoreFailure,
)
@@ -1647,6 +1653,52 @@ mod tests {
.await;
}
#[tokio::test]
async fn walk_dir_skip_total_timeout_keeps_stream_pending() {
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint =
Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
let wrapper = LocalDiskWrapper::new(disk, false);
let bucket = "test-bucket";
let object = "test-object";
wrapper.make_volume(bucket).await.expect("bucket should be created");
let mut file_info = FileInfo::new(&format!("{bucket}/{object}"), 1, 0);
file_info.volume = bucket.to_string();
file_info.name = object.to_string();
file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
file_info.erasure.index = 1;
wrapper
.write_metadata("", bucket, object, file_info)
.await
.expect("object metadata should be written");
let mut writer = PendingWriter;
let result = tokio::time::timeout(
Duration::from_millis(20),
wrapper.walk_dir(
WalkDirOptions {
bucket: bucket.to_string(),
recursive: true,
skip_total_timeout: true,
..Default::default()
},
&mut writer,
),
)
.await;
assert!(result.is_err(), "skip_total_timeout should leave backpressured walk pending");
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
assert!(!wrapper.health.is_faulty());
})
.await;
}
#[tokio::test]
async fn walk_dir_timeout_does_not_break_followup_stat_volume() {
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async {
crates/ecstore/src/disk/mod.rs
+6
View File
@@ -698,6 +698,10 @@ pub struct WalkDirOptions {
// DiskID contains the disk ID of the disk.
// Leave empty to not check disk ID.
pub disk_id: String,
// Skip the wrapper-level total timeout for long streaming walks.
#[serde(default)]
pub skip_total_timeout: bool,
}
#[derive(Clone, Debug, Default)]
@@ -894,6 +898,7 @@ mod tests {
forward_to: Some("object/path".to_string()),
limit: 100,
disk_id: "disk-123".to_string(),
skip_total_timeout: false,
};
assert_eq!(opts.bucket, "test-bucket");
@@ -904,6 +909,7 @@ mod tests {
assert_eq!(opts.forward_to, Some("object/path".to_string()));
assert_eq!(opts.limit, 100);
assert_eq!(opts.disk_id, "disk-123");
assert!(!opts.skip_total_timeout);
}
/// Test DeleteOptions structure
crates/ecstore/src/rebalance.rs
+1469 -83
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/rpc/remote_disk.rs
+40 -1
View File
@@ -1282,6 +1282,11 @@ impl DiskAPI for RemoteDisk {
let bucket = opts.bucket.clone();
let base_dir = opts.base_dir.clone();
let disk_for_log = disk.clone();
let timeout_duration = if opts.skip_total_timeout {
Duration::ZERO
} else {
get_drive_walkdir_timeout()
};
self.execute_with_timeout_for_op_and_health_action(
"walk_dir",
@@ -1329,7 +1334,7 @@ impl DiskAPI for RemoteDisk {
Err(last_err.unwrap_or_else(|| DiskError::other("walk_dir retry exhausted without captured error")))
},
get_drive_walkdir_timeout(),
timeout_duration,
FailureHealthAction::IgnoreFailure,
)
.await
@@ -2444,6 +2449,7 @@ mod tests {
forward_to: None,
limit: 10,
disk_id: String::new(),
..Default::default()
};
let expected_body = serde_json::to_vec(&opts).unwrap();
let mut writer = Vec::new();
@@ -2463,6 +2469,37 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_disk_walk_dir_preserves_skip_total_timeout_option() {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let opts = WalkDirOptions {
bucket: "bucket".to_string(),
base_dir: "prefix".to_string(),
recursive: true,
skip_total_timeout: true,
..Default::default()
};
let mut writer = Vec::new();
remote_disk
.walk_dir(opts, &mut writer)
.await
.expect("walk_dir should be sent through configured data transport");
let calls = transport.calls();
assert_eq!(calls.len(), 1);
match &calls[0] {
RecordedTransportCall::WalkDir(request) => {
let sent_opts: WalkDirOptions =
serde_json::from_slice(&request.body).expect("walk_dir request body should deserialize");
assert!(sent_opts.skip_total_timeout);
assert_eq!(request.stall_timeout, Some(get_drive_walkdir_stall_timeout()));
}
other => panic!("expected walk-dir transport call, got {other:?}"),
}
}
#[tokio::test]
async fn test_remote_disk_walk_dir_retries_once_on_retryable_transport_error() {
let transport = RetryingWalkDirInternodeDataTransport::with_steps(vec![
@@ -2479,6 +2516,7 @@ mod tests {
forward_to: None,
limit: 10,
disk_id: String::new(),
..Default::default()
};
let mut writer = Vec::new();
@@ -2510,6 +2548,7 @@ mod tests {
forward_to: None,
limit: 10,
disk_id: String::new(),
..Default::default()
};
let mut writer = Vec::new();
crates/ecstore/src/store/multipart.rs
+19 -3
View File
@@ -115,10 +115,24 @@ impl ECStore {
object: &str,
opts: &ObjectOptions,
) -> Result<MultipartUploadResult> {
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
.await
.map(|(res, _)| res)
}
pub(crate) async fn handle_new_multipart_upload_with_pool_idx(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Result<(MultipartUploadResult, usize)> {
check_new_multipart_args(bucket, object)?;
if self.single_pool() {
return self.pools[0].new_multipart_upload(bucket, object, opts).await;
return self.pools[0]
.new_multipart_upload(bucket, object, opts)
.await
.map(|res| (res, 0));
}
for (idx, pool) in self.pools.iter().enumerate() {
@@ -130,7 +144,8 @@ impl ECStore {
.await?;
if !res.uploads.is_empty() {
return self.pools[idx].new_multipart_upload(bucket, object, opts).await;
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
return Ok((res, idx));
}
}
let idx = self.get_pool_idx(bucket, object, -1).await?;
@@ -142,7 +157,8 @@ impl ECStore {
));
}
self.pools[idx].new_multipart_upload(bucket, object, opts).await
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
Ok((res, idx))
}
#[instrument(skip(self))]
crates/ecstore/src/store/object.rs
+423 -7
View File
@@ -233,6 +233,89 @@ fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> Object
lookup_opts
}
fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
info.get_actual_size().ok()
}
fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &ObjectInfo) -> bool {
is_data_movement_delete_marker(source)
&& is_data_movement_delete_marker(target)
&& source.version_id == target.version_id
&& source.mod_time == target.mod_time
}
fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
info.delete_marker
}
fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
source.version_id == target.version_id
&& !target.delete_marker
&& source.size == target.size
&& source.get_etag() == target.etag
&& source.checksum == target.checksum
&& source.mod_time == target.mod_time
&& source.transition_status == target.transitioned_object.status
&& source.transitioned_objname == target.transitioned_object.name
&& source.transition_tier == target.transitioned_object.tier
&& source
.transition_version_id
.map(|version_id| version_id.to_string())
.unwrap_or_default()
== target.transitioned_object.version_id
&& effective_object_actual_size(target) == Some(source.size)
}
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
target_pool_idx != src_pool_idx
}
fn resolve_data_movement_resume_target_pool(
selected_target_pool_idx: usize,
resume_target_pool_idx: Option<usize>,
src_pool_idx: usize,
) -> usize {
if should_check_data_movement_resume_target(src_pool_idx, selected_target_pool_idx) {
selected_target_pool_idx
} else {
resume_target_pool_idx.unwrap_or(selected_target_pool_idx)
}
}
fn resolve_data_movement_delete_marker_resume_result(
target_result: Result<Option<ObjectInfo>>,
source: &ObjectInfo,
src_pool_idx: usize,
target_pool_idx: usize,
) -> Result<bool> {
if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
return Ok(false);
}
let Some(target) = target_result? else {
return Ok(false);
};
Ok(is_equivalent_data_movement_delete_marker(source, &target))
}
fn resolve_data_movement_tiered_resume_result(
target_result: Result<Option<ObjectInfo>>,
source: &rustfs_filemeta::FileInfo,
src_pool_idx: usize,
target_pool_idx: usize,
) -> Result<bool> {
if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
return Ok(false);
}
let Some(target) = target_result? else {
return Ok(false);
};
Ok(is_equivalent_data_movement_tiered_object(source, &target))
}
impl ECStore {
fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
match err {
@@ -377,6 +460,62 @@ impl ECStore {
}
}
async fn find_data_movement_target_info(
&self,
bucket: &str,
object: &str,
target_pool_idx: usize,
opts: &ObjectOptions,
) -> Result<Option<ObjectInfo>> {
let lookup_opts = version_aware_lookup_opts(opts, true);
let Some(pool) = self.pools.get(target_pool_idx) else {
return Err(Error::other(format!(
"data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
)));
};
match pool.get_object_info(bucket, object, &lookup_opts).await {
Ok(info) => Ok(Some(info)),
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
Err(err) => Err(err),
}
}
async fn has_equivalent_data_movement_delete_marker(
&self,
bucket: &str,
object: &str,
source: &ObjectInfo,
opts: &ObjectOptions,
target_pool_idx: usize,
) -> Result<bool> {
resolve_data_movement_delete_marker_resume_result(
self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
.await,
source,
opts.src_pool_idx,
target_pool_idx,
)
}
async fn has_equivalent_data_movement_tiered_object(
&self,
bucket: &str,
object: &str,
source: &rustfs_filemeta::FileInfo,
opts: &ObjectOptions,
target_pool_idx: usize,
) -> Result<bool> {
resolve_data_movement_tiered_resume_result(
self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
.await,
source,
opts.src_pool_idx,
target_pool_idx,
)
}
fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
}
@@ -419,6 +558,17 @@ impl ECStore {
)?
};
if opts.data_movement && idx == opts.src_pool_idx {
let resume_target_pool_idx = self
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
.await;
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
if self
.has_equivalent_data_movement_tiered_object(bucket, &object, fi, opts, target_pool_idx)
.await?
{
return Ok(());
}
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
@@ -666,17 +816,44 @@ impl ECStore {
let gopts = version_aware_lookup_opts(&opts, true);
if opts.data_movement {
let existing_pool_idx = self
.get_pool_info_existing_with_opts(bucket, object, &gopts)
.await
.map(|(pinfo, _)| pinfo.index);
let target_pool_idx =
let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
let existing_pool_idx = existing_pool_info
.as_ref()
.map(|(pinfo, _)| pinfo.index)
.map_err(Clone::clone);
let selected_target_pool_idx =
match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
Some(pool_idx) => pool_idx,
None => self.get_pool_idx_no_lock(bucket, object, 0).await?,
};
let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
.await
} else {
None
};
let target_pool_idx =
resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
if opts.src_pool_idx == selected_target_pool_idx {
if let Ok((source_pool_info, _)) = existing_pool_info
&& opts.delete_marker
&& is_data_movement_delete_marker(&source_pool_info.object_info)
&& self
.has_equivalent_data_movement_delete_marker(
bucket,
object,
&source_pool_info.object_info,
&opts,
target_pool_idx,
)
.await?
{
let mut obj = source_pool_info.object_info;
obj.name = decode_dir_object(object);
return Ok(obj);
}
if opts.src_pool_idx == target_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
@@ -684,7 +861,9 @@ impl ECStore {
));
}
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
let mut obj = self.pools[selected_target_pool_idx]
.delete_object(bucket, object, opts)
.await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
@@ -1086,6 +1265,9 @@ impl ECStore {
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::lifecycle::bucket_lifecycle_ops::TransitionedObject;
use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
use bytes::Bytes;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
@@ -1108,6 +1290,240 @@ mod tests {
assert_eq!(target, Some(0));
}
#[test]
fn equivalent_data_movement_delete_marker_requires_same_version_and_mod_time() {
let version_id = Uuid::nil();
let mod_time = OffsetDateTime::UNIX_EPOCH;
let source = ObjectInfo {
version_id: Some(version_id),
delete_marker: true,
mod_time: Some(mod_time),
..Default::default()
};
let target = source.clone();
assert!(is_equivalent_data_movement_delete_marker(&source, &target));
let mismatched = ObjectInfo {
mod_time: Some(mod_time + Duration::from_secs(1)),
..target
};
assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
}
#[test]
fn equivalent_data_movement_delete_marker_rejects_live_object() {
let source = ObjectInfo {
delete_marker: true,
..Default::default()
};
let target = ObjectInfo {
delete_marker: false,
..source.clone()
};
assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
}
#[test]
fn data_movement_delete_marker_resume_accepts_equivalent_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
delete_marker: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 1)
.expect("equivalent delete marker target should be evaluated");
assert!(should_resume);
}
#[test]
fn data_movement_delete_marker_resume_rejects_source_pool_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
delete_marker: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 0)
.expect("source-pool target should be rejected before target lookup");
assert!(!should_resume);
}
#[test]
fn data_movement_resume_target_prefers_selected_non_source_pool() {
let target_pool_idx = resolve_data_movement_resume_target_pool(2, Some(3), 1);
assert_eq!(target_pool_idx, 2);
}
#[test]
fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
assert_eq!(target_pool_idx, 3);
}
#[test]
fn data_movement_resume_target_keeps_source_when_no_other_pool_is_available() {
let target_pool_idx = resolve_data_movement_resume_target_pool(1, None, 1);
assert_eq!(target_pool_idx, 1);
}
#[test]
fn data_movement_delete_marker_resume_propagates_target_lookup_error() {
let source = ObjectInfo {
delete_marker: true,
..Default::default()
};
let result = resolve_data_movement_delete_marker_resume_result(Err(Error::SlowDown), &source, 0, 1);
assert!(matches!(result, Err(Error::SlowDown)));
}
#[test]
fn equivalent_data_movement_tiered_object_accepts_matching_transition_metadata() {
let version_id = Uuid::nil();
let transition_version_id = Uuid::new_v4();
let mod_time = OffsetDateTime::UNIX_EPOCH;
let source = FileInfo {
version_id: Some(version_id),
size: 1024,
mod_time: Some(mod_time),
checksum: Some(Bytes::from_static(b"checksum")),
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_tier: "WARM".to_string(),
transition_version_id: Some(transition_version_id),
metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
..Default::default()
};
let target = ObjectInfo {
version_id: Some(version_id),
size: 1024,
mod_time: Some(mod_time),
checksum: Some(Bytes::from_static(b"checksum")),
etag: Some("etag-value".to_string()),
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
version_id: transition_version_id.to_string(),
tier: "WARM".to_string(),
status: TRANSITION_COMPLETE.to_string(),
..Default::default()
},
..Default::default()
};
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
}
#[test]
fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
let source = FileInfo {
version_id: Some(Uuid::nil()),
size: 1024,
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/source".to_string(),
transition_tier: "WARM".to_string(),
..Default::default()
};
let target = ObjectInfo {
version_id: source.version_id,
size: 1024,
transitioned_object: TransitionedObject {
name: "remote/target".to_string(),
tier: "WARM".to_string(),
status: TRANSITION_COMPLETE.to_string(),
..Default::default()
},
..Default::default()
};
assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
}
#[test]
fn data_movement_tiered_resume_accepts_equivalent_target() {
let version_id = Uuid::nil();
let transition_version_id = Uuid::new_v4();
let source = FileInfo {
version_id: Some(version_id),
size: 1024,
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_tier: "WARM".to_string(),
transition_version_id: Some(transition_version_id),
metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
..Default::default()
};
let target = ObjectInfo {
version_id: Some(version_id),
size: 1024,
etag: Some("etag-value".to_string()),
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
version_id: transition_version_id.to_string(),
tier: "WARM".to_string(),
status: TRANSITION_COMPLETE.to_string(),
..Default::default()
},
..Default::default()
};
let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
.expect("equivalent tiered target should be evaluated");
assert!(should_resume);
}
#[test]
fn data_movement_tiered_resume_rejects_source_pool_target() {
let version_id = Uuid::nil();
let source = FileInfo {
version_id: Some(version_id),
size: 1024,
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_tier: "WARM".to_string(),
metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
..Default::default()
};
let target = ObjectInfo {
version_id: Some(version_id),
size: 1024,
etag: Some("etag-value".to_string()),
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
tier: "WARM".to_string(),
status: TRANSITION_COMPLETE.to_string(),
..Default::default()
},
..Default::default()
};
let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
.expect("source-pool target should be rejected before target lookup");
assert!(!should_resume);
}
#[test]
fn data_movement_tiered_resume_rejects_missing_target() {
let source = FileInfo {
version_id: Some(Uuid::nil()),
size: 1024,
..Default::default()
};
let should_resume = resolve_data_movement_tiered_resume_result(Ok(None), &source, 0, 1)
.expect("missing tiered target should be evaluated");
assert!(!should_resume);
}
#[test]
fn latest_object_access_delete_marker_error_returns_none_for_live_object() {
let info = ObjectInfo::default();
crates/ecstore/src/store/rebalance.rs
+235 -20
View File
@@ -21,6 +21,11 @@ struct LatestObjectInfoCandidate {
err: Option<Error>,
}
struct RebalanceDeletePoolResult {
pool_idx: usize,
result: Result<ObjectInfo>,
}
fn pool_lookup_not_found_error(bucket: &str, object: &str, opts: &ObjectOptions) -> Error {
let object = decode_dir_object(object);
@@ -39,6 +44,52 @@ fn resolve_rebalance_delete_from_all_pools_result(result: Result<ObjectInfo>, bu
result.map_err(|err| Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}")))
}
fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
is_err_object_not_found(err) || is_err_version_not_found(err)
}
fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
}
fn resolve_rebalance_delete_from_all_pools_results(
results: Vec<RebalanceDeletePoolResult>,
bucket: &str,
object: &str,
) -> Result<ObjectInfo> {
let mut deleted = None;
let mut ignored_error = None;
for pool_result in results {
let pool_idx = pool_result.pool_idx;
match pool_result.result {
Ok(info) => {
if deleted.is_none() {
deleted = Some(info);
}
}
Err(err) if is_ignorable_rebalance_delete_error(&err) => {
ignored_error = Some((pool_idx, err));
}
Err(err) => {
return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
}
}
}
if let Some(info) = deleted {
return Ok(info);
}
if let Some((pool_idx, err)) = ignored_error {
return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
}
Err(Error::other(format!(
"failed to delete rebalance source object {bucket}/{object}: no pools were attempted"
)))
}
fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, pool_count: usize) -> Error {
Error::other(format!(
"failed to resolve rebalance disk set: pool index {pool_idx}, set index {set_idx}, pool count {pool_count}",
@@ -220,6 +271,41 @@ impl ECStore {
None
}
pub(super) async fn get_available_pool_idx_excluding(
&self,
bucket: &str,
object: &str,
size: i64,
excluded_pool_idx: usize,
) -> Option<usize> {
let mut server_pools = self.get_server_pools_available_space(bucket, object, size).await;
server_pools.filter_max_used(100 - (100_f64 * DISK_RESERVE_FRACTION) as u64);
if let Some(pool) = server_pools.0.get_mut(excluded_pool_idx) {
pool.available = 0;
}
let total = server_pools.total_available();
if total == 0 {
return None;
}
let mut rng = rand::rng();
let random_u64: u64 = rng.random_range(0..total);
let choose = random_u64 % total;
let mut at_total = 0;
for pool in server_pools.iter() {
at_total += pool.available;
if at_total > choose && pool.available > 0 {
return Some(pool.index);
}
}
None
}
async fn get_server_pools_available_space(&self, bucket: &str, object: &str, size: i64) -> ServerPoolsAvailableSpace {
let mut n_sets = vec![0; self.pools.len()];
let mut infos = vec![Vec::new(); self.pools.len()];
@@ -507,38 +593,34 @@ impl ECStore {
opts: &ObjectOptions,
errs: Vec<PoolErr>,
) -> Result<ObjectInfo> {
let mut objs = Vec::new();
let mut derrs = Vec::new();
let mut results = Vec::with_capacity(errs.len());
for pe in errs.iter() {
if let Some(err) = &pe.err
&& err == &StorageError::ErasureWriteQuorum
{
objs.push(None);
derrs.push(Some(StorageError::ErasureWriteQuorum));
if let Some(idx) = pe.index {
results.push(RebalanceDeletePoolResult {
pool_idx: idx,
result: Err(StorageError::ErasureWriteQuorum),
});
}
continue;
}
if let Some(idx) = pe.index {
match self.pools[idx].delete_object(bucket, object, opts.clone()).await {
Ok(res) => {
objs.push(Some(res));
derrs.push(None);
}
Err(err) => {
objs.push(None);
derrs.push(Some(err));
}
}
results.push(RebalanceDeletePoolResult {
pool_idx: idx,
result: self.pools[idx].delete_object(bucket, object, opts.clone()).await,
});
}
}
if let Some(e) = &derrs[0] {
return resolve_rebalance_delete_from_all_pools_result(Err(e.clone()), bucket, object);
}
resolve_rebalance_delete_from_all_pools_result(Ok(objs[0].as_ref().unwrap().clone()), bucket, object)
resolve_rebalance_delete_from_all_pools_result(
resolve_rebalance_delete_from_all_pools_results(results, bucket, object),
bucket,
object,
)
}
pub async fn reload_pool_meta(&self) -> Result<()> {
@@ -902,6 +984,139 @@ mod tests {
assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_fails_on_later_pool_error() {
let err = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Ok(ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
}),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Err(Error::SlowDown),
},
],
"bucket",
"object",
)
.expect_err("non-ignorable errors from later pools must not be hidden");
let rendered = err.to_string();
assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_ignores_later_not_found_after_success() {
let info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
};
let resolved = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Ok(info.clone()),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
],
"bucket",
"object",
)
.expect("not-found errors from other pools should be ignored when a delete succeeds");
assert_eq!(resolved.bucket, info.bucket);
assert_eq!(resolved.name, info.name);
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_accepts_success_after_not_found() {
let info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
};
let resolved = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Ok(info.clone()),
},
],
"bucket",
"object",
)
.expect("a successful delete should pass even when an earlier pool reports not-found");
assert_eq!(resolved.bucket, info.bucket);
assert_eq!(resolved.name, info.name);
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_fails_when_all_results_are_ignored_errors() {
let err = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Err(Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())),
},
],
"bucket",
"object",
)
.expect_err("all ignored errors without any successful delete should still fail");
let rendered = err.to_string();
assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
assert!(rendered.contains("Version not found"), "{rendered}");
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_fails_on_write_quorum_even_with_success() {
let err = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Ok(ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
}),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Err(Error::ErasureWriteQuorum),
},
],
"bucket",
"object",
)
.expect_err("write quorum failures must fail the aggregate delete");
let rendered = err.to_string();
assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
assert!(rendered.contains(&Error::ErasureWriteQuorum.to_string()), "{rendered}");
}
#[test]
fn rebalance_disk_set_lookup_error_formats_pool_and_set_context() {
let err = rebalance_disk_set_lookup_error(2, 7, 3);
crates/scanner/src/scanner_folder.rs
+23 -2
View File
@@ -147,6 +147,13 @@ fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool {
yield_every > 0 && object_count.is_multiple_of(yield_every)
}
const SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT: usize = 16;
const SCANNER_FAILED_OBJECT_LOG_EVERY: usize = 1024;
fn should_log_failed_object(failed_objects: usize) -> bool {
failed_objects <= SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT || failed_objects.is_multiple_of(SCANNER_FAILED_OBJECT_LOG_EVERY)
}
fn record_scanner_ilm_action_if_queued(metrics: &Metrics, count: u64, queued: bool) -> bool {
if queued {
metrics.record_scanner_ilm_action(count);
@@ -1286,8 +1293,12 @@ impl FolderScanner {
into.failed_objects += 1;
self.record_failed(&item.path);
// Only log non-skip errors to avoid noise
warn!("scan_folder: failed to get size for item {}: {}", item.path, e);
if should_log_failed_object(into.failed_objects) {
warn!(
failed_objects = into.failed_objects,
"scan_folder: failed to get size for item {}: {}", item.path, e
);
}
}
timer.sleep().await;
@@ -2975,4 +2986,14 @@ mod tests {
assert!(result.is_ok(), "expected symlinked child directory to be ignored");
assert_eq!(into.failed_objects, 0, "expected ignored symlink not to count as a failed object");
}
#[test]
fn test_should_log_failed_object_samples_after_initial_limit() {
assert!(should_log_failed_object(1));
assert!(should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT));
assert!(!should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT + 1));
assert!(should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_EVERY));
assert!(!should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_EVERY + 1));
assert!(should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_EVERY * 2));
}
}
crates/scanner/src/scanner_io.rs
+3 -5
View File
@@ -945,13 +945,11 @@ impl ScannerIODisk for Disk {
let data = match self.read_metadata(&item.bucket, &item.object_path()).await {
Ok(data) => data,
Err(e) => {
warn!(
"Failed to read metadata: {e}, bucket={}, object_path={}",
return Err(StorageError::other(format!(
"failed to read metadata: {e}, bucket={}, object_path={}",
&item.bucket,
&item.object_path()
);
return Err(StorageError::other("failed to read metadata".to_string()));
)));
}
};