From a236b0d01d40a152309446a553756ea991c9f901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Thu, 26 Mar 2026 11:44:02 +0800 Subject: [PATCH] feat(ecstore): implement decommission and rebalance (#2281) Co-authored-by: weisd Co-authored-by: houseme --- AGENTS.md | 8 + .../bucket/lifecycle/bucket_lifecycle_ops.rs | 47 +- crates/ecstore/src/data_movement.rs | 401 ++ crates/ecstore/src/error.rs | 46 + crates/ecstore/src/lib.rs | 1 + crates/ecstore/src/notification_sys.rs | 136 +- crates/ecstore/src/pools.rs | 2386 +++++++++-- crates/ecstore/src/rebalance.rs | 3734 ++++++++++++++--- crates/ecstore/src/set_disk.rs | 49 +- crates/ecstore/src/store/init.rs | 234 +- crates/ecstore/src/store/object.rs | 367 +- crates/ecstore/src/store/rebalance.rs | 493 ++- crates/ecstore/src/tier/tier.rs | 17 +- crates/filemeta/src/metacache.rs | 76 +- crates/rio/src/compress_index.rs | 92 +- crates/rio/src/http_reader.rs | 72 +- rustfs/src/admin/handlers/pools.rs | 310 +- rustfs/src/admin/handlers/rebalance.rs | 619 ++- rustfs/src/error.rs | 6 + rustfs/src/storage/rpc/node_service.rs | 22 +- 20 files changed, 7812 insertions(+), 1304 deletions(-) create mode 100644 crates/ecstore/src/data_movement.rs diff --git a/AGENTS.md b/AGENTS.md index 3dc66c756..e9abdd3db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,14 @@ If repo-level instructions conflict, follow the nearest file and keep behavior a - Respond in the same language used by the requester. - Keep source code, comments, commit messages, and PR title/body in English. +## Change Style for Existing Logic + +- Prefer direct, local code over extracting one-off helpers. +- Extract a helper only when logic is reused or the extraction materially clarifies a non-trivial flow. +- Preserve the existing control-flow and logic shape when fixing bugs or addressing review comments, especially in init, distributed coordination, locking, metadata, and concurrency paths. +- Do not refactor existing code only to make it easier to unit test. +- Keep fixes narrowly aligned with the requested behavior; avoid semantic-adjacent rewrites while touching sensitive paths. + ## Sources of Truth - Workspace layout and crate membership: `Cargo.toml` (`[workspace].members`) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 0bed2372a..18745dbbc 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -716,6 +716,12 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu Ok(()) } +fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) { + if remote_delete_succeeded { + opts.skip_decommissioned = true; + } +} + pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) { if let Some(lc) = GLOBAL_LifecycleSys.get(&oi.bucket).await { enqueue_transition_with_lifecycle(oi, &lc, &src).await; @@ -795,11 +801,10 @@ pub async fn expire_transitioned_object( &oi.transitioned_object.tier, ) .await; - if ret.is_ok() { - opts.skip_decommissioned = true; - } else { + if ret.is_err() { //transitionLogIf(ctx, err); } + mark_delete_opts_skip_decommissioned_on_remote_success(&mut opts, ret.is_ok()); let dobj = match api.delete_object(&oi.bucket, &oi.name, opts).await { Ok(obj) => obj, @@ -1278,3 +1283,39 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, } success } + +#[cfg(test)] +mod tests { + use super::mark_delete_opts_skip_decommissioned_on_remote_success; + use crate::store_api::ObjectOptions; + + #[test] + fn mark_delete_opts_skip_decommissioned_on_remote_success_sets_flag_on_success() { + let mut opts = ObjectOptions::default(); + + mark_delete_opts_skip_decommissioned_on_remote_success(&mut opts, true); + + assert!(opts.skip_decommissioned); + } + + #[test] + fn mark_delete_opts_skip_decommissioned_on_remote_success_preserves_false_on_failure() { + let mut opts = ObjectOptions::default(); + + mark_delete_opts_skip_decommissioned_on_remote_success(&mut opts, false); + + assert!(!opts.skip_decommissioned); + } + + #[test] + fn mark_delete_opts_skip_decommissioned_on_remote_success_preserves_existing_true_on_failure() { + let mut opts = ObjectOptions { + skip_decommissioned: true, + ..ObjectOptions::default() + }; + + mark_delete_opts_skip_decommissioned_on_remote_success(&mut opts, false); + + assert!(opts.skip_decommissioned); + } +} diff --git a/crates/ecstore/src/data_movement.rs b/crates/ecstore/src/data_movement.rs new file mode 100644 index 000000000..5d7721911 --- /dev/null +++ b/crates/ecstore/src/data_movement.rs @@ -0,0 +1,401 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::error::{Error, Result}; +use crate::store::ECStore; +use crate::store_api::{CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader}; +use bytes::Bytes; +use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, Reader, TryGetIndex, WarpReader}; +use std::io::Cursor; +use std::pin::Pin; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf}; +use tracing::error; + +pub struct IndexedDataMovementReader { + inner: R, + index: Option, +} + +impl IndexedDataMovementReader { + pub fn new(inner: R, index: Option) -> Self { + Self { inner, index } + } +} + +impl AsyncRead for IndexedDataMovementReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl EtagResolvable for IndexedDataMovementReader {} + +impl HashReaderDetector for IndexedDataMovementReader {} + +impl TryGetIndex for IndexedDataMovementReader { + fn try_get_index(&self) -> Option<&Index> { + self.index.as_ref() + } +} + +impl Reader for IndexedDataMovementReader {} + +pub fn decode_part_index(index: Option<&Bytes>) -> Option { + let bytes = index?; + let mut decoded = Index::new(); + if decoded.load(bytes.as_ref()).is_ok() { + Some(decoded) + } else { + None + } +} + +pub fn put_obj_reader_from_chunk(chunk: Vec, size: i64, actual_size: i64, index: Option) -> Result { + use sha2::{Digest, Sha256}; + + let sha256hex = if !chunk.is_empty() { + Some(hex_simd::encode_to_string(Sha256::digest(&chunk), hex_simd::AsciiCase::Lower)) + } else { + None + }; + + let reader = IndexedDataMovementReader::new(WarpReader::new(Cursor::new(chunk)), index); + let hash_reader = HashReader::new(Box::new(reader), size, actual_size, None, sha256hex, false)?; + Ok(PutObjReader::new(hash_reader)) +} + +pub fn new_multipart_abort_flag() -> Arc { + Arc::new(AtomicBool::new(true)) +} + +pub fn should_abort_multipart_upload(flag: &Arc) -> bool { + flag.load(Ordering::Relaxed) +} + +pub fn mark_multipart_upload_completed(flag: &Arc) { + flag.store(false, Ordering::Relaxed); +} + +fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions { + ObjectOptions { + versioned: object_info.version_id.is_some(), + version_id: object_info.version_id.as_ref().map(|v| v.to_string()), + user_defined: object_info.user_defined.clone(), + preserve_etag: object_info.etag.clone(), + src_pool_idx, + data_movement: true, + ..Default::default() + } +} + +fn data_movement_complete_multipart_opts(object_info: &ObjectInfo) -> ObjectOptions { + ObjectOptions { + versioned: object_info.version_id.is_some(), + version_id: object_info.version_id.as_ref().map(|v| v.to_string()), + data_movement: true, + mod_time: object_info.mod_time, + preserve_etag: object_info.etag.clone(), + ..Default::default() + } +} + +fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions { + ObjectOptions { + versioned: object_info.version_id.is_some(), + src_pool_idx, + data_movement: true, + version_id: object_info.version_id.as_ref().map(|v| v.to_string()), + mod_time: object_info.mod_time, + user_defined: object_info.user_defined.clone(), + preserve_etag: object_info.etag.clone(), + ..Default::default() + } +} + +fn resolve_data_movement_abort_result( + op_label: &str, + bucket: &str, + object: &str, + upload_id: &str, + primary_err: Error, + abort_err: Error, +) -> Error { + Error::other(format!( + "{op_label}: abort_multipart_upload failed for {bucket}/{object} upload {upload_id} after error {primary_err}: {abort_err}" + )) +} + +pub(crate) async fn migrate_object( + store: Arc, + pool_idx: usize, + bucket: String, + rd: GetObjectReader, + op_label: &str, +) -> Result<()> { + let object_info = rd.object_info.clone(); + + if object_info.is_multipart() { + let res = match store + .new_multipart_upload(&bucket, &object_info.name, &data_movement_new_multipart_opts(&object_info, pool_idx)) + .await + { + Ok(res) => res, + Err(err) => { + error!("{op_label}: new_multipart_upload err {:?}", &err); + return Err(err); + } + }; + + let abort_multipart_flag = new_multipart_abort_flag(); + let multipart_result: Result<()> = async { + let mut parts = vec![CompletePart::default(); object_info.parts.len()]; + let mut reader = rd.stream; + + for (i, part) in object_info.parts.iter().enumerate() { + let mut chunk = vec![0u8; part.size]; + reader.read_exact(&mut chunk).await?; + + let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?; + let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size }; + let index = decode_part_index(part.index.as_ref()); + let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?; + + let pi = match store + .put_object_part( + &bucket, + &object_info.name, + &res.upload_id, + part.number, + &mut data, + &ObjectOptions { + preserve_etag: Some(part.etag.clone()), + ..Default::default() + }, + ) + .await + { + Ok(pi) => pi, + Err(err) => { + error!("{op_label}: put_object_part {i} err {:?}", &err); + return Err(err); + } + }; + + parts[i] = CompletePart { + part_num: pi.part_num, + etag: pi.etag, + ..Default::default() + }; + } + + if let Err(err) = store + .clone() + .complete_multipart_upload( + &bucket, + &object_info.name, + &res.upload_id, + parts, + &data_movement_complete_multipart_opts(&object_info), + ) + .await + { + error!("{op_label}: complete_multipart_upload err {:?}", &err); + return Err(err); + } + + mark_multipart_upload_completed(&abort_multipart_flag); + Ok(()) + } + .await; + + if let Err(primary_err) = multipart_result { + if should_abort_multipart_upload(&abort_multipart_flag) { + return match store + .abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default()) + .await + { + Ok(()) => Err(primary_err), + Err(abort_err) => { + error!("{op_label}: abort_multipart_upload err {:?}", &abort_err); + Err(resolve_data_movement_abort_result( + op_label, + bucket.as_str(), + object_info.name.as_str(), + res.upload_id.as_str(), + primary_err, + abort_err, + )) + } + }; + } + return Err(primary_err); + } + + return Ok(()); + } + + let actual_size = object_info.get_actual_size()?; + let index = object_info + .parts + .first() + .and_then(|part| decode_part_index(part.index.as_ref())); + let reader = IndexedDataMovementReader::new(WarpReader::new(BufReader::new(rd.stream)), index); + let hrd = HashReader::new(Box::new(reader), object_info.size, actual_size, object_info.etag.clone(), None, false)?; + let mut data = PutObjReader::new(hrd); + + if let Err(err) = store + .put_object( + &bucket, + &object_info.name, + &mut data, + &data_movement_put_object_opts(&object_info, pool_idx), + ) + .await + { + error!("{op_label}: put_object err {:?}", &err); + return Err(err); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use time::OffsetDateTime; + use uuid::Uuid; + + #[test] + fn test_new_multipart_abort_flag_defaults_to_abort_enabled() { + let flag = new_multipart_abort_flag(); + assert!(should_abort_multipart_upload(&flag)); + } + + #[test] + fn test_mark_multipart_upload_completed_disables_abort_cleanup() { + let flag = new_multipart_abort_flag(); + mark_multipart_upload_completed(&flag); + assert!(!should_abort_multipart_upload(&flag)); + } + + #[test] + fn test_resolve_data_movement_abort_result_wraps_abort_context() { + let err = resolve_data_movement_abort_result( + "rebalance_object", + "bucket-a", + "object-a", + "upload-1", + Error::SlowDown, + Error::OperationCanceled, + ); + let message = err.to_string(); + assert!(message.contains("rebalance_object: abort_multipart_upload failed")); + assert!(message.contains("bucket-a/object-a")); + assert!(message.contains("upload upload-1")); + assert!(message.contains(Error::SlowDown.to_string().as_str())); + } + + #[test] + fn test_decode_part_index_returns_none_when_absent() { + assert!(decode_part_index(None).is_none()); + } + + #[test] + fn test_decode_part_index_returns_none_for_invalid_payload() { + let invalid = Bytes::from_static(b"not-a-valid-index"); + assert!(decode_part_index(Some(&invalid)).is_none()); + } + + #[test] + fn test_decode_part_index_returns_some_for_valid_payload() { + let mut index = Index::new(); + index.add(0, 0).expect("first index entry should be accepted"); + index + .add(2_097_152, 2_097_152) + .expect("second index entry should advance totals"); + + let encoded = index.into_vec(); + let decoded = decode_part_index(Some(&encoded)).expect("valid index payload should decode"); + + assert_eq!(decoded.total_uncompressed, 2_097_152); + assert_eq!(decoded.total_compressed, 2_097_152); + } + + #[test] + fn test_data_movement_new_multipart_opts_preserves_etag_and_version() { + let version_id = Uuid::nil(); + let object_info = ObjectInfo { + version_id: Some(version_id), + etag: Some("etag-value".to_string()), + user_defined: std::collections::HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]), + ..Default::default() + }; + + let opts = data_movement_new_multipart_opts(&object_info, 7); + + assert!(opts.versioned); + assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str())); + assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value")); + assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value")); + assert_eq!(opts.src_pool_idx, 7); + assert!(opts.data_movement); + } + + #[test] + fn test_data_movement_complete_multipart_opts_preserves_mod_time_version_and_etag() { + let mod_time = OffsetDateTime::now_utc(); + let version_id = Uuid::nil(); + let object_info = ObjectInfo { + version_id: Some(version_id), + mod_time: Some(mod_time), + etag: Some("etag-value".to_string()), + ..Default::default() + }; + + let opts = data_movement_complete_multipart_opts(&object_info); + + assert!(opts.versioned); + assert!(opts.data_movement); + assert_eq!(opts.mod_time, Some(mod_time)); + assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str())); + assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value")); + } + + #[test] + fn test_data_movement_put_object_opts_preserves_version_and_etag() { + let version_id = Uuid::nil(); + let object_info = ObjectInfo { + version_id: Some(version_id), + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + etag: Some("etag-value".to_string()), + user_defined: std::collections::HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]), + ..Default::default() + }; + + let opts = data_movement_put_object_opts(&object_info, 9); + + assert!(opts.versioned); + assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str())); + assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value")); + assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value")); + assert_eq!(opts.src_pool_idx, 9); + assert!(opts.data_movement); + assert_eq!(opts.mod_time, object_info.mod_time); + } +} diff --git a/crates/ecstore/src/error.rs b/crates/ecstore/src/error.rs index 7c7ed1f90..805bcf4f6 100644 --- a/crates/ecstore/src/error.rs +++ b/crates/ecstore/src/error.rs @@ -144,6 +144,10 @@ pub enum StorageError { DecommissionNotStarted, #[error("Decommission already running")] DecommissionAlreadyRunning, + #[error("Rebalance already running")] + RebalanceAlreadyRunning, + #[error("Operation canceled")] + OperationCanceled, #[error("No heal required")] NoHealRequired, #[error("DoneForNow")] @@ -414,6 +418,8 @@ impl Clone for StorageError { StorageError::EntityTooSmall(a, b, c) => StorageError::EntityTooSmall(*a, *b, *c), StorageError::DoneForNow => StorageError::DoneForNow, StorageError::DecommissionAlreadyRunning => StorageError::DecommissionAlreadyRunning, + StorageError::RebalanceAlreadyRunning => StorageError::RebalanceAlreadyRunning, + StorageError::OperationCanceled => StorageError::OperationCanceled, StorageError::ErasureReadQuorum => StorageError::ErasureReadQuorum, StorageError::ErasureWriteQuorum => StorageError::ErasureWriteQuorum, StorageError::NotFirstDisk => StorageError::NotFirstDisk, @@ -482,6 +488,8 @@ impl StorageError { StorageError::InvalidPart(_, _, _) => 0x2E, StorageError::DoneForNow => 0x2F, StorageError::DecommissionAlreadyRunning => 0x30, + StorageError::RebalanceAlreadyRunning => 0x40, + StorageError::OperationCanceled => 0x41, StorageError::ErasureReadQuorum => 0x31, StorageError::ErasureWriteQuorum => 0x32, StorageError::NotFirstDisk => 0x33, @@ -554,6 +562,8 @@ impl StorageError { 0x2E => Some(StorageError::InvalidPart(Default::default(), Default::default(), Default::default())), 0x2F => Some(StorageError::DoneForNow), 0x30 => Some(StorageError::DecommissionAlreadyRunning), + 0x40 => Some(StorageError::RebalanceAlreadyRunning), + 0x41 => Some(StorageError::OperationCanceled), 0x31 => Some(StorageError::ErasureReadQuorum), 0x32 => Some(StorageError::ErasureWriteQuorum), 0x33 => Some(StorageError::NotFirstDisk), @@ -682,6 +692,22 @@ pub fn is_err_data_movement_overwrite(err: &Error) -> bool { matches!(err, &StorageError::DataMovementOverwriteErr(_, _, _)) } +pub fn is_err_decommission_running(err: &Error) -> bool { + matches!(err, &StorageError::DecommissionAlreadyRunning) +} + +pub fn is_err_rebalance_running(err: &Error) -> bool { + matches!(err, &StorageError::RebalanceAlreadyRunning) +} + +pub fn is_err_operation_canceled(err: &Error) -> bool { + matches!(err, &StorageError::OperationCanceled) +} + +pub fn is_err_not_initialized(err: &Error) -> bool { + err.to_string().contains("errServerNotInitialized") || err.to_string().contains("ServerNotInitialized") +} + pub fn is_err_io(err: &Error) -> bool { matches!(err, &StorageError::Io(_)) } @@ -944,6 +970,8 @@ mod tests { assert_eq!(StorageError::VolumeExists.to_u32(), 0x05); assert_eq!(StorageError::FileNotFound.to_u32(), 0x06); assert_eq!(StorageError::DecommissionAlreadyRunning.to_u32(), 0x30); + assert_eq!(StorageError::RebalanceAlreadyRunning.to_u32(), 0x40); + assert_eq!(StorageError::OperationCanceled.to_u32(), 0x41); } #[test] @@ -956,6 +984,8 @@ mod tests { assert!(matches!(StorageError::from_u32(0x03), Some(StorageError::DiskFull))); assert!(matches!(StorageError::from_u32(0x04), Some(StorageError::VolumeNotFound))); assert!(matches!(StorageError::from_u32(0x30), Some(StorageError::DecommissionAlreadyRunning))); + assert!(matches!(StorageError::from_u32(0x40), Some(StorageError::RebalanceAlreadyRunning))); + assert!(matches!(StorageError::from_u32(0x41), Some(StorageError::OperationCanceled))); // Test invalid code returns None assert!(StorageError::from_u32(0xFF).is_none()); @@ -980,6 +1010,20 @@ mod tests { assert_ne!(bucket1, disk_error); } + #[test] + fn test_error_running_state_helpers() { + assert!(is_err_decommission_running(&StorageError::DecommissionAlreadyRunning)); + assert!(!is_err_decommission_running(&StorageError::RebalanceAlreadyRunning)); + + assert!(is_err_rebalance_running(&StorageError::RebalanceAlreadyRunning)); + assert!(!is_err_rebalance_running(&StorageError::DecommissionAlreadyRunning)); + assert!(is_err_operation_canceled(&StorageError::OperationCanceled)); + assert!(!is_err_operation_canceled(&StorageError::RebalanceAlreadyRunning)); + assert!(is_err_not_initialized(&StorageError::other("errServerNotInitialized"))); + assert!(is_err_not_initialized(&StorageError::other("ServerNotInitialized"))); + assert!(!is_err_not_initialized(&StorageError::DecommissionAlreadyRunning)); + } + #[test] fn test_storage_error_from_disk_error() { // Test conversion from DiskError @@ -1067,6 +1111,8 @@ mod tests { StorageError::BucketExists("test".to_string()), StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()), StorageError::DecommissionAlreadyRunning, + StorageError::RebalanceAlreadyRunning, + StorageError::OperationCanceled, ]; for original_error in test_errors { diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 5bab6c1e3..891318b32 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -22,6 +22,7 @@ pub mod bucket; pub mod cache_value; pub mod compress; pub mod config; +mod data_movement; pub mod data_usage; pub mod disk; pub mod disks_layout; diff --git a/crates/ecstore/src/notification_sys.rs b/crates/ecstore/src/notification_sys.rs index 6981a3e1b..1df246d23 100644 --- a/crates/ecstore/src/notification_sys.rs +++ b/crates/ecstore/src/notification_sys.rs @@ -17,6 +17,7 @@ use crate::admin_server_info::get_commit_id; use crate::error::{Error, Result}; use crate::global::{GLOBAL_BOOT_TIME, get_global_endpoints}; use crate::metrics_realtime::{CollectMetricsOpts, MetricType}; +use crate::rebalance::RebalSaveOpt; use crate::rpc::PeerRestClient; use crate::{endpoints::EndpointServerPools, new_object_layer_fn}; use futures::future::join_all; @@ -376,67 +377,112 @@ impl NotificationSys { join_all(futures).await } - pub async fn reload_pool_meta(&self) { + pub async fn reload_pool_meta(&self) -> Result<()> { + let mut failures = Vec::new(); let mut futures = Vec::with_capacity(self.peer_clients.len()); - for client in self.peer_clients.iter().flatten() { - futures.push(client.reload_pool_meta()); - } - - let results = join_all(futures).await; - for result in results { - if let Err(err) = result { - error!("notification reload_pool_meta err {:?}", err); + for (idx, client) in self.peer_clients.iter().enumerate() { + if let Some(client) = client { + let host = client.grid_host.clone(); + futures.push(async move { client.reload_pool_meta().await.map_err(|err| (host, err)) }); + } else { + failures.push(format!("peer[{idx}] reload_pool_meta failed: peer is not reachable")); } } + + for result in join_all(futures).await { + if let Err((host, err)) = result { + let failure = format!("peer {host} reload_pool_meta failed: {err}"); + error!("notification reload_pool_meta err {}", failure); + failures.push(failure); + } + } + + aggregate_notification_failures("reload_pool_meta", failures) } #[tracing::instrument(skip(self))] - pub async fn load_rebalance_meta(&self, start: bool) { + pub async fn load_rebalance_meta(&self, start: bool) -> Result<()> { + let operation = format!("load_rebalance_meta(start={start})"); + let mut failures = Vec::new(); let mut futures = Vec::with_capacity(self.peer_clients.len()); - for (i, client) in self.peer_clients.iter().flatten().enumerate() { - warn!( - "notification load_rebalance_meta start: {}, index: {}, client: {:?}", - start, i, client.host - ); - futures.push(client.load_rebalance_meta(start)); + for (idx, client) in self.peer_clients.iter().enumerate() { + if let Some(client) = client { + warn!( + "notification load_rebalance_meta start: {}, index: {}, client: {:?}", + start, idx, client.host + ); + let host = client.grid_host.clone(); + futures.push(async move { client.load_rebalance_meta(start).await.map_err(|err| (host, err)) }); + } else { + failures.push(format!("peer[{idx}] {operation} failed: peer is not reachable")); + } } - let results = join_all(futures).await; - for result in results { - if let Err(err) = result { - error!("notification load_rebalance_meta err {:?}", err); + for result in join_all(futures).await { + if let Err((host, err)) = result { + let failure = format!("peer {host} {operation} failed: {err}"); + error!("notification load_rebalance_meta err {}", failure); + failures.push(failure); } else { warn!("notification load_rebalance_meta success"); } } + + aggregate_notification_failures("load_rebalance_meta", failures) } - pub async fn stop_rebalance(&self) { + pub async fn stop_rebalance(&self) -> Result<()> { warn!("notification stop_rebalance start"); let Some(store) = new_object_layer_fn() else { error!("stop_rebalance: not init"); - return; + return Err(Error::other("stop_rebalance: object layer not initialized")); }; // warn!("notification stop_rebalance load_rebalance_meta"); // self.load_rebalance_meta(false).await; // warn!("notification stop_rebalance load_rebalance_meta done"); + let mut failures = Vec::new(); + let mut futures = Vec::with_capacity(self.peer_clients.len()); - for client in self.peer_clients.iter().flatten() { - futures.push(client.stop_rebalance()); + for (idx, client) in self.peer_clients.iter().enumerate() { + if let Some(client) = client { + let host = client.grid_host.clone(); + futures.push(async move { client.stop_rebalance().await.map_err(|err| (host, err)) }); + } else { + failures.push(format!("peer[{idx}] stop_rebalance failed: peer is not reachable")); + } } - let results = join_all(futures).await; - for result in results { - if let Err(err) = result { - error!("notification stop_rebalance err {:?}", err); + for result in join_all(futures).await { + if let Err((host, err)) = result { + let failure = format!("peer {host} stop_rebalance failed: {err}"); + error!("notification stop_rebalance err {}", failure); + failures.push(failure); } } warn!("notification stop_rebalance stop_rebalance start"); - let _ = store.stop_rebalance().await; + match store.stop_rebalance().await { + Ok(_) => { + if let Err(err) = store.save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt).await { + error!("notification stop_rebalance local save err {:?}", err); + return Err(Error::other(format!( + "local stop_rebalance save_rebalance_stats(stopped_at) failed: {err}" + ))); + } + } + Err(err) => { + error!("notification stop_rebalance local stop err {:?}", err); + return Err(Error::other(format!("local stop_rebalance stop failed: {err}"))); + } + } + + if let Err(err) = aggregate_notification_failures("stop_rebalance", failures) { + warn!("{err}"); + } warn!("notification stop_rebalance stop_rebalance done"); + Ok(()) } pub async fn load_bucket_metadata(&self, bucket: &str) -> Vec { @@ -773,6 +819,18 @@ fn get_offline_disks(offline_host: &str, endpoints: &EndpointServerPools) -> Vec offline_disks } +fn aggregate_notification_failures(operation: &str, failures: Vec) -> Result<()> { + if failures.is_empty() { + return Ok(()); + } + + Err(Error::other(format!( + "{operation} encountered {} failure(s): {}", + failures.len(), + failures.join(" | ") + ))) +} + #[cfg(test)] mod tests { use super::*; @@ -825,4 +883,24 @@ mod tests { assert_eq!(result.endpoint, "fallback"); } + + #[test] + fn aggregate_notification_failures_returns_ok_when_empty() { + assert!(aggregate_notification_failures("stop_rebalance", Vec::new()).is_ok()); + } + + #[test] + fn aggregate_notification_failures_returns_joined_error_when_non_empty() { + let err = aggregate_notification_failures( + "load_rebalance_meta", + vec!["peer-1 failed".to_string(), "local save failed".to_string()], + ) + .expect_err("non-empty failures should return error"); + + let msg = err.to_string(); + assert!(msg.contains("load_rebalance_meta")); + assert!(msg.contains("2 failure(s)")); + assert!(msg.contains("peer-1 failed")); + assert!(msg.contains("local save failed")); + } } diff --git a/crates/ecstore/src/pools.rs b/crates/ecstore/src/pools.rs index 3de9c4501..9c2d748c9 100644 --- a/crates/ecstore/src/pools.rs +++ b/crates/ecstore/src/pools.rs @@ -26,24 +26,24 @@ use crate::bucket::{ }; use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; use crate::config::com::{CONFIG_PREFIX, read_config, save_config}; +use crate::data_movement; use crate::data_usage::DATA_USAGE_CACHE_NAME; use crate::disk::error::DiskError; use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; use crate::error::{Error, Result}; use crate::error::{ StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_data_movement_overwrite, is_err_object_not_found, - is_err_version_not_found, + is_err_operation_canceled, is_err_version_not_found, }; use crate::new_object_layer_fn; use crate::notification_sys::get_global_notification_sys; use crate::set_disk::SetDisks; use crate::store_api::{ - BucketOperations, BucketOptions, CompletePart, GetObjectReader, HealOperations, MakeBucketOptions, MultipartOperations, - ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, StorageAPI, + BucketOperations, BucketOptions, GetObjectReader, HealOperations, MakeBucketOptions, ObjectIO, ObjectOperations, + ObjectOptions, StorageAPI, }; use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; -use bytes::Bytes; use futures::future::BoxFuture; use http::HeaderMap; #[cfg(test)] @@ -52,23 +52,21 @@ use rmp_serde::Serializer; use rustfs_common::defer; use rustfs_common::heal_channel::HealOpts; use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; -use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, Reader, TryGetIndex, WarpReader}; use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join}; use rustfs_workers::workers::Workers; use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt::Display; -use std::io::{Cursor, Write}; +#[cfg(test)] +use std::io::Cursor; +use std::io::Write; use std::path::PathBuf; -use std::pin::Pin; use std::sync::{ Arc, atomic::{AtomicUsize, Ordering}, }; -use std::task::{Context, Poll}; use time::{Duration, OffsetDateTime}; -use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -76,6 +74,377 @@ pub const POOL_META_NAME: &str = "pool.bin"; pub const POOL_META_FORMAT: u16 = 1; pub const POOL_META_VERSION: u16 = 1; +fn dedup_indices(indices: &[usize]) -> Vec { + let mut seen = HashSet::with_capacity(indices.len()); + let mut output = Vec::with_capacity(indices.len()); + for idx in indices { + if seen.insert(*idx) { + output.push(*idx); + } + } + + output +} + +fn bind_decommission_cancelers( + indices: &[usize], + parent: &CancellationToken, + cancelers: &mut [Option], +) -> Vec<(usize, CancellationToken)> { + let mut bound = Vec::with_capacity(indices.len()); + + for idx in indices { + if let Some(slot) = cancelers.get_mut(*idx) { + if let Some(existing) = slot.take() { + existing.cancel(); + } + let token = parent.child_token(); + *slot = Some(token.clone()); + bound.push((*idx, token)); + } + } + + bound +} + +fn take_decommission_canceler(cancelers: &mut [Option], idx: usize) -> Option { + cancelers.get_mut(idx).and_then(Option::take) +} + +fn has_active_decommission_canceler(cancelers: &[Option]) -> bool { + cancelers.iter().any(Option::is_some) +} + +fn cancel_decommission_canceler(canceler: Option) -> bool { + if let Some(canceler) = canceler { + canceler.cancel(); + true + } else { + false + } +} + +fn ensure_decommission_routines_scheduled(bound_count: usize, expected_count: usize) -> Result<()> { + if bound_count == 0 || bound_count != expected_count { + return Err(Error::other(format!( + "failed to start decommission routines: scheduled {bound_count} of {expected_count} expected workers" + ))); + } + + Ok(()) +} + +fn ensure_decommission_not_rebalancing(rebalance_running: bool) -> Result<()> { + if rebalance_running { + return Err(Error::RebalanceAlreadyRunning); + } + + Ok(()) +} + +fn is_decommission_active(complete: bool, failed: bool, canceled: bool) -> bool { + !complete && !failed && !canceled +} + +fn invalid_decommission_pool_index_error(pool_count: usize, idx: usize) -> Error { + Error::other(format!("invalid decommission pool index {idx} for {pool_count} pools")) +} + +fn ensure_decommission_start_allowed(pool_present: bool, decommission_active: bool) -> Result<()> { + if !pool_present { + return Err(Error::other("failed to start decommission: target pool was not found")); + } + + if decommission_active { + return Err(StorageError::DecommissionAlreadyRunning); + } + + Ok(()) +} + +fn ensure_valid_decommission_pool_index(pool_count: usize, idx: usize) -> Result<()> { + if idx >= pool_count { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + } + + Ok(()) +} + +fn get_by_index<'a, T>(items: &'a [T], idx: usize, operation: &'static str) -> Result<&'a T> { + items.get(idx).ok_or_else(|| { + Error::other(format!( + "failed to {operation}: invalid decommission pool index {idx} for {pool_count} pools", + pool_count = items.len() + )) + }) +} + +fn decommission_metadata_not_initialized_error(operation: &str) -> Error { + Error::other(format!("failed to {operation}: decommission metadata not initialized")) +} + +fn resolve_decommission_bucket_state(meta: &PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result { + let pool_count = meta.pools.len(); + ensure_valid_decommission_pool_index(pool_count, idx)?; + + let Some(pool) = meta.pools.get(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let Some(info) = pool.decommission.as_ref() else { + return Err(decommission_metadata_not_initialized_error("resolve decommission bucket state")); + }; + + Ok(info.is_bucket_decommissioned(&bucket.to_string())) +} + +fn mark_decommission_bucket_done(meta: &mut PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result { + let pool_count = meta.pools.len(); + ensure_valid_decommission_pool_index(pool_count, idx)?; + + let Some(pool) = meta.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let Some(info) = pool.decommission.as_mut() else { + return Err(decommission_metadata_not_initialized_error("mark decommission bucket done")); + }; + + Ok(info.bucket_pop(&bucket.to_string())) +} + +fn count_decommission_item(meta: &mut PoolMeta, idx: usize, size: usize, failed: bool) -> Result<()> { + let pool_count = meta.pools.len(); + ensure_valid_decommission_pool_index(pool_count, idx)?; + + let Some(pool) = meta.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let Some(info) = pool.decommission.as_mut() else { + return Err(decommission_metadata_not_initialized_error("count decommission item")); + }; + + if failed { + info.items_decommission_failed += 1; + info.bytes_failed += size; + } else { + info.items_decommissioned += 1; + info.bytes_done += size; + } + + Ok(()) +} + +fn track_decommission_current_object(meta: &mut PoolMeta, idx: usize, bucket: &str, object: &str) -> Result<()> { + let pool_count = meta.pools.len(); + ensure_valid_decommission_pool_index(pool_count, idx)?; + + let Some(pool) = meta.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let Some(info) = pool.decommission.as_mut() else { + return Err(decommission_metadata_not_initialized_error("track decommission current object")); + }; + + info.object = object.to_string(); + info.bucket = bucket.to_string(); + Ok(()) +} + +fn resolve_decommission_update_after_result(result: Result) -> Result { + result.map_err(|err| Error::other(format!("decommission metadata update failed: {err}"))) +} + +fn resolve_decommission_preflight_heal_result(bucket: &str, result: Result) -> Result { + result.map_err(|err| Error::other(format!("decommission preflight heal failed for bucket {bucket}: {err}"))) +} + +fn resolve_decommission_bucket_done_save_result(result: Result<()>, idx: usize, bucket: &str) -> Result<()> { + result.map_err(|err| Error::other(format!("decommission metadata save failed for pool {idx} bucket {bucket}: {err}"))) +} + +fn resolve_decommission_optional_bucket_config_result(bucket: &str, stage: &str, result: Result) -> Result> { + match result { + Ok(config) => Ok(Some(config)), + Err(Error::ConfigNotFound) => Ok(None), + Err(err) => Err(Error::other(format!( + "decommission {stage} config load failed for bucket {bucket}: {err}" + ))), + } +} + +fn resolve_decommission_entry_cleanup_delete_result(result: Result, bucket: &str, object_name: &str) -> Result<()> { + match result { + Ok(_) => Ok(()), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()), + Err(err) => Err(Error::other(format!( + "decommission cleanup_delete_object failed for {bucket}/{object_name}: {err}" + ))), + } +} + +fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> { + result.map_err(|err| Error::other(format!("decommission reload_pool_meta failed for {bucket}/{object_name}: {err}"))) +} + +fn resolve_decommission_terminal_mark_result(result: Result<()>, stage: &str, pool_label: &str) -> Result<()> { + result.map_err(|err| Error::other(format!("decommission terminal mark {stage} failed for pool {pool_label}: {err}"))) +} + +fn resolve_decommission_terminal_mark_after_error_result(result: Result<()>, idx: usize, primary_err: &Error) -> Result<()> { + result.map_err(|err| { + Error::other(format!( + "decommission terminal mark failed after background error on pool {idx}: {primary_err}; mark error: {err}" + )) + }) +} + +fn resolve_decommission_spawn_failure_result(spawn_err: Error, rollback_err: Option) -> Error { + if let Some(rollback_err) = rollback_err { + Error::other(format!( + "decommission spawn routines failed: {spawn_err}; rollback failed: {rollback_err}" + )) + } else { + spawn_err + } +} + +fn decommission_item_size(size: T) -> usize +where + usize: TryFrom, +{ + usize::try_from(size).unwrap_or_default() +} + +fn with_decommission_entry_context(stage: &str, bucket: &str, object: &str, err: E) -> Error { + Error::other(format!("decommission entry {stage} failed for bucket {bucket} object {object}: {err}")) +} + +fn load_decommission_entry_versions(entry: &MetaCacheEntry, bucket: &str, stage: &str) -> Result { + entry + .file_info_versions(bucket) + .map_err(|err| with_decommission_entry_context(stage, bucket, &entry.name, err)) +} + +fn resolve_decommission_check_after_list_result(list_result: Result<()>, entry_error: Option) -> Result<()> { + if let Some(err) = entry_error { Err(err) } else { list_result } +} + +fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> { + result.map_err(|err| Error::other(format!("decommission pool meta reload failed during {stage}: {err}"))) +} + +fn ensure_pool_not_left_in_cmdline_after_decommission(position: usize, cmd_line: &str, completed: bool) -> Result<()> { + if completed { + return Err(Error::other(format!( + "pool({}) = {} is decommissioned, please remove from server command line", + position + 1, + cmd_line + ))); + } + + Ok(()) +} + +fn resolve_decommission_listing_worker_result( + set_idx: usize, + worker_result: std::result::Result<(), tokio::task::JoinError>, +) -> Result<()> { + worker_result.map_err(|err| Error::other(format!("decommission listing worker {set_idx} task join error: {err}"))) +} + +fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: bool, failure: bool) -> bool { + cleanup_ignored || (!ignore && !failure) +} + +fn should_cleanup_decommission_source_entry(decommissioned: usize, total_versions: usize, expired: usize) -> bool { + expired == 0 && decommissioned == total_versions +} + +fn decommission_start_guard_state(pool: Option<&PoolStatus>) -> (bool, bool) { + if let Some(pool) = pool { + let active = pool + .decommission + .as_ref() + .is_some_and(|info| is_decommission_active(info.complete, info.failed, info.canceled)); + (true, active) + } else { + (false, false) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DecommissionTerminalState { + Completed, + Failed, +} + +fn classify_decommission_terminal_state(failed_items_present: bool) -> DecommissionTerminalState { + if failed_items_present { + DecommissionTerminalState::Failed + } else { + DecommissionTerminalState::Completed + } +} + +fn should_preserve_decommission_canceled_state(meta_canceled: bool, cancel_signal: bool) -> bool { + meta_canceled || cancel_signal +} + +fn decommission_cancel_signal_result(cancel_signal: bool) -> Result<()> { + if cancel_signal { + Err(StorageError::OperationCanceled) + } else { + Ok(()) + } +} + +fn is_decommission_cancel_terminal(complete: bool, failed: bool, canceled: bool) -> bool { + complete || failed || canceled +} + +fn ensure_decommission_cancel_allowed(pool_present: bool, decommission_present: bool, terminal: bool) -> Result<()> { + if !pool_present { + return Err(Error::other("failed to cancel decommission: target pool was not found")); + } + + if !decommission_present || terminal { + return Err(StorageError::DecommissionNotStarted); + } + + Ok(()) +} + +fn ensure_decommission_terminal_operation_supported(single_pool: bool, operation: &str) -> Result<()> { + if single_pool { + return Err(Error::other(format!( + "failed to {operation}: single pool deployments do not support decommission" + ))); + } + + Ok(()) +} + +fn validate_start_decommission_request(indices: &[usize], single_pool: bool) -> Result<()> { + if indices.is_empty() { + return Err(Error::other("failed to start decommission: no target pools were provided")); + } + + ensure_decommission_terminal_operation_supported(single_pool, "start decommission") +} + +fn require_decommission_store(store: Option, operation: &str) -> Result { + store.ok_or_else(|| Error::other(format!("failed to {operation}: store not initialized"))) +} + +fn ensure_decommission_listing_disks_available(has_disks: bool, bucket: &str) -> Result<()> { + if !has_disks { + return Err(Error::other(format!( + "failed to list objects to decommission for bucket {bucket}: no disks available" + ))); + } + + Ok(()) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PoolStatus { #[serde(rename = "id")] @@ -287,11 +656,7 @@ impl PoolMeta { } pub fn is_suspended(&self, idx: usize) -> bool { - if idx >= self.pools.len() { - return false; - } - - self.pools[idx].decommission.is_some() + self.pools.get(idx).is_some_and(|pool| pool.decommission.is_some()) } pub async fn load(&mut self, pool: Arc, _pools: Vec>) -> Result<()> { @@ -300,7 +665,7 @@ impl PoolMeta { if data.is_empty() { return Ok(()); } else if data.len() <= 4 { - return Err(Error::other("poolMeta: no data")); + return Err(Error::other("pool metadata load failed: metadata payload is too short")); } data } @@ -313,17 +678,20 @@ impl PoolMeta { }; let format = LittleEndian::read_u16(&data[0..2]); if format != POOL_META_FORMAT { - return Err(Error::other(format!("PoolMeta: unknown format: {format}"))); + return Err(Error::other(format!("pool metadata load failed: unknown format {format}"))); } let version = LittleEndian::read_u16(&data[2..4]); if version != POOL_META_VERSION { - return Err(Error::other(format!("PoolMeta: unknown version: {version}"))); + return Err(Error::other(format!("pool metadata load failed: unknown version {version}"))); } *self = Self::decode_pool_meta_payload(&data[4..])?; if self.version != POOL_META_VERSION { - return Err(Error::other(format!("unexpected PoolMeta version: {}", self.version))); + return Err(Error::other(format!( + "pool metadata load failed: unexpected decoded version {}", + self.version + ))); } Ok(()) } @@ -416,31 +784,36 @@ impl PoolMeta { } } pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> { - if let Some(pool) = self.pools.get_mut(idx) { - if let Some(ref info) = pool.decommission - && !info.complete - && !info.failed - && !info.canceled - { - return Err(StorageError::DecommissionAlreadyRunning); - } + let pool_count = self.pools.len(); + ensure_valid_decommission_pool_index(pool_count, idx)?; - let now = OffsetDateTime::now_utc(); - pool.last_update = now; - pool.decommission = Some(PoolDecommissionInfo { - start_time: Some(now), - start_size: pi.free, - total_size: pi.total, - current_size: pi.free, - ..Default::default() - }); - } + let Some(pool) = self.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + + let decommission_active = pool + .decommission + .as_ref() + .is_some_and(|info| is_decommission_active(info.complete, info.failed, info.canceled)); + ensure_decommission_start_allowed(true, decommission_active)?; + + let now = OffsetDateTime::now_utc(); + pool.last_update = now; + pool.decommission = Some(PoolDecommissionInfo { + start_time: Some(now), + start_size: pi.free, + total_size: pi.total, + current_size: pi.free, + ..Default::default() + }); Ok(()) } pub fn queue_buckets(&mut self, idx: usize, bks: Vec) { - for bk in bks.iter() { - if let Some(dec) = self.pools[idx].decommission.as_mut() { + if let Some(pool) = self.pools.get_mut(idx) + && let Some(dec) = pool.decommission.as_mut() + { + for bk in bks.iter() { dec.bucket_push(bk); } } @@ -461,11 +834,10 @@ impl PoolMeta { } pub fn is_bucket_decommissioned(&self, idx: usize, bucket: String) -> bool { - if let Some(ref info) = self.pools[idx].decommission { - info.is_bucket_decommissioned(&bucket) - } else { - false - } + self.pools + .get(idx) + .and_then(|pool| pool.decommission.as_ref()) + .is_some_and(|info| info.is_bucket_decommissioned(&bucket)) } pub fn bucket_done(&mut self, idx: usize, bucket: String) -> bool { @@ -508,14 +880,23 @@ impl PoolMeta { } pub async fn update_after(&mut self, idx: usize, pools: Vec>, duration: Duration) -> Result { - if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) { - return Err(Error::other("InvalidArgument")); - } + let pool_count = self.pools.len(); + ensure_valid_decommission_pool_index(pool_count, idx)?; + let last_update = match self.pools.get(idx) { + Some(pool) if pool.decommission.is_some() => pool.last_update, + Some(_) => { + return Err(decommission_metadata_not_initialized_error("update decommission metadata timestamp")); + } + None => return Err(invalid_decommission_pool_index_error(pool_count, idx)), + }; let now = OffsetDateTime::now_utc(); - if now.unix_timestamp() - self.pools[idx].last_update.unix_timestamp() > duration.whole_seconds() { - self.pools[idx].last_update = now; + if now.unix_timestamp() - last_update.unix_timestamp() > duration.whole_seconds() { + let Some(pool) = self.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + pool.last_update = now; self.save(pools).await?; return Ok(true); @@ -562,18 +943,7 @@ impl PoolMeta { // Determine whether the selected pool should be removed from the retired list. for k in specified_pools.keys() { if let Some(pi) = remembered_pools.get(k) { - if pi.completed { - error!( - "pool({}) = {} is decommissioned, please remove from server command line", - pi.position + 1, - k - ); - // return Err(Error::other(format!( - // "pool({}) = {} is decommissioned, please remove from server command line", - // pi.position + 1, - // k - // ))); - } + ensure_pool_not_left_in_cmdline_after_decommission(pi.position, k, pi.completed)?; } else { // If the previous pool no longer exists, allow updates because a new pool may have been added. update = true; @@ -762,8 +1132,16 @@ fn determine_decommission_final_state(items_failed: usize, was_cancelled: bool) } } -fn remaining_versions_after_decommission(fivs: &FileInfoVersions) -> usize { - fivs.versions.iter().filter(|version| !version.deleted).count() +fn decommission_remaining_version_count(total_versions: usize, expired: usize) -> usize { + total_versions.saturating_sub(expired) +} + +fn should_skip_decommission_delete_marker( + version: &rustfs_filemeta::FileInfo, + remaining_versions: usize, + replication_configured: bool, +) -> bool { + version.deleted && remaining_versions == 1 && !replication_configured } fn decommission_delete_marker_opts( @@ -784,7 +1162,24 @@ fn decommission_delete_marker_opts( } } -async fn should_skip_lifecycle_for_decommission( +fn decommission_remote_tiered_opts( + version: &rustfs_filemeta::FileInfo, + version_id: Option, + src_pool_idx: usize, +) -> ObjectOptions { + ObjectOptions { + versioned: version_id.is_some(), + version_id, + mod_time: version.mod_time, + user_defined: version.metadata.clone(), + src_pool_idx, + data_movement: true, + ..Default::default() + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn should_skip_lifecycle_for_data_movement( store: Arc, bucket: &str, version: &rustfs_filemeta::FileInfo, @@ -792,6 +1187,7 @@ async fn should_skip_lifecycle_for_decommission( lock_retention: Option, replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, apply_actions: bool, + event_source: &LcEventSrc, ) -> bool { let Some(lifecycle_config) = lifecycle_config else { return false; @@ -804,7 +1200,7 @@ async fn should_skip_lifecycle_for_decommission( match event.action { IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => { if apply_actions && object_info.is_remote() { - let _ = apply_expiry_on_transitioned_object(store, &object_info, &event, &LcEventSrc::Decom).await; + let _ = apply_expiry_on_transitioned_object(store, &object_info, &event, event_source).await; } false } @@ -813,7 +1209,7 @@ async fn should_skip_lifecycle_for_decommission( | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => { if apply_actions { - let _ = apply_expiry_rule(&event, &LcEventSrc::Decom, &object_info).await; + let _ = apply_expiry_rule(&event, event_source, &object_info).await; } true } @@ -821,66 +1217,13 @@ async fn should_skip_lifecycle_for_decommission( } } -struct IndexedDecommissionReader { - inner: R, - index: Option, -} - -impl IndexedDecommissionReader { - fn new(inner: R, index: Option) -> Self { - Self { inner, index } - } -} - -impl AsyncRead for IndexedDecommissionReader { - fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - Pin::new(&mut self.inner).poll_read(cx, buf) - } -} - -impl EtagResolvable for IndexedDecommissionReader {} - -impl HashReaderDetector for IndexedDecommissionReader {} - -impl TryGetIndex for IndexedDecommissionReader { - fn try_get_index(&self) -> Option<&Index> { - self.index.as_ref() - } -} - -impl Reader for IndexedDecommissionReader {} - -fn decode_part_index(index: Option<&Bytes>) -> Option { - let bytes = index?; - let mut decoded = Index::new(); - if decoded.load(bytes.as_ref()).is_ok() { - Some(decoded) - } else { - None - } -} - -fn put_obj_reader_from_chunk(chunk: Vec, size: i64, actual_size: i64, index: Option) -> Result { - use sha2::{Digest, Sha256}; - - let sha256hex = if !chunk.is_empty() { - Some(hex_simd::encode_to_string(Sha256::digest(&chunk), hex_simd::AsciiCase::Lower)) - } else { - None - }; - - let reader = IndexedDecommissionReader::new(WarpReader::new(Cursor::new(chunk)), index); - let hash_reader = HashReader::new(Box::new(reader), size, actual_size, None, sha256hex, false)?; - Ok(PutObjReader::new(hash_reader)) -} - impl ECStore { pub async fn status(&self, idx: usize) -> Result { let space_info = self.get_decommission_pool_space_info(idx).await?; let pool_meta = self.pool_meta.read().await; - let mut pool_info = pool_meta.pools[idx].clone(); + let mut pool_info = get_by_index(pool_meta.pools.as_slice(), idx, "fetch decommission status")?.clone(); if let Some(d) = pool_info.decommission.as_mut() { d.total_size = space_info.total; d.current_size = space_info.free; @@ -909,45 +1252,62 @@ impl ECStore { used: total - free, }) } else { - Err(Error::other("InvalidArgument")) + Err(invalid_decommission_pool_index_error(self.pools.len(), idx)) } } #[tracing::instrument(skip(self))] pub async fn decommission_cancel(&self, idx: usize) -> Result<()> { - if self.single_pool() { - return Err(Error::other("InvalidArgument")); - } + ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; + + let mut lock = self.pool_meta.write().await; + let (pool_present, decommission_present, terminal) = if let Some(pool) = lock.pools.get(idx) { + if let Some(info) = pool.decommission.as_ref() { + (true, true, is_decommission_cancel_terminal(info.complete, info.failed, info.canceled)) + } else { + (true, false, false) + } + } else { + (false, false, false) + }; + + ensure_decommission_cancel_allowed(pool_present, decommission_present, terminal)?; + + let should_reload_pool_meta = if lock.decommission_cancel(idx) { + lock.save(self.pools.clone()).await?; + true + } else { + false + }; + drop(lock); let canceler = { let mut cancelers = self.decommission_cancelers.write().await; - let Some(slot) = cancelers.get_mut(idx) else { - return Err(Error::other("InvalidArgument")); - }; - - let Some(canceler) = slot.take() else { - return Err(StorageError::DecommissionNotStarted); - }; - - canceler + take_decommission_canceler(cancelers.as_mut_slice(), idx) }; - - let mut lock = self.pool_meta.write().await; - if lock.decommission_cancel(idx) { - lock.save(self.pools.clone()).await?; - - drop(lock); - - if let Some(notification_sys) = get_global_notification_sys() { - notification_sys.reload_pool_meta().await; - } + if !cancel_decommission_canceler(canceler) { + warn!("decommission_cancel: no active canceler found for pool {}", idx); } - canceler.cancel(); + if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() { + let stage = format!("decommission_cancel for pool {idx}"); + if let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) + { + warn!("{err}"); + } + } Ok(()) } pub async fn is_decommission_running(&self) -> bool { + { + let cancelers = self.decommission_cancelers.read().await; + if has_active_decommission_canceler(cancelers.as_slice()) { + return true; + } + } + let pool_meta = self.pool_meta.read().await; for pool in pool_meta.pools.iter() { if let Some(ref info) = pool.decommission @@ -962,29 +1322,63 @@ impl ECStore { false } + pub(crate) async fn spawn_decommission_routines( + &self, + store: Arc, + rx: CancellationToken, + indices: Vec, + ) -> Result<()> { + let indices = dedup_indices(&indices); + if indices.is_empty() { + return Ok(()); + } + + let index_cancelers = { + let mut cancelers = self.decommission_cancelers.write().await; + bind_decommission_cancelers(indices.as_slice(), &rx, cancelers.as_mut_slice()) + }; + + ensure_decommission_routines_scheduled(index_cancelers.len(), indices.len())?; + + for (idx, canceler) in index_cancelers { + let store = store.clone(); + tokio::spawn(async move { + if let Err(err) = store.do_decommission_in_routine(canceler, idx).await { + error!("decommission: routine failed for idx {}: {err}", idx); + } + }); + } + + Ok(()) + } + #[tracing::instrument(skip(self, rx))] pub async fn decommission(&self, rx: CancellationToken, indices: Vec) -> Result<()> { - warn!("decommission: {:?}", indices); - if indices.is_empty() { - return Err(Error::other("InvalidArgument")); - } + let indices = dedup_indices(&indices); - if self.single_pool() { - return Err(Error::other("InvalidArgument")); - } + warn!("decommission: {:?}", indices); + validate_start_decommission_request(&indices, self.single_pool())?; + + ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?; + + let store = require_decommission_store(new_object_layer_fn(), "start decommission")?; self.start_decommission(indices.clone()).await?; - - let rx_clone = rx.clone(); - tokio::spawn(async move { - let Some(store) = new_object_layer_fn() else { - error!("store not init"); - return; - }; - for idx in indices.iter() { - store.do_decommission_in_routine(rx_clone.clone(), *idx).await; + if let Err(err) = self.spawn_decommission_routines(store, rx, indices.clone()).await { + let mut rollback_err: Option = None; + for idx in indices { + if let Err(cancel_err) = self.decommission_cancel(idx).await { + error!( + "decommission: failed to rollback decommission state for idx {} after spawn error: {:?}", + idx, cancel_err + ); + if rollback_err.is_none() { + rollback_err = Some(Error::other(format!("decommission rollback failed for idx {idx}: {cancel_err}"))); + } + } } - }); + return Err(resolve_decommission_spawn_failure_result(err, rollback_err)); + } Ok(()) } @@ -1001,21 +1395,15 @@ impl ECStore { lifecycle_config: Option, lock_retention: Option, replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, - ) { + ) -> Result<()> { warn!("decommission_entry: {} {}", &bucket, &entry.name); wk.give().await; if entry.is_dir() { warn!("decommission_entry: skip dir {}", &entry.name); - return; + return Ok(()); } - let mut fivs = match entry.file_info_versions(&bucket) { - Ok(f) => f, - Err(err) => { - error!("decommission_pool: file_info_versions err {:?}", &err); - return; - } - }; + let mut fivs = load_decommission_entry_versions(&entry, &bucket, "file_info_versions")?; fivs.versions.sort_by(|a, b| b.mod_time.cmp(&a.mod_time)); @@ -1023,7 +1411,7 @@ impl ECStore { let mut expired: usize = 0; for version in fivs.versions.iter() { - if should_skip_lifecycle_for_decommission( + if should_skip_lifecycle_for_data_movement( self.clone(), &bucket, version, @@ -1031,16 +1419,16 @@ impl ECStore { lock_retention.clone(), replication_config.clone(), true, + &LcEventSrc::Decom, ) .await { expired += 1; - decommissioned += 1; continue; } - let remaining_versions = fivs.versions.len() - expired; - if version.deleted && remaining_versions == 1 && replication_config.is_none() { + let remaining_versions = decommission_remaining_version_count(fivs.versions.len(), expired); + if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) { // decommissioned += 1; info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped"); @@ -1050,6 +1438,7 @@ impl ECStore { let version_id = version.version_id.map(|v| v.to_string()); let mut ignore = false; + let mut cleanup_ignored = false; let mut failure = false; let mut error = None; if version.deleted { @@ -1067,16 +1456,32 @@ impl ECStore { &bucket, &version.name, &version_id, &err ); ignore = true; - continue; + cleanup_ignored = true; + } else { + failure = true; + + error = Some(err) } + } - failure = true; - - error = Some(err) + if ignore { + if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) { + decommissioned += 1; + } + info!("decommission_pool: ignore {}", &version.name); + continue; } { - self.pool_meta.write().await.count_item(idx, 0, failure); + let mut pool_meta = self.pool_meta.write().await; + if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, failure) { + return Err(with_decommission_entry_context( + "count_decommission_item", + bucket.as_str(), + entry.name.as_str(), + err, + )); + } } if !failure { @@ -1097,20 +1502,14 @@ impl ECStore { bucket.as_str(), &version.name, version, - &ObjectOptions { - version_id: version_id.clone(), - mod_time: version.mod_time, - user_defined: version.metadata.clone(), - src_pool_idx: idx, - data_movement: true, - ..Default::default() - }, + &decommission_remote_tiered_opts(version, version_id.clone(), idx), ) .await { if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { ignore = true; + cleanup_ignored = true; break; } @@ -1141,6 +1540,7 @@ impl ECStore { Err(err) => { if is_err_object_not_found(&err) || is_err_version_not_found(&err) { ignore = true; + cleanup_ignored = true; break; } @@ -1165,6 +1565,7 @@ impl ECStore { if let Err(err) = self.clone().decommission_object(idx, bucket, rd).await { if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { ignore = true; + cleanup_ignored = true; break; } @@ -1184,24 +1585,36 @@ impl ECStore { } if ignore { + if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) { + decommissioned += 1; + } info!("decommission_pool: ignore {}", &version.name); continue; } { - let size = usize::try_from(version.size).unwrap_or_default(); - self.pool_meta.write().await.count_item(idx, size, failure); + let mut pool_meta = self.pool_meta.write().await; + if let Err(err) = count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), failure) { + return Err(with_decommission_entry_context( + "count_decommission_item", + bucket.as_str(), + entry.name.as_str(), + err, + )); + } } if failure { break; } - decommissioned += 1; + if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) { + decommissioned += 1; + } } - if decommissioned == fivs.versions.len() - && let Err(err) = set + if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) { + let cleanup_result = set .delete_object( bucket.as_str(), &encode_dir_object(&entry.name), @@ -1212,36 +1625,55 @@ impl ECStore { ..Default::default() }, ) - .await - { - error!("decommission_pool: delete_object err {:?}", &err); - } else if decommissioned != fivs.versions.len() { + .await; + resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())? + } else if decommissioned != fivs.versions.len() || expired > 0 { warn!( - "decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned", + "decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned and {} expired by lifecycle", &bucket, &entry.name, decommissioned, - fivs.versions.len() + fivs.versions.len(), + expired ); } { let mut pool_meta = self.pool_meta.write().await; - pool_meta.track_current_bucket_object(idx, bucket.clone(), entry.name.clone()); + if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) { + return Err(with_decommission_entry_context( + "track_decommission_current_object", + bucket.as_str(), + entry.name.as_str(), + err, + )); + } - let ok = pool_meta - .update_after(idx, self.pools.clone(), Duration::seconds(30)) - .await - .unwrap_or_default(); + let ok = match resolve_decommission_update_after_result( + pool_meta.update_after(idx, self.pools.clone(), Duration::seconds(30)).await, + ) { + Ok(ok) => ok, + Err(err) => { + return Err(with_decommission_entry_context("update_after", bucket.as_str(), entry.name.as_str(), err)); + } + }; drop(pool_meta); - if ok && let Some(notification_sys) = get_global_notification_sys() { - notification_sys.reload_pool_meta().await; + if ok + && let Some(notification_sys) = get_global_notification_sys() + && let Err(err) = resolve_decommission_entry_reload_result( + notification_sys.reload_pool_meta().await, + bucket.as_str(), + entry.name.as_str(), + ) + { + warn!("{err}"); } } warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name); + Ok(()) } #[tracing::instrument(skip(self, rx))] @@ -1253,20 +1685,26 @@ impl ECStore { bi: DecomBucketInfo, ) -> Result<()> { let wk = Workers::new(pool.disk_set.len() * 2).map_err(Error::other)?; + let entry_error = Arc::new(tokio::sync::Mutex::new(None::)); + let mut listing_workers = Vec::with_capacity(pool.disk_set.len()); let mut lifecycle_config = None; let mut lock_retention = None; let mut replication_config = None; if bi.name != RUSTFS_META_BUCKET { - let _ = BucketVersioningSys::get(&bi.name).await?; + let _ = resolve_decommission_optional_bucket_config_result( + &bi.name, + "versioning", + BucketVersioningSys::get(&bi.name).await, + )?; lifecycle_config = GLOBAL_LifecycleSys.get(&bi.name).await; lock_retention = BucketObjectLockSys::get(&bi.name).await; - replication_config = match metadata_sys::get_replication_config(&bi.name).await { - Ok(config) => Some(config), - Err(Error::ConfigNotFound) => None, - Err(err) => return Err(err), - }; + replication_config = resolve_decommission_optional_bucket_config_result( + &bi.name, + "replication", + metadata_sys::get_replication_config(&bi.name).await, + )?; } for (set_idx, set) in pool.disk_set.iter().enumerate() { @@ -1282,6 +1720,8 @@ impl ECStore { let lifecycle_config = lifecycle_config.clone(); let lock_retention = lock_retention.clone(); let replication_config = replication_config.clone(); + let entry_error = entry_error.clone(); + let callback_rx = rx.clone(); move |entry: MetaCacheEntry| { let this = this.clone(); let bucket = bucket.clone(); @@ -1290,11 +1730,22 @@ impl ECStore { let lifecycle_config = lifecycle_config.clone(); let lock_retention = lock_retention.clone(); let replication_config = replication_config.clone(); + let entry_error = entry_error.clone(); + let callback_rx = callback_rx.clone(); Box::pin(async move { wk.take().await; - this.decommission_entry(idx, entry, bucket, set, wk, lifecycle_config, lock_retention, replication_config) + if let Err(err) = this + .decommission_entry(idx, entry, bucket, set, wk, lifecycle_config, lock_retention, replication_config) .await + { + error!("decommission_pool: decommission_entry failed: {err}"); + let mut first_err = entry_error.lock().await; + if first_err.is_none() { + *first_err = Some(err); + callback_rx.cancel(); + } + } }) } }); @@ -1304,7 +1755,7 @@ impl ECStore { let bi = bi.clone(); let set_id = set_idx; let wk_clone = wk.clone(); - tokio::spawn(async move { + let worker = tokio::spawn(async move { loop { if rx_clone.is_cancelled() { warn!("decommission_pool: cancel {}", set_id); @@ -1334,90 +1785,121 @@ impl ECStore { wk_clone.give().await; }); + listing_workers.push((set_id, worker)); } warn!("decommission_pool: decommission_pool wait {} {}", idx, &bi.name); + let mut listing_worker_error = None; + for (set_id, worker) in listing_workers { + if let Err(err) = resolve_decommission_listing_worker_result(set_id, worker.await) { + rx.cancel(); + wk.give().await; + if listing_worker_error.is_none() { + listing_worker_error = Some(err); + } + } + } + wk.wait().await; + if let Some(err) = listing_worker_error { + return Err(err); + } + + if let Some(err) = entry_error.lock().await.clone() { + return Err(err); + } + + if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) { + warn!("decommission_pool: canceled after wait {} {}", idx, &bi.name); + return Err(err); + } + warn!("decommission_pool: decommission_pool done {} {}", idx, &bi.name); Ok(()) } #[tracing::instrument(skip(self, rx))] - pub async fn do_decommission_in_routine(self: &Arc, rx: CancellationToken, idx: usize) { - let decommission_token = rx.child_token(); - { + pub async fn do_decommission_in_routine(self: &Arc, rx: CancellationToken, idx: usize) -> Result<()> { + defer!(|| async { let mut cancelers = self.decommission_cancelers.write().await; - if let Some(slot) = cancelers.get_mut(idx) { - *slot = Some(decommission_token.clone()); + if take_decommission_canceler(cancelers.as_mut_slice(), idx).is_none() { + warn!("decommission: canceler already cleared for pool {}", idx); } - } + }); - if let Err(err) = self.decommission_in_background(decommission_token.clone(), idx).await { - error!("decom err {:?}", &err); - if let Err(er) = self.decommission_failed(idx).await { - error!("decom failed err {:?}", &er); + let result = self.decommission_in_background(rx.clone(), idx).await; + + let (final_state, canceled, cmd_line) = { + let pool_meta = self.pool_meta.read().await; + let Some(pool) = pool_meta.pools.get(idx) else { + error!("decommission: pool metadata missing for idx {}", idx); + return Err(Error::other(format!( + "failed to resolve decommission final state: pool metadata missing for idx {idx}" + ))); + }; + + let (final_state, canceled) = if let Some(info) = &pool.decommission { + ( + determine_decommission_final_state(info.items_decommission_failed, info.canceled), + info.canceled, + ) } else { - warn!("decommission: decommission_failed {}", idx); + (DecommissionFinalState::Failed, false) + }; + let cmd_line = pool.cmd_line.clone(); + (final_state, canceled, cmd_line) + }; + + if let Err(err) = result { + error!("decom err {:?}", &err); + + if is_err_operation_canceled(&err) || should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) { + warn!("decommission: canceled for pool {}, preserving canceled state", cmd_line); + return Ok(()); } - return; + resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?; + warn!("decommission: decommission_failed {}", idx); + + return Ok(()); } warn!("decommission: decommission_in_background complete {}", idx); - let (final_state, cmd_line) = { - let pool_meta = self.pool_meta.read().await; - let final_state = { - if let Some(info) = &pool_meta.pools[idx].decommission { - determine_decommission_final_state(info.items_decommission_failed, info.canceled) - } else { - DecommissionFinalState::Failed - } - }; - let cmd_line = pool_meta.pools[idx].cmd_line.clone(); - (final_state, cmd_line) - }; - - let mut completed_successfully = false; - - if final_state == DecommissionFinalState::Complete { - warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line); - if let Err(err) = self.check_after_decommission(idx).await { - error!("decom post-check err {:?}", &err); - if let Err(er) = self.decommission_failed(idx).await { - error!("decom failed err {:?}", &er); - } - } else if let Err(er) = self.complete_decommission(idx).await { - error!("decom complete err {:?}", &er); - } else { - completed_successfully = true; - } - } else if let Err(er) = self.decommission_failed(idx).await { - error!("decom failed err {:?}", &er); + if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) { + warn!("decommission: canceled for pool {}, skipping terminal state overwrite", cmd_line); + return Ok(()); } - { - let mut cancelers = self.decommission_cancelers.write().await; - if let Some(slot) = cancelers.get_mut(idx) { - *slot = None; + match final_state { + DecommissionFinalState::Complete => { + warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line); + if let Err(err) = self.check_after_decommission(idx).await { + resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?; + return Err(Error::other(format!( + "failed to finalize decommission for pool {cmd_line}: post-check failed: {err}" + ))); + } + + warn!("Decommissioning complete for pool {}, marking completed state", cmd_line); + resolve_decommission_terminal_mark_result(self.complete_decommission(idx).await, "completed", &cmd_line)?; + } + DecommissionFinalState::Failed => { + warn!("Decommissioning finished with failed items for pool {}, marking failed state", cmd_line); + resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?; } } - if completed_successfully { - warn!("Decommissioning complete for pool {}", cmd_line); - } else { - warn!("Decommissioning finished in failed state for pool {}", cmd_line); - } + warn!("Decommissioning complete for pool {}", cmd_line); + Ok(()) } #[tracing::instrument(skip(self))] pub async fn decommission_failed(&self, idx: usize) -> Result<()> { - if self.single_pool() { - return Err(Error::other("errInvalidArgument")); - } + ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?; let mut pool_meta = self.pool_meta.write().await; if pool_meta.decommission_failed(idx) { @@ -1426,7 +1908,12 @@ impl ECStore { drop(pool_meta); if let Some(notification_sys) = get_global_notification_sys() { - notification_sys.reload_pool_meta().await; + let stage = format!("decommission_failed for pool {idx}"); + if let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) + { + warn!("{err}"); + } } } @@ -1443,16 +1930,19 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn complete_decommission(&self, idx: usize) -> Result<()> { - if self.single_pool() { - return Err(Error::other("errInvalidArgument")); - } + ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?; let mut pool_meta = self.pool_meta.write().await; if pool_meta.decommission_complete(idx) { pool_meta.save(self.pools.clone()).await?; drop(pool_meta); if let Some(notification_sys) = get_global_notification_sys() { - notification_sys.reload_pool_meta().await; + let stage = format!("complete_decommission for pool {idx}"); + if let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) + { + warn!("{err}"); + } } } @@ -1469,7 +1959,7 @@ impl ECStore { #[tracing::instrument(skip(self, rx))] async fn decommission_in_background(self: &Arc, rx: CancellationToken, idx: usize) -> Result<()> { - let pool = self.pools[idx].clone(); + let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone(); let pending = { let pool_meta = self.pool_meta.read().await; @@ -1479,7 +1969,7 @@ impl ECStore { for bucket in pending.iter() { let is_decommissioned = { let pool_meta = self.pool_meta.read().await; - pool_meta.is_bucket_decommissioned(idx, bucket.to_string()) + resolve_decommission_bucket_state(&pool_meta, idx, bucket)? }; if is_decommissioned { @@ -1487,10 +1977,12 @@ impl ECStore { { let mut pool_meta = self.pool_meta.write().await; - if pool_meta.bucket_done(idx, bucket.to_string()) - && let Err(err) = pool_meta.save(self.pools.clone()).await - { - error!("decom pool_meta.save err {:?}", err); + if mark_decommission_bucket_done(&mut pool_meta, idx, bucket)? { + resolve_decommission_bucket_done_save_result( + pool_meta.save(self.pools.clone()).await, + idx, + bucket.name.as_str(), + )?; } } continue; @@ -1505,12 +1997,19 @@ impl ECStore { warn!("decommission: decommission_pool done {}", &bucket.name); } + if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) { + warn!("decommission: cancellation observed after decommission_pool {}", &bucket.name); + return Err(err); + } + { let mut pool_meta = self.pool_meta.write().await; - if pool_meta.bucket_done(idx, bucket.to_string()) - && let Err(err) = pool_meta.save(self.pools.clone()).await - { - error!("decom pool_meta.save err {:?}", err); + if mark_decommission_bucket_done(&mut pool_meta, idx, bucket)? { + resolve_decommission_bucket_done_save_result( + pool_meta.save(self.pools.clone()).await, + idx, + bucket.name.as_str(), + )?; } warn!("decommission: decommission_pool bucket_done {}", &bucket.name); @@ -1522,18 +2021,27 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn start_decommission(&self, indices: Vec) -> Result<()> { - if indices.is_empty() { - return Err(Error::other("errInvalidArgument")); + let indices = dedup_indices(&indices); + validate_start_decommission_request(&indices, self.single_pool())?; + + ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?; + + for idx in indices.iter().copied() { + ensure_valid_decommission_pool_index(self.pools.len(), idx)?; } - if self.single_pool() { - return Err(Error::other("errInvalidArgument")); + { + let pool_meta = self.pool_meta.read().await; + for idx in indices.iter().copied() { + let (pool_present, decommission_active) = decommission_start_guard_state(pool_meta.pools.get(idx)); + ensure_decommission_start_allowed(pool_present, decommission_active)?; + } } let decom_buckets = self.get_buckets_to_decommission().await?; for bk in decom_buckets.iter() { - let _ = self.heal_bucket(&bk.name, &HealOpts::default()).await; + resolve_decommission_preflight_heal_result(&bk.name, self.heal_bucket(&bk.name, &HealOpts::default()).await)?; } let meta_buckets = [ @@ -1552,19 +2060,32 @@ impl ECStore { } } + let mut space_infos = Vec::with_capacity(indices.len()); + for idx in indices.iter().copied() { + let pi = self.get_decommission_pool_space_info(idx).await?; + space_infos.push((idx, pi)); + } + + ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?; + let mut pool_meta = self.pool_meta.write().await; - for idx in indices.iter() { - let pi = self.get_decommission_pool_space_info(*idx).await?; + for idx in indices.iter().copied() { + let (pool_present, decommission_active) = decommission_start_guard_state(pool_meta.pools.get(idx)); + ensure_decommission_start_allowed(pool_present, decommission_active)?; + } - pool_meta.decommission(*idx, pi)?; - - pool_meta.queue_buckets(*idx, decom_buckets.clone()); + for (idx, pi) in space_infos { + pool_meta.decommission(idx, pi)?; + pool_meta.queue_buckets(idx, decom_buckets.clone()); } pool_meta.save(self.pools.clone()).await?; - if let Some(notification_sys) = get_global_notification_sys() { - notification_sys.reload_pool_meta().await; + if let Some(notification_sys) = get_global_notification_sys() + && let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, "start_decommission") + { + warn!("{err}"); } Ok(()) @@ -1605,29 +2126,39 @@ impl ECStore { if bucket_info.name != RUSTFS_META_BUCKET { lifecycle_config = GLOBAL_LifecycleSys.get(&bucket_info.name).await; lock_retention = BucketObjectLockSys::get(&bucket_info.name).await; - replication_config = match metadata_sys::get_replication_config(&bucket_info.name).await { - Ok(config) => Some(config), - Err(Error::ConfigNotFound) => None, - Err(err) => return Err(err), - }; + replication_config = resolve_decommission_optional_bucket_config_result( + &bucket_info.name, + "replication", + metadata_sys::get_replication_config(&bucket_info.name).await, + )?; } let versions_found = Arc::new(AtomicUsize::new(0)); + let entry_error = Arc::new(tokio::sync::Mutex::new(None::)); + let callback_rx = CancellationToken::new(); let versions_found_cb = versions_found.clone(); + let entry_error_cb = entry_error.clone(); let bucket_name = bucket_info.name.clone(); let lifecycle_config_cb = lifecycle_config.clone(); let lock_retention_cb = lock_retention.clone(); let replication_config_cb = replication_config.clone(); let store = Arc::clone(self); + let callback_rx_cb = callback_rx.clone(); let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| { let versions_found = versions_found_cb.clone(); + let entry_error = entry_error_cb.clone(); let bucket_name = bucket_name.clone(); let lifecycle_config = lifecycle_config_cb.clone(); let lock_retention = lock_retention_cb.clone(); let replication_config = replication_config_cb.clone(); let store = Arc::clone(&store); + let callback_rx = callback_rx_cb.clone(); Box::pin(async move { + if callback_rx.is_cancelled() { + return; + } + if !entry.is_object() { return; } @@ -1636,8 +2167,20 @@ impl ECStore { return; } - let Ok(fivs) = entry.file_info_versions(&bucket_name) else { - return; + let fivs = match load_decommission_entry_versions( + &entry, + &bucket_name, + "check_after_decommission.file_info_versions", + ) { + Ok(fivs) => fivs, + Err(err) => { + let mut first_err = entry_error.lock().await; + if first_err.is_none() { + *first_err = Some(err); + callback_rx.cancel(); + } + return; + } }; let mut remaining = 0; @@ -1645,7 +2188,7 @@ impl ECStore { if version.deleted { continue; } - if should_skip_lifecycle_for_decommission( + if should_skip_lifecycle_for_data_movement( Arc::clone(&store), &bucket_name, version, @@ -1653,6 +2196,7 @@ impl ECStore { lock_retention.clone(), replication_config.clone(), false, + &LcEventSrc::Decom, ) .await { @@ -1665,8 +2209,11 @@ impl ECStore { }) }); - set.list_objects_to_decommission(CancellationToken::new(), bucket_info.clone(), callback) - .await?; + let list_result = set + .list_objects_to_decommission(callback_rx, bucket_info.clone(), callback) + .await; + let entry_error = entry_error.lock().await.clone(); + resolve_decommission_check_after_list_result(list_result, entry_error)?; let versions_found = versions_found.load(Ordering::Relaxed); if versions_found > 0 { @@ -1684,143 +2231,12 @@ impl ECStore { #[tracing::instrument(skip(self, rd))] async fn decommission_object(self: Arc, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> { warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name); - let object_info = rd.object_info.clone(); - - // TODO: check : use size or actual_size ? - let _actual_size = object_info.get_actual_size()?; - - if object_info.is_multipart() { - let res = match self - .new_multipart_upload( - &bucket, - &object_info.name, - &ObjectOptions { - version_id: object_info.version_id.as_ref().map(|v| v.to_string()), - user_defined: object_info.user_defined.clone(), - src_pool_idx: pool_idx, - data_movement: true, - ..Default::default() - }, - ) - .await - { - Ok(res) => res, - Err(err) => { - error!("decommission_object: new_multipart_upload err {:?}", &err); - return Err(err); - } - }; - - defer!(|| async { - if let Err(err) = self - .abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default()) - .await - { - error!("decommission_object: abort_multipart_upload err {:?}", &err); - } - }); - - let mut parts = vec![CompletePart::default(); object_info.parts.len()]; - - let mut reader = rd.stream; - - for (i, part) in object_info.parts.iter().enumerate() { - let mut chunk = vec![0u8; part.size]; - - reader.read_exact(&mut chunk).await?; - - let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?; - let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size }; - let index = decode_part_index(part.index.as_ref()); - let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?; - - let pi = match self - .put_object_part( - &bucket, - &object_info.name, - &res.upload_id, - part.number, - &mut data, - &ObjectOptions { - preserve_etag: Some(part.etag.clone()), - ..Default::default() - }, - ) - .await - { - Ok(pi) => pi, - Err(err) => { - error!("decommission_object: put_object_part {} err {:?}", i, &err); - return Err(err); - } - }; - - warn!("decommission_object: put_object_part {} done {} {}", i, &bucket, &object_info.name); - - parts[i] = CompletePart { - part_num: pi.part_num, - etag: pi.etag, - - ..Default::default() - }; - } - - if let Err(err) = self - .clone() - .complete_multipart_upload( - &bucket, - &object_info.name, - &res.upload_id, - parts, - &ObjectOptions { - data_movement: true, - mod_time: object_info.mod_time, - ..Default::default() - }, - ) - .await - { - error!("decommission_object: complete_multipart_upload err {:?}", &err); - return Err(err); - } - - warn!("decommission_object: complete_multipart_upload done {} {}", &bucket, &object_info.name); - return Ok(()); + let object_name = rd.object_info.name.clone(); + let result = data_movement::migrate_object(self, pool_idx, bucket.clone(), rd, "decommission_object").await; + if result.is_ok() { + warn!("decommission_object: migrated {} {}", &bucket, &object_name); } - - let actual_size = object_info.get_actual_size()?; - let index = object_info - .parts - .first() - .and_then(|part| decode_part_index(part.index.as_ref())); - let reader = IndexedDecommissionReader::new(WarpReader::new(BufReader::new(rd.stream)), index); - let hrd = HashReader::new(Box::new(reader), object_info.size, actual_size, object_info.etag.clone(), None, false)?; - let mut data = PutObjReader::new(hrd); - - if let Err(err) = self - .put_object( - &bucket, - &object_info.name, - &mut data, - &ObjectOptions { - src_pool_idx: pool_idx, - data_movement: true, - version_id: object_info.version_id.as_ref().map(|v| v.to_string()), - mod_time: object_info.mod_time, - user_defined: object_info.user_defined.clone(), - preserve_etag: object_info.etag.clone(), - - ..Default::default() - }, - ) - .await - { - error!("decommission_object: put_object err {:?}", &err); - return Err(err); - } - - warn!("decommission_object: put_object done {} {}", &bucket, &object_info.name); - Ok(()) + result } } @@ -1829,6 +2245,22 @@ impl ECStore { mod tests { use super::*; + #[test] + fn ensure_pool_not_left_in_cmdline_after_decommission_allows_active_pool() { + assert!(ensure_pool_not_left_in_cmdline_after_decommission(0, "http://node{1...4}/disk{1...4}", false).is_ok()); + } + + #[test] + fn ensure_pool_not_left_in_cmdline_after_decommission_rejects_completed_pool() { + let err = ensure_pool_not_left_in_cmdline_after_decommission(1, "http://node{1...4}/disk{1...4}", true) + .expect_err("completed decommissioned pool should fail validation"); + + assert!( + err.to_string() + .contains("pool(2) = http://node{1...4}/disk{1...4} is decommissioned, please remove from server command line") + ); + } + #[test] fn determine_decommission_final_state_marks_failures_and_cancellations() { assert_eq!(determine_decommission_final_state(0, false), DecommissionFinalState::Complete); @@ -1837,24 +2269,47 @@ mod tests { } #[test] - fn remaining_versions_after_decommission_ignores_delete_markers() { - let fivs = FileInfoVersions { - versions: vec![ - rustfs_filemeta::FileInfo { - deleted: false, - size: 128, - ..Default::default() - }, - rustfs_filemeta::FileInfo { - deleted: true, - size: 0, - ..Default::default() - }, - ], + fn decommission_remaining_version_count_excludes_only_expired_versions() { + assert_eq!(decommission_remaining_version_count(1, 0), 1); + assert_eq!(decommission_remaining_version_count(2, 1), 1); + assert_eq!(decommission_remaining_version_count(1, 1), 0); + } + + #[test] + fn should_skip_decommission_delete_marker_when_last_remaining_without_replication() { + let version = rustfs_filemeta::FileInfo { + deleted: true, ..Default::default() }; - assert_eq!(remaining_versions_after_decommission(&fivs), 1); + assert!(should_skip_decommission_delete_marker(&version, 1, false)); + } + + #[test] + fn should_skip_decommission_delete_marker_rejects_configured_replication() { + let version = rustfs_filemeta::FileInfo { + deleted: true, + ..Default::default() + }; + + assert!(!should_skip_decommission_delete_marker(&version, 1, true)); + } + + #[test] + fn should_skip_decommission_delete_marker_rejects_non_deleted_versions() { + let version = rustfs_filemeta::FileInfo::default(); + + assert!(!should_skip_decommission_delete_marker(&version, 1, false)); + } + + #[test] + fn should_skip_decommission_delete_marker_rejects_multiple_remaining_versions() { + let version = rustfs_filemeta::FileInfo { + deleted: true, + ..Default::default() + }; + + assert!(!should_skip_decommission_delete_marker(&version, 2, false)); } #[test] @@ -1886,6 +2341,25 @@ mod tests { assert_eq!(replication.replicate_decision_str, "existing"); } + #[test] + fn decommission_remote_tiered_opts_preserves_versioning_context() { + let mod_time = OffsetDateTime::now_utc(); + let version = rustfs_filemeta::FileInfo { + mod_time: Some(mod_time), + metadata: std::collections::HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]), + ..Default::default() + }; + + let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9); + + assert!(opts.versioned); + assert!(opts.data_movement); + assert_eq!(opts.src_pool_idx, 9); + assert_eq!(opts.version_id.as_deref(), Some("version-id")); + assert_eq!(opts.mod_time, Some(mod_time)); + assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value")); + } + #[test] fn decommission_state_transitions_preserve_start_time() { let start_time = OffsetDateTime::now_utc(); @@ -2047,9 +2521,7 @@ impl SetDisks { cb_func: ListCallback, ) -> Result<()> { let (disks, _) = self.get_online_disks_with_healing(false).await; - if disks.is_empty() { - return Err(Error::other("errNoDiskAvailable")); - } + ensure_decommission_listing_disks_available(!disks.is_empty(), &bucket_info.name)?; let listing_quorum = self.set_drive_count.div_ceil(2); @@ -2272,3 +2744,1005 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi total } + +#[cfg(test)] +mod pools_tests { + use super::{ + DecomBucketInfo, DecommissionTerminalState, PoolDecommissionInfo, PoolMeta, PoolStatus, bind_decommission_cancelers, + cancel_decommission_canceler, classify_decommission_terminal_state, count_decommission_item, + decommission_cancel_signal_result, decommission_item_size, decommission_start_guard_state, dedup_indices, + ensure_decommission_cancel_allowed, ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, + ensure_decommission_start_allowed, ensure_decommission_terminal_operation_supported, + ensure_valid_decommission_pool_index, get_by_index, has_active_decommission_canceler, is_decommission_active, + is_decommission_cancel_terminal, load_decommission_entry_versions, mark_decommission_bucket_done, + require_decommission_store, resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, + resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result, + resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result, + resolve_decommission_optional_bucket_config_result, resolve_decommission_pool_meta_reload_result, + resolve_decommission_preflight_heal_result, resolve_decommission_spawn_failure_result, + resolve_decommission_terminal_mark_after_error_result, resolve_decommission_terminal_mark_result, + resolve_decommission_update_after_result, should_cleanup_decommission_source_entry, + should_count_decommission_version_complete, should_preserve_decommission_canceled_state, take_decommission_canceler, + track_decommission_current_object, validate_start_decommission_request, with_decommission_entry_context, + }; + use crate::data_movement; + use crate::error::Error; + use rustfs_filemeta::MetaCacheEntry; + use rustfs_rio::Index; + use time::{Duration, OffsetDateTime}; + use tokio_util::sync::CancellationToken; + + #[test] + fn test_dedup_indices_removes_duplicates_preserving_order() { + assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]); + } + + #[test] + fn test_dedup_indices_handles_empty_input() { + let empty: Vec = Vec::new(); + assert!(dedup_indices(&empty).is_empty()); + } + + #[test] + fn test_get_by_index_returns_value_when_in_range() { + let values = vec!["a", "b", "c"]; + let value = get_by_index(values.as_slice(), 1, "fetch decommission status").expect("in-range index should return value"); + assert_eq!(*value, "b"); + } + + #[test] + fn test_get_by_index_returns_error_when_out_of_range() { + let values = vec![1_u8]; + let err = + get_by_index(values.as_slice(), 2, "load decommission background pool").expect_err("out-of-range index should fail"); + assert!( + err.to_string() + .contains("failed to load decommission background pool: invalid decommission pool index 2 for 1 pools") + ); + } + + #[test] + fn test_pool_meta_is_suspended_returns_false_for_out_of_range() { + let meta = PoolMeta::default(); + assert!(!meta.is_suspended(1)); + } + + #[test] + fn test_pool_meta_queue_buckets_ignores_out_of_range_index() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo::default()), + }], + ..Default::default() + }; + + meta.queue_buckets( + 9, + vec![DecomBucketInfo { + name: "bucket-a".to_string(), + prefix: String::new(), + }], + ); + + let queued = meta.pools[0] + .decommission + .as_ref() + .expect("pool should have decommission info") + .queued_buckets + .clone(); + assert!(queued.is_empty()); + } + + #[test] + fn test_pool_meta_is_bucket_decommissioned_returns_false_for_out_of_range() { + let meta = PoolMeta::default(); + assert!(!meta.is_bucket_decommissioned(7, "bucket-a".to_string())); + } + + #[test] + fn test_resolve_decommission_bucket_state_rejects_out_of_range_index() { + let meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo::default()), + }], + ..Default::default() + }; + + let bucket = DecomBucketInfo { + name: "bucket-a".to_string(), + prefix: String::new(), + }; + let err = + resolve_decommission_bucket_state(&meta, 3, &bucket).expect_err("out-of-range index should return invalid argument"); + assert!(err.to_string().contains("invalid decommission pool index 3 for 1 pools")); + } + + #[test] + fn test_resolve_decommission_bucket_state_rejects_missing_decommission_meta() { + let meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }], + ..Default::default() + }; + + let bucket = DecomBucketInfo { + name: "bucket-a".to_string(), + prefix: String::new(), + }; + let err = resolve_decommission_bucket_state(&meta, 0, &bucket) + .expect_err("missing decommission metadata should return explicit error"); + assert!( + err.to_string() + .contains("failed to resolve decommission bucket state: decommission metadata not initialized") + ); + } + + #[test] + fn test_resolve_decommission_bucket_state_returns_true_for_done_bucket() { + let meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + decommissioned_buckets: vec!["bucket-a".to_string()], + ..Default::default() + }), + }], + ..Default::default() + }; + + let bucket = DecomBucketInfo { + name: "bucket-a".to_string(), + prefix: String::new(), + }; + let done = resolve_decommission_bucket_state(&meta, 0, &bucket).expect("valid state should resolve"); + assert!(done); + } + + #[test] + fn test_mark_decommission_bucket_done_rejects_missing_decommission_meta() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }], + ..Default::default() + }; + + let bucket = DecomBucketInfo { + name: "bucket-a".to_string(), + prefix: String::new(), + }; + let err = mark_decommission_bucket_done(&mut meta, 0, &bucket) + .expect_err("missing decommission metadata should return explicit error"); + assert!( + err.to_string() + .contains("failed to mark decommission bucket done: decommission metadata not initialized") + ); + } + + #[test] + fn test_mark_decommission_bucket_done_rejects_out_of_range_index() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo::default()), + }], + ..Default::default() + }; + + let bucket = DecomBucketInfo { + name: "bucket-a".to_string(), + prefix: String::new(), + }; + let err = + mark_decommission_bucket_done(&mut meta, 1, &bucket).expect_err("out-of-range index should return invalid argument"); + assert!(err.to_string().contains("invalid decommission pool index 1 for 1 pools")); + } + + #[test] + fn test_mark_decommission_bucket_done_pops_bucket_when_present() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + queued_buckets: vec!["bucket-a".to_string()], + ..Default::default() + }), + }], + ..Default::default() + }; + + let bucket = DecomBucketInfo { + name: "bucket-a".to_string(), + prefix: String::new(), + }; + let popped = mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("valid state should mark bucket done"); + assert!(popped); + } + + #[test] + fn test_count_decommission_item_rejects_missing_decommission_meta() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }], + ..Default::default() + }; + + let err = count_decommission_item(&mut meta, 0, 64, true) + .expect_err("missing decommission metadata should return explicit error"); + assert!( + err.to_string() + .contains("failed to count decommission item: decommission metadata not initialized") + ); + } + + #[test] + fn test_count_decommission_item_updates_done_and_failed_counters() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo::default()), + }], + ..Default::default() + }; + + count_decommission_item(&mut meta, 0, 32, false).expect("success counter should be updated"); + count_decommission_item(&mut meta, 0, 16, true).expect("failed counter should be updated"); + + let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist"); + assert_eq!(info.items_decommissioned, 1); + assert_eq!(info.bytes_done, 32); + assert_eq!(info.items_decommission_failed, 1); + assert_eq!(info.bytes_failed, 16); + } + + #[test] + fn test_track_decommission_current_object_rejects_missing_decommission_meta() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }], + ..Default::default() + }; + + let err = track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a") + .expect_err("missing decommission metadata should return explicit error"); + assert!( + err.to_string() + .contains("failed to track decommission current object: decommission metadata not initialized") + ); + } + + #[test] + fn test_track_decommission_current_object_updates_bucket_and_object() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo::default()), + }], + ..Default::default() + }; + + track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a").expect("valid state should track bucket/object"); + + let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist"); + assert_eq!(info.bucket, "bucket-a"); + assert_eq!(info.object, "object-a"); + } + + #[test] + fn test_resolve_decommission_update_after_result_passthrough_ok() { + let ok = resolve_decommission_update_after_result(Ok(true)).expect("ok value should pass through"); + assert!(ok); + } + + #[test] + fn test_resolve_decommission_update_after_result_wraps_error_context() { + let err = resolve_decommission_update_after_result(ensure_valid_decommission_pool_index(0, 0).map(|_| false)) + .expect_err("invalid argument should be wrapped with context"); + assert!(err.to_string().contains("decommission metadata update failed")); + assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools")); + } + + #[test] + fn test_resolve_decommission_preflight_heal_result_passthrough_ok() { + assert!(resolve_decommission_preflight_heal_result::<()>("bucket-a", Ok(())).is_ok()); + } + + #[test] + fn test_resolve_decommission_preflight_heal_result_wraps_error_context() { + let err = resolve_decommission_preflight_heal_result::<()>("bucket-a", Err(Error::SlowDown)) + .expect_err("heal failure should carry preflight context"); + assert!( + err.to_string() + .contains("decommission preflight heal failed for bucket bucket-a") + ); + } + + #[test] + fn test_resolve_decommission_bucket_done_save_result_passthrough_ok() { + assert!(resolve_decommission_bucket_done_save_result(Ok(()), 1, "bucket-a").is_ok()); + } + + #[test] + fn test_resolve_decommission_bucket_done_save_result_wraps_error_context() { + let err = resolve_decommission_bucket_done_save_result(Err(Error::SlowDown), 2, "bucket-a") + .expect_err("metadata save failure should carry pool/bucket context"); + assert!( + err.to_string() + .contains("decommission metadata save failed for pool 2 bucket bucket-a") + ); + } + + #[test] + fn test_resolve_decommission_optional_bucket_config_result_passthrough() { + let result = resolve_decommission_optional_bucket_config_result("bucket-a", "replication", Ok(42_u8)) + .expect("bucket config should pass through"); + assert_eq!(result, Some(42)); + } + + #[test] + fn test_resolve_decommission_optional_bucket_config_result_returns_none_for_missing_config() { + let result = + resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "versioning", Err(Error::ConfigNotFound)) + .expect("missing bucket config should map to None"); + assert!(result.is_none()); + } + + #[test] + fn test_resolve_decommission_optional_bucket_config_result_wraps_other_errors() { + let err = resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "replication", Err(Error::SlowDown)) + .expect_err("unexpected bucket config errors should be wrapped with context"); + assert!( + err.to_string() + .contains("decommission replication config load failed for bucket bucket-a") + ); + } + + #[test] + fn test_resolve_decommission_entry_cleanup_delete_result_passthrough_ok() { + assert!(resolve_decommission_entry_cleanup_delete_result(Ok(()), "bucket-a", "obj.txt").is_ok()); + } + + #[test] + fn test_resolve_decommission_entry_cleanup_delete_result_ignores_not_found() { + assert!(resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::FileNotFound), "bucket-a", "obj.txt").is_ok()); + } + + #[test] + fn test_resolve_decommission_entry_cleanup_delete_result_wraps_error_context() { + let err = resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::SlowDown), "bucket-a", "obj.txt") + .expect_err("cleanup delete failure should be wrapped with explicit context"); + assert!( + err.to_string() + .contains("decommission cleanup_delete_object failed for bucket-a/obj.txt") + ); + } + + #[test] + fn test_resolve_decommission_entry_reload_result_passthrough_ok() { + assert!(resolve_decommission_entry_reload_result(Ok(()), "bucket-a", "obj.txt").is_ok()); + } + + #[test] + fn test_resolve_decommission_entry_reload_result_wraps_error_context() { + let err = resolve_decommission_entry_reload_result(Err(Error::SlowDown), "bucket-a", "obj.txt") + .expect_err("reload failure should be wrapped with explicit context"); + assert!( + err.to_string() + .contains("decommission reload_pool_meta failed for bucket-a/obj.txt") + ); + } + + #[test] + fn test_resolve_decommission_terminal_mark_result_passthrough_ok() { + assert!(resolve_decommission_terminal_mark_result(Ok(()), "completed", "pool-a").is_ok()); + } + + #[test] + fn test_resolve_decommission_terminal_mark_result_wraps_error_context() { + let err = resolve_decommission_terminal_mark_result(Err(Error::SlowDown), "failed", "pool-a") + .expect_err("terminal mark failure should include stage and pool context"); + let message = err.to_string(); + assert!(message.contains("decommission terminal mark failed failed for pool pool-a")); + } + + #[test] + fn test_resolve_decommission_terminal_mark_after_error_result_passthrough_ok() { + assert!(resolve_decommission_terminal_mark_after_error_result(Ok(()), 3, &Error::SlowDown).is_ok()); + } + + #[test] + fn test_resolve_decommission_terminal_mark_after_error_result_wraps_error_context() { + let err = resolve_decommission_terminal_mark_after_error_result(Err(Error::OperationCanceled), 3, &Error::SlowDown) + .expect_err("terminal mark after-error failure should include both errors"); + let message = err.to_string(); + assert!(message.contains("decommission terminal mark failed after background error on pool 3")); + assert!(message.contains("mark error")); + } + + #[test] + fn test_resolve_decommission_spawn_failure_result_keeps_primary_without_rollback_error() { + let err = resolve_decommission_spawn_failure_result(Error::SlowDown, None); + assert!(matches!(err, Error::SlowDown)); + } + + #[test] + fn test_resolve_decommission_spawn_failure_result_wraps_rollback_error() { + let err = resolve_decommission_spawn_failure_result(Error::SlowDown, Some(Error::OperationCanceled)); + let message = err.to_string(); + assert!(message.contains("decommission spawn routines failed")); + assert!(message.contains("rollback failed")); + } + + #[test] + fn test_decommission_item_size_converts_positive_values() { + assert_eq!(decommission_item_size(42_i64), 42); + } + + #[test] + fn test_decommission_item_size_clamps_negative_values_to_zero() { + assert_eq!(decommission_item_size(-1_i64), 0); + } + + #[test] + fn test_new_multipart_abort_flag_defaults_to_abort_enabled() { + let flag = data_movement::new_multipart_abort_flag(); + assert!(data_movement::should_abort_multipart_upload(&flag)); + } + + #[test] + fn test_mark_multipart_upload_completed_disables_abort_cleanup() { + let flag = data_movement::new_multipart_abort_flag(); + data_movement::mark_multipart_upload_completed(&flag); + assert!(!data_movement::should_abort_multipart_upload(&flag)); + } + + #[test] + fn test_decode_part_index_returns_some_for_valid_payload() { + let mut index = Index::new(); + index.add(0, 0).expect("first index entry should be accepted"); + index + .add(2_097_152, 2_097_152) + .expect("second index entry should advance totals"); + + let encoded = index.into_vec(); + let decoded = data_movement::decode_part_index(Some(&encoded)).expect("valid index payload should decode"); + + assert_eq!(decoded.total_uncompressed, 2_097_152); + assert_eq!(decoded.total_compressed, 2_097_152); + } + + #[test] + fn test_with_decommission_entry_context_formats_stage_bucket_and_object() { + let err = with_decommission_entry_context("update_after", "bucket-a", "obj.txt", Error::SlowDown); + let message = err.to_string(); + assert!(message.contains("decommission entry update_after failed")); + assert!(message.contains("bucket bucket-a")); + assert!(message.contains("object obj.txt")); + } + + #[test] + fn test_load_decommission_entry_versions_wraps_parse_errors_with_context() { + let entry = MetaCacheEntry { + name: "obj.txt".to_string(), + metadata: vec![1, 2, 3], + cached: None, + reusable: false, + }; + + let err = load_decommission_entry_versions(&entry, "bucket-a", "check_after_decommission.file_info_versions") + .expect_err("invalid metadata should fail"); + let message = err.to_string(); + assert!(message.contains("decommission entry check_after_decommission.file_info_versions failed")); + assert!(message.contains("bucket bucket-a")); + assert!(message.contains("object obj.txt")); + } + + #[test] + fn test_resolve_decommission_check_after_list_result_prefers_entry_error() { + let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), Some(Error::SlowDown)) + .expect_err("entry error should win over cancellation"); + assert!(matches!(err, Error::SlowDown)); + } + + #[test] + fn test_resolve_decommission_check_after_list_result_returns_list_result_without_entry_error() { + let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), None) + .expect_err("list result should be preserved without entry error"); + assert!(matches!(err, Error::OperationCanceled)); + } + + #[test] + fn test_resolve_decommission_pool_meta_reload_result_passthrough_ok() { + assert!(resolve_decommission_pool_meta_reload_result(Ok(()), "start_decommission").is_ok()); + } + + #[test] + fn test_resolve_decommission_pool_meta_reload_result_wraps_error_context() { + let err = resolve_decommission_pool_meta_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3") + .expect_err("reload failure should be wrapped with stage context"); + let message = err.to_string(); + assert!(message.contains("decommission pool meta reload failed during decommission_failed for pool 3")); + assert!(message.contains(Error::SlowDown.to_string().as_str())); + } + + #[test] + fn test_resolve_decommission_listing_worker_result_passthrough_ok() { + assert!(resolve_decommission_listing_worker_result(2, Ok(())).is_ok()); + } + + #[tokio::test] + async fn test_resolve_decommission_listing_worker_result_wraps_join_error_context() { + let join_error = tokio::spawn(async { + panic!("listing worker panic"); + }) + .await + .expect_err("panic task should return JoinError"); + + let err = resolve_decommission_listing_worker_result(4, Err(join_error)) + .expect_err("join error should be wrapped with context"); + let message = err.to_string(); + assert!(message.contains("decommission listing worker 4 task join error")); + assert!(message.contains("panic")); + } + + #[test] + fn test_should_count_decommission_version_complete_for_cleanup_safe_ignored_result() { + assert!(should_count_decommission_version_complete(true, true, false)); + } + + #[test] + fn test_should_count_decommission_version_complete_rejects_skip_only_ignored_result() { + assert!(!should_count_decommission_version_complete(true, false, false)); + } + + #[test] + fn test_should_count_decommission_version_complete_for_completed_result() { + assert!(should_count_decommission_version_complete(false, false, false)); + } + + #[test] + fn test_should_count_decommission_version_complete_rejects_failed_result() { + assert!(!should_count_decommission_version_complete(false, false, true)); + } + + #[test] + fn test_should_cleanup_decommission_source_entry_accepts_all_versions_completed() { + assert!(should_cleanup_decommission_source_entry(3, 3, 0)); + } + + #[test] + fn test_should_cleanup_decommission_source_entry_rejects_versions_only_expired_by_lifecycle() { + assert!(!should_cleanup_decommission_source_entry(2, 3, 1)); + } + + #[tokio::test] + async fn test_pool_meta_update_after_rejects_out_of_range_index() { + let mut meta = PoolMeta::default(); + let err = meta + .update_after(1, Vec::new(), Duration::seconds(1)) + .await + .expect_err("out-of-range index should fail"); + assert!(err.to_string().contains("invalid decommission pool index 1 for 0 pools")); + } + + #[tokio::test] + async fn test_pool_meta_update_after_rejects_when_decommission_missing() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }], + ..Default::default() + }; + + let err = meta + .update_after(0, Vec::new(), Duration::seconds(1)) + .await + .expect_err("pool without decommission should fail"); + assert!( + err.to_string() + .contains("failed to update decommission metadata timestamp: decommission metadata not initialized") + ); + } + + #[test] + fn test_ensure_decommission_not_rebalancing_rejects_running_rebalance() { + let err = ensure_decommission_not_rebalancing(true).expect_err("rebalance running should be rejected"); + assert!(matches!(err, Error::RebalanceAlreadyRunning)); + } + + #[test] + fn test_ensure_decommission_not_rebalancing_allows_idle() { + assert!(ensure_decommission_not_rebalancing(false).is_ok()); + } + + #[test] + fn test_is_decommission_active_true_only_when_not_terminal() { + assert!(is_decommission_active(false, false, false)); + assert!(!is_decommission_active(true, false, false)); + assert!(!is_decommission_active(false, true, false)); + assert!(!is_decommission_active(false, false, true)); + } + + #[test] + fn test_ensure_decommission_start_allowed_rejects_missing_pool() { + let err = ensure_decommission_start_allowed(false, false).expect_err("missing pool should be invalid"); + assert!( + err.to_string() + .contains("failed to start decommission: target pool was not found") + ); + } + + #[test] + fn test_ensure_decommission_start_allowed_rejects_running_state() { + let err = ensure_decommission_start_allowed(true, true).expect_err("active decommission should be rejected"); + assert!(matches!(err, Error::DecommissionAlreadyRunning)); + } + + #[test] + fn test_ensure_decommission_start_allowed_allows_terminal_state() { + assert!(ensure_decommission_start_allowed(true, false).is_ok()); + } + + #[test] + fn test_decommission_start_guard_state_reports_missing_pool() { + assert_eq!(decommission_start_guard_state(None), (false, false)); + } + + #[test] + fn test_decommission_start_guard_state_reports_idle_pool_without_decommission_info() { + let pool = PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }; + + assert_eq!(decommission_start_guard_state(Some(&pool)), (true, false)); + } + + #[test] + fn test_decommission_start_guard_state_reports_active_pool_when_not_terminal() { + let pool = PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + complete: false, + failed: false, + canceled: false, + ..Default::default() + }), + }; + + assert_eq!(decommission_start_guard_state(Some(&pool)), (true, true)); + } + + #[test] + fn test_decommission_start_guard_state_reports_terminal_pool_as_not_active() { + let pool = PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + complete: false, + failed: false, + canceled: true, + ..Default::default() + }), + }; + + assert_eq!(decommission_start_guard_state(Some(&pool)), (true, false)); + } + + #[test] + fn test_ensure_valid_decommission_pool_index_accepts_in_range_index() { + assert!(ensure_valid_decommission_pool_index(4, 3).is_ok()); + } + + #[test] + fn test_ensure_valid_decommission_pool_index_rejects_out_of_range_index() { + let err = ensure_valid_decommission_pool_index(2, 2).expect_err("out-of-range index should fail"); + assert!(err.to_string().contains("invalid decommission pool index 2 for 2 pools")); + } + + #[test] + fn test_ensure_valid_decommission_pool_index_rejects_when_pool_count_zero() { + let err = ensure_valid_decommission_pool_index(0, 0).expect_err("empty pool list should reject all indices"); + assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools")); + } + + #[test] + fn test_classify_decommission_terminal_state_completed_when_no_failures() { + assert_eq!(classify_decommission_terminal_state(false), DecommissionTerminalState::Completed); + } + + #[test] + fn test_classify_decommission_terminal_state_failed_when_failures_present() { + assert_eq!(classify_decommission_terminal_state(true), DecommissionTerminalState::Failed); + } + + #[test] + fn test_should_preserve_decommission_canceled_state_when_meta_canceled() { + assert!(should_preserve_decommission_canceled_state(true, false)); + } + + #[test] + fn test_should_preserve_decommission_canceled_state_when_signal_canceled() { + assert!(should_preserve_decommission_canceled_state(false, true)); + } + + #[test] + fn test_should_preserve_decommission_canceled_state_when_not_canceled() { + assert!(!should_preserve_decommission_canceled_state(false, false)); + } + + #[test] + fn test_decommission_cancel_signal_result_returns_err_when_canceled() { + let err = decommission_cancel_signal_result(true).expect_err("canceled signal should return operation-canceled"); + assert!(matches!(err, Error::OperationCanceled)); + } + + #[test] + fn test_decommission_cancel_signal_result_returns_ok_when_not_canceled() { + assert!(decommission_cancel_signal_result(false).is_ok()); + } + + #[test] + fn test_ensure_decommission_cancel_allowed_rejects_missing_pool() { + let err = ensure_decommission_cancel_allowed(false, false, false).expect_err("missing pool should be invalid"); + assert!( + err.to_string() + .contains("failed to cancel decommission: target pool was not found") + ); + } + + #[test] + fn test_is_decommission_cancel_terminal_true_when_completed() { + assert!(is_decommission_cancel_terminal(true, false, false)); + } + + #[test] + fn test_is_decommission_cancel_terminal_true_when_failed() { + assert!(is_decommission_cancel_terminal(false, true, false)); + } + + #[test] + fn test_is_decommission_cancel_terminal_true_when_canceled() { + assert!(is_decommission_cancel_terminal(false, false, true)); + } + + #[test] + fn test_is_decommission_cancel_terminal_false_when_active() { + assert!(!is_decommission_cancel_terminal(false, false, false)); + } + + #[test] + fn test_ensure_decommission_cancel_allowed_rejects_not_started() { + let err = + ensure_decommission_cancel_allowed(true, false, false).expect_err("not-started decommission should be rejected"); + assert!(matches!(err, Error::DecommissionNotStarted)); + } + + #[test] + fn test_ensure_decommission_cancel_allowed_rejects_terminal() { + let err = ensure_decommission_cancel_allowed(true, true, true).expect_err("terminal decommission should be rejected"); + assert!(matches!(err, Error::DecommissionNotStarted)); + } + + #[test] + fn test_ensure_decommission_cancel_allowed_allows_active() { + assert!(ensure_decommission_cancel_allowed(true, true, false).is_ok()); + } + + #[test] + fn test_contextualized_decommission_terminal_operation_supported_rejects_single_pool() { + let err = ensure_decommission_terminal_operation_supported(true, "complete decommission") + .expect_err("single-pool decommission terminal operations should be rejected"); + assert!( + err.to_string() + .contains("failed to complete decommission: single pool deployments do not support decommission") + ); + } + + #[test] + fn test_contextualized_decommission_terminal_operation_supported_allows_multi_pool() { + assert!(ensure_decommission_terminal_operation_supported(false, "mark decommission failed").is_ok()); + } + + #[test] + fn test_contextualized_decommission_start_request_rejects_empty_indices() { + let err = validate_start_decommission_request(&[], false).expect_err("empty decommission target list should be rejected"); + assert!( + err.to_string() + .contains("failed to start decommission: no target pools were provided") + ); + } + + #[test] + fn test_contextualized_decommission_start_request_rejects_single_pool() { + let err = validate_start_decommission_request(&[0], true) + .expect_err("single-pool deployments should reject decommission start"); + assert!( + err.to_string() + .contains("failed to start decommission: single pool deployments do not support decommission") + ); + } + + #[test] + fn test_contextualized_decommission_start_request_allows_non_empty_multi_pool() { + assert!(validate_start_decommission_request(&[0, 1], false).is_ok()); + } + + #[test] + fn test_contextualized_decommission_listing_disks_available_rejects_empty_set() { + let err = ensure_decommission_listing_disks_available(false, "bucket-a") + .expect_err("missing online disks should be reported with bucket context"); + assert!( + err.to_string() + .contains("failed to list objects to decommission for bucket bucket-a: no disks available") + ); + } + + #[test] + fn test_contextualized_decommission_listing_disks_available_allows_online_disks() { + assert!(ensure_decommission_listing_disks_available(true, "bucket-a").is_ok()); + } + + #[test] + fn test_require_decommission_store_returns_value_when_present() { + let store = require_decommission_store(Some(7_u8), "start decommission").expect("present store should be returned"); + assert_eq!(store, 7); + } + + #[test] + fn test_require_decommission_store_returns_error_when_missing() { + let err = require_decommission_store::(None, "start decommission").expect_err("missing store should return error"); + assert!( + err.to_string() + .contains("failed to start decommission: store not initialized") + ); + } + + #[test] + fn test_bind_decommission_cancelers_binds_existing_slots_only() { + let parent = CancellationToken::new(); + let mut cancelers = vec![None, None]; + + let bound = bind_decommission_cancelers(&[0, 3, 1], &parent, cancelers.as_mut_slice()); + + assert_eq!(bound.len(), 2); + assert_eq!(bound[0].0, 0); + assert_eq!(bound[1].0, 1); + assert!(cancelers[0].is_some()); + assert!(cancelers[1].is_some()); + } + + #[test] + fn test_bind_decommission_cancelers_child_tokens_follow_parent_cancel() { + let parent = CancellationToken::new(); + let mut cancelers = vec![None]; + + let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice()); + assert_eq!(bound.len(), 1); + assert!(!bound[0].1.is_cancelled()); + + parent.cancel(); + assert!(bound[0].1.is_cancelled()); + } + + #[test] + fn test_bind_decommission_cancelers_replaces_existing_slot() { + let parent = CancellationToken::new(); + let existing = CancellationToken::new(); + let mut cancelers = vec![Some(existing.clone())]; + + let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice()); + + assert_eq!(bound.len(), 1); + assert_eq!(bound[0].0, 0); + assert!(existing.is_cancelled()); + let replacement = cancelers[0].as_ref().expect("replacement token should be stored"); + assert!(!replacement.is_cancelled()); + parent.cancel(); + assert!(replacement.is_cancelled()); + } + + #[test] + fn test_take_decommission_canceler_takes_and_clears_slot() { + let token = CancellationToken::new(); + let mut cancelers = vec![Some(token.clone())]; + + let taken = take_decommission_canceler(cancelers.as_mut_slice(), 0); + assert!(taken.is_some()); + assert!(cancelers[0].is_none()); + } + + #[test] + fn test_take_decommission_canceler_returns_none_for_missing_slot() { + let mut cancelers: Vec> = Vec::new(); + assert!(take_decommission_canceler(cancelers.as_mut_slice(), 0).is_none()); + } + + #[test] + fn test_has_active_decommission_canceler_true_when_any_slot_present() { + let cancelers = vec![None, Some(CancellationToken::new())]; + assert!(has_active_decommission_canceler(cancelers.as_slice())); + } + + #[test] + fn test_has_active_decommission_canceler_false_when_all_empty() { + let cancelers = vec![None, None]; + assert!(!has_active_decommission_canceler(cancelers.as_slice())); + } + + #[test] + fn test_cancel_decommission_canceler_cancels_when_present() { + let token = CancellationToken::new(); + let canceled = cancel_decommission_canceler(Some(token.clone())); + + assert!(canceled); + assert!(token.is_cancelled()); + } + + #[test] + fn test_cancel_decommission_canceler_returns_false_when_missing() { + assert!(!cancel_decommission_canceler(None)); + } + + #[test] + fn test_ensure_decommission_routines_scheduled_accepts_positive_bound_count() { + assert!(super::ensure_decommission_routines_scheduled(2, 2).is_ok()); + } + + #[test] + fn test_ensure_decommission_routines_scheduled_rejects_zero_bound_count() { + let err = super::ensure_decommission_routines_scheduled(0, 1).expect_err("zero bound count should be rejected"); + assert!( + err.to_string() + .contains("failed to start decommission routines: scheduled 0 of 1 expected workers") + ); + } + + #[test] + fn test_ensure_decommission_routines_scheduled_rejects_partial_binding() { + let err = super::ensure_decommission_routines_scheduled(1, 2).expect_err("partial binding should be rejected"); + assert!( + err.to_string() + .contains("failed to start decommission routines: scheduled 1 of 2 expected workers") + ); + } +} diff --git a/crates/ecstore/src/rebalance.rs b/crates/ecstore/src/rebalance.rs index 79621de0b..617142740 100644 --- a/crates/ecstore/src/rebalance.rs +++ b/crates/ecstore/src/rebalance.rs @@ -15,27 +15,27 @@ use crate::StorageAPI; use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; use crate::config::com::{read_config_with_metadata, save_config_with_opts}; +use crate::data_movement; +use crate::data_usage::DATA_USAGE_CACHE_NAME; use crate::disk::error::DiskError; use crate::error::{Error, Result}; -use crate::error::{is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found}; +use crate::error::{ + is_err_data_movement_overwrite, is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, +}; use crate::global::get_global_endpoints; use crate::pools::ListCallback; use crate::set_disk::SetDisks; use crate::store::ECStore; -use crate::store_api::{ - CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, -}; +use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions}; use http::HeaderMap; -use rustfs_common::defer; use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; -use rustfs_rio::{HashReader, WarpReader}; use rustfs_utils::path::encode_dir_object; use serde::{Deserialize, Serialize}; use std::fmt; +use std::future::Future; use std::io::Cursor; use std::sync::Arc; use time::OffsetDateTime; -use tokio::io::{AsyncReadExt, BufReader}; use tokio::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tracing::{error, info}; @@ -78,12 +78,18 @@ impl RebalanceStats { } self.num_versions += 1; - let on_disk_size = if !fi.deleted { - fi.size * (fi.erasure.data_blocks + fi.erasure.parity_blocks) as i64 / fi.erasure.data_blocks as i64 - } else { + let on_disk_size = if fi.deleted || fi.erasure.data_blocks == 0 || fi.size <= 0 { 0 + } else { + let data_blocks = fi.erasure.data_blocks as i64; + let total_blocks = fi.erasure.data_blocks.saturating_add(fi.erasure.parity_blocks) as i64; + fi.size + .saturating_mul(total_blocks) + .checked_div(data_blocks) + .unwrap_or(0) + .max(0) as u64 }; - self.bytes += on_disk_size as u64; + self.bytes = self.bytes.saturating_add(on_disk_size); self.bucket = bucket; self.object = fi.name.clone(); } @@ -91,6 +97,283 @@ impl RebalanceStats { pub type RStats = Vec>; +#[derive(Debug, Default)] +struct RebalanceBucketConfigs { + lifecycle_config: Option, + lock_retention: Option, + replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>, +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct MigrationVersionResult { + pub moved: bool, + pub ignored: bool, + pub cleanup_ignored: bool, + pub failed: bool, + pub error: Option, +} + +fn rebalance_delete_marker_opts(version: &FileInfo, version_id: Option, src_pool_idx: usize) -> ObjectOptions { + ObjectOptions { + versioned: true, + version_id, + mod_time: version.mod_time, + src_pool_idx, + data_movement: true, + delete_marker: true, + skip_decommissioned: true, + delete_replication: version.replication_state_internal.clone(), + ..Default::default() + } +} + +fn rebalance_remote_tiered_opts(version: &FileInfo, version_id: Option, src_pool_idx: usize) -> ObjectOptions { + ObjectOptions { + versioned: version_id.is_some(), + version_id, + mod_time: version.mod_time, + user_defined: version.metadata.clone(), + src_pool_idx, + data_movement: true, + ..Default::default() + } +} + +#[async_trait::async_trait] +pub(crate) trait MigrationBackend: Send + Sync { + async fn get_object_reader_for_migration( + &self, + bucket: &str, + object: &str, + range: Option, + h: HeaderMap, + opts: &ObjectOptions, + ) -> Result; + + async fn delete_object_for_migration(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result; + + async fn move_remote_version_for_migration( + &self, + bucket: &str, + object: &str, + fi: &FileInfo, + opts: &ObjectOptions, + ) -> Result<()>; +} + +#[async_trait::async_trait] +impl MigrationBackend for SetDisks { + async fn get_object_reader_for_migration( + &self, + bucket: &str, + object: &str, + range: Option, + h: HeaderMap, + opts: &ObjectOptions, + ) -> Result { + self.get_object_reader(bucket, object, range, h, opts).await + } + + async fn delete_object_for_migration(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result { + self.delete_object(bucket, object, opts).await + } + + async fn move_remote_version_for_migration( + &self, + bucket: &str, + object: &str, + fi: &FileInfo, + opts: &ObjectOptions, + ) -> Result<()> { + self.decommission_tiered_object(bucket, object, fi, opts).await + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn migrate_entry_version( + set: &Backend, + bucket: String, + pool_index: usize, + version: &FileInfo, + version_id: Option, + max_attempts: usize, + ignore_data_usage_cache: bool, + mut transfer: F, +) -> MigrationVersionResult +where + Backend: MigrationBackend + ?Sized, + F: FnMut(usize, String, GetObjectReader) -> Fut + Send, + Fut: Future> + Send, +{ + let max_attempts = max_attempts.max(1); + + if ignore_data_usage_cache && bucket == crate::disk::RUSTFS_META_BUCKET && version.name.contains(DATA_USAGE_CACHE_NAME) { + return MigrationVersionResult { + moved: false, + ignored: true, + cleanup_ignored: false, + failed: false, + error: None, + }; + } + + if version.is_remote() { + if let Err(err) = set + .move_remote_version_for_migration( + &bucket, + &version.name, + version, + &rebalance_remote_tiered_opts(version, version_id, pool_index), + ) + .await + { + if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { + return MigrationVersionResult { + moved: false, + ignored: true, + cleanup_ignored: true, + failed: false, + error: None, + }; + } + + return MigrationVersionResult { + moved: false, + ignored: false, + cleanup_ignored: false, + failed: true, + error: Some(err), + }; + } + + return MigrationVersionResult { + moved: true, + ignored: false, + cleanup_ignored: false, + failed: false, + error: None, + }; + } + + if version.deleted { + if let Err(err) = set + .delete_object_for_migration(&bucket, &version.name, rebalance_delete_marker_opts(version, version_id, pool_index)) + .await + { + if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { + return MigrationVersionResult { + moved: false, + ignored: true, + cleanup_ignored: true, + failed: false, + error: None, + }; + } + + return MigrationVersionResult { + moved: false, + ignored: false, + cleanup_ignored: false, + failed: true, + error: Some(err), + }; + } + + return MigrationVersionResult { + moved: true, + ignored: false, + cleanup_ignored: false, + failed: false, + error: None, + }; + } + + let mut last_error: Option = None; + for attempt in 0..max_attempts { + let rd = match set + .get_object_reader_for_migration( + &bucket, + &encode_dir_object(&version.name), + None, + HeaderMap::new(), + &ObjectOptions { + version_id: version_id.clone(), + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(rd) => rd, + Err(err) => { + if is_err_object_not_found(&err) || is_err_version_not_found(&err) { + return MigrationVersionResult { + moved: false, + ignored: true, + cleanup_ignored: true, + failed: false, + error: None, + }; + } + + last_error = Some(err); + if attempt + 1 >= max_attempts { + return MigrationVersionResult { + moved: false, + ignored: false, + cleanup_ignored: false, + failed: true, + error: last_error, + }; + } + + continue; + } + }; + + if let Err(err) = transfer(pool_index, bucket.clone(), rd).await { + if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { + return MigrationVersionResult { + moved: false, + ignored: true, + cleanup_ignored: true, + failed: false, + error: None, + }; + } + + last_error = Some(err); + if attempt + 1 >= max_attempts { + return MigrationVersionResult { + moved: false, + ignored: false, + cleanup_ignored: false, + failed: true, + error: last_error, + }; + } + + continue; + } + + return MigrationVersionResult { + moved: true, + ignored: false, + cleanup_ignored: false, + failed: false, + error: None, + }; + } + + MigrationVersionResult { + moved: false, + ignored: false, + cleanup_ignored: false, + failed: true, + error: last_error, + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] pub enum RebalStatus { #[default] @@ -139,6 +422,8 @@ pub struct RebalanceInfo { pub start_time: Option, // Time at which rebalance-start was issued #[serde(rename = "stopTs")] pub end_time: Option, // Time at which rebalance operation completed or rebalance-stop was called + #[serde(rename = "err")] + pub last_error: Option, // Last rebalance error message #[serde(rename = "status")] pub status: RebalStatus, // Current state of rebalance operation } @@ -166,6 +451,38 @@ pub struct RebalanceMeta { pub pool_stats: Vec, // Per-pool rebalance stats keyed by pool index } +fn is_rebalance_pool_started(pool_stat: &RebalanceStats) -> bool { + pool_stat.participating && pool_stat.info.status == RebalStatus::Started +} + +fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool { + if meta.stopped_at.is_some() { + return false; + } + + meta.pool_stats.iter().any(is_rebalance_pool_started) +} + +fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool { + is_rebalance_in_progress(meta) +} + +fn first_rebalance_bucket(pool_stat: &RebalanceStats) -> Option { + pool_stat.buckets.first().cloned() +} + +fn rebalance_meta_load_no_data_error() -> Error { + Error::other("rebalance metadata load failed: metadata payload is too short") +} + +fn rebalance_meta_load_unknown_format_error(fmt: u16) -> Error { + Error::other(format!("rebalance metadata load failed: unknown format {fmt}")) +} + +fn rebalance_meta_load_unknown_version_error(ver: u16) -> Error { + Error::other(format!("rebalance metadata load failed: unknown version {ver}")) +} + impl RebalanceMeta { pub fn new() -> Self { Self::default() @@ -181,17 +498,17 @@ impl RebalanceMeta { return Ok(()); } if data.len() <= 4 { - return Err(Error::other("rebalanceMeta load_with_opts: no data")); + return Err(rebalance_meta_load_no_data_error()); } // Read header match u16::from_le_bytes([data[0], data[1]]) { REBAL_META_FMT => {} - fmt => return Err(Error::other(format!("rebalanceMeta load_with_opts: unknown format: {fmt}"))), + fmt => return Err(rebalance_meta_load_unknown_format_error(fmt)), } match u16::from_le_bytes([data[2], data[3]]) { REBAL_META_VER => {} - ver => return Err(Error::other(format!("rebalanceMeta load_with_opts: unknown version: {ver}"))), + ver => return Err(rebalance_meta_load_unknown_version_error(ver)), } let meta: Self = rmp_serde::from_read(Cursor::new(&data[4..]))?; @@ -233,33 +550,23 @@ impl ECStore { pub async fn load_rebalance_meta(&self) -> Result<()> { let mut meta = RebalanceMeta::new(); info!("rebalanceMeta: store load rebalance meta"); - match meta.load(self.pools[0].clone()).await { - Ok(_) => { - info!("rebalanceMeta: rebalance meta loaded0"); - { - let mut rebalance_meta = self.rebalance_meta.write().await; + let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?; + if resolve_rebalance_meta_load_result(meta.load(pool).await)? { + info!("rebalanceMeta: rebalance meta loaded0"); + { + let mut rebalance_meta = self.rebalance_meta.write().await; - *rebalance_meta = Some(meta); + *rebalance_meta = Some(meta); - drop(rebalance_meta); - } - - info!("rebalanceMeta: rebalance meta loaded1"); - - if let Err(err) = self.update_rebalance_stats().await { - error!("Failed to update rebalance stats: {}", err); - } else { - info!("rebalanceMeta: rebalance meta loaded2"); - } + drop(rebalance_meta); } - Err(err) => { - if err != Error::ConfigNotFound { - error!("rebalanceMeta: load rebalance meta err {:?}", &err); - return Err(err); - } - info!("rebalanceMeta: not found, rebalance not started"); - } + info!("rebalanceMeta: rebalance meta loaded1"); + + resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?; + info!("rebalanceMeta: rebalance meta loaded2"); + } else { + info!("rebalanceMeta: not found, rebalance not started"); } Ok(()) @@ -271,7 +578,7 @@ impl ECStore { let pool_stats = { let rebalance_meta = self.rebalance_meta.read().await; - rebalance_meta.as_ref().map(|v| v.pool_stats.clone()).unwrap_or_default() + clone_rebalance_pool_stats(rebalance_meta.as_ref())? }; info!("update_rebalance_stats: pool_stats: {:?}", &pool_stats); @@ -294,7 +601,8 @@ impl ECStore { let rebalance_meta = self.rebalance_meta.read().await; if let Some(meta) = rebalance_meta.as_ref() { - meta.save(self.pools[0].clone()).await?; + let pool = clone_first_arc(&self.pools, "update_rebalance_stats: no pools available")?; + resolve_rebalance_meta_save_result(meta.save(pool).await, "update_rebalance_stats")?; } } @@ -330,7 +638,7 @@ impl ECStore { disk_stats[disk.pool_index as usize].available_space += disk.available_space; } - let percent_free_goal = total_free as f64 / total_cap as f64; + let percent_free_goal = percent_free_ratio(total_free, total_cap); let mut pool_stats = Vec::with_capacity(self.pools.len()); @@ -345,7 +653,7 @@ impl ECStore { ..Default::default() }; - if (disk_stat.available_space as f64 / disk_stat.total_space as f64) < percent_free_goal { + if should_pool_participate(disk_stat.available_space, disk_stat.total_space, percent_free_goal) { pool_stat.participating = true; pool_stat.info = RebalanceInfo { start_time: Some(now), @@ -364,7 +672,8 @@ impl ECStore { ..Default::default() }; - meta.save(self.pools[0].clone()).await?; + let pool = clone_first_arc(&self.pools, "init_rebalance_meta: no pools available")?; + resolve_rebalance_meta_save_result(meta.save(pool).await, "init_rebalance_meta")?; info!("init_rebalance_meta: rebalance meta saved"); @@ -396,65 +705,18 @@ impl ECStore { info!("next_rebal_bucket: pool_index: {}", pool_index); let rebalance_meta = self.rebalance_meta.read().await; info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta); - if let Some(meta) = rebalance_meta.as_ref() - && let Some(pool_stat) = meta.pool_stats.get(pool_index) - { - if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating { - info!("next_rebal_bucket: pool_index: {} completed or not participating", pool_index); - return Ok(None); - } - - if pool_stat.buckets.is_empty() { - info!("next_rebal_bucket: pool_index: {} buckets is empty", pool_index); - return Ok(None); - } - info!("next_rebal_bucket: pool_index: {} bucket: {}", pool_index, pool_stat.buckets[0]); - return Ok(Some(pool_stat.buckets[0].clone())); - } - - info!("next_rebal_bucket: pool_index: {} None", pool_index); - Ok(None) + resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index) } #[tracing::instrument(skip(self))] pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String) -> Result<()> { let mut rebalance_meta = self.rebalance_meta.write().await; - if let Some(meta) = rebalance_meta.as_mut() - && let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) - { - info!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets); - - // Use retain to filter out buckets slated for removal - let mut found = false; - pool_stat.buckets.retain(|b| { - if b.as_str() == bucket.as_str() { - found = true; - pool_stat.rebalanced_buckets.push(b.clone()); - false // Remove this element - } else { - true // Keep this element - } - }); - - if found { - info!("bucket_rebalance_done: bucket {} rebalanced", &bucket); - return Ok(()); - } else { - info!("bucket_rebalance_done: bucket {} not found", bucket); - } - } - info!("bucket_rebalance_done: bucket {} not found", bucket); - Ok(()) + mark_rebalance_bucket_done(rebalance_meta.as_mut(), pool_index, &bucket) } pub async fn is_rebalance_started(&self) -> bool { let rebalance_meta = self.rebalance_meta.read().await; - if let Some(ref meta) = *rebalance_meta { - if meta.stopped_at.is_some() { - info!("is_rebalance_started: rebalance stopped"); - return false; - } - + if let Some(meta) = rebalance_meta.as_ref() { meta.pool_stats.iter().enumerate().for_each(|(i, v)| { info!( "is_rebalance_started: pool_index: {}, participating: {:?}, status: {:?}", @@ -462,11 +724,8 @@ impl ECStore { ); }); - if meta - .pool_stats - .iter() - .any(|v| v.participating && v.info.status != RebalStatus::Completed) - { + let started = is_rebalance_conflicting_with_decommission(meta); + if started { info!("is_rebalance_started: rebalance started"); return true; } @@ -476,6 +735,13 @@ impl ECStore { false } + pub async fn is_rebalance_conflicting_with_decommission(&self) -> bool { + let rebalance_meta = self.rebalance_meta.read().await; + rebalance_meta + .as_ref() + .is_some_and(is_rebalance_conflicting_with_decommission) + } + pub async fn is_pool_rebalancing(&self, pool_index: usize) -> bool { let rebalance_meta = self.rebalance_meta.read().await; if let Some(ref meta) = *rebalance_meta { @@ -493,19 +759,23 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn stop_rebalance(self: &Arc) -> Result<()> { - let rebalance_meta = self.rebalance_meta.read().await; - if let Some(meta) = rebalance_meta.as_ref() - && let Some(cancel_tx) = meta.cancel.as_ref() - { - cancel_tx.cancel(); + let meta_to_save = { + let mut rebalance_meta = self.rebalance_meta.write().await; + stop_rebalance_meta_snapshot(rebalance_meta.as_mut(), OffsetDateTime::now_utc()) + }; + + if let Some(meta_to_save) = meta_to_save { + let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?; + resolve_rebalance_meta_save_result(meta_to_save.save(pool).await, "stop_rebalance")?; } Ok(()) } #[tracing::instrument(skip_all)] - pub async fn start_rebalance(self: &Arc) { + pub async fn start_rebalance(self: &Arc) -> Result<()> { info!("start_rebalance: start rebalance"); + let decommission_running = self.is_decommission_running().await; // let rebalance_meta = self.rebalance_meta.read().await; let cancel_tx = CancellationToken::new(); @@ -513,40 +783,25 @@ impl ECStore { { let mut rebalance_meta = self.rebalance_meta.write().await; + validate_start_rebalance_state(decommission_running, rebalance_meta.is_some())?; - if let Some(meta) = rebalance_meta.as_mut() { - meta.cancel = Some(cancel_tx) - } else { - info!("start_rebalance: rebalance_meta is None exit"); - return; + let Some(meta) = rebalance_meta.as_mut() else { + return Err(Error::ConfigNotFound); + }; + if should_skip_start_rebalance(meta.cancel.is_some(), is_rebalance_in_progress(meta)) { + info!("start_rebalance: already in progress, skip duplicate start"); + return Ok(()); } + meta.cancel = Some(cancel_tx); drop(rebalance_meta); } - let participants = { - if let Some(ref meta) = *self.rebalance_meta.read().await { - // if meta.stopped_at.is_some() { - // warn!("start_rebalance: rebalance already stopped exit"); - // return; - // } - - let mut participants = vec![false; meta.pool_stats.len()]; - for (i, pool_stat) in meta.pool_stats.iter().enumerate() { - info!("start_rebalance: pool {} status: {:?}", i, pool_stat.info.status); - if pool_stat.info.status != RebalStatus::Started { - info!("start_rebalance: pool {} not started, skipping", i); - continue; - } - - info!("start_rebalance: pool {} participating: {:?}", i, pool_stat.participating); - participants[i] = pool_stat.participating; - } - participants - } else { - info!("start_rebalance:2 rebalance_meta is None exit"); - Vec::new() - } + let participants = if let Some(ref meta) = *self.rebalance_meta.read().await { + resolve_rebalance_participants(meta.pool_stats.as_slice(), self.pools.len()) + } else { + info!("start_rebalance:2 rebalance_meta is None exit"); + Vec::new() }; for (idx, participating) in participants.iter().enumerate() { @@ -579,10 +834,13 @@ impl ECStore { } info!("start_rebalance: rebalance started done"); + Ok(()) } #[tracing::instrument(skip(self, rx))] async fn rebalance_buckets(self: &Arc, rx: CancellationToken, pool_index: usize) -> Result<()> { + ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?; + let (done_tx, mut done_rx) = tokio::sync::mpsc::channel::>(1); // Save rebalance metadata periodically @@ -594,41 +852,35 @@ impl ECStore { loop { tokio::select! { - // TODO: cancel rebalance - Some(result) = done_rx.recv() => { + result = done_rx.recv() => { quit = true; let now = OffsetDateTime::now_utc(); - - let state = match result { - Ok(_) => { - info!("rebalance_buckets: completed"); - msg = format!("Rebalance completed at {now:?}"); - RebalStatus::Completed}, - Err(err) => { - info!("rebalance_buckets: error: {:?}", err); - // TODO: check stop - if err.to_string().contains("canceled") { - msg = format!("Rebalance stopped at {now:?}"); - RebalStatus::Stopped + let terminal_event = classify_rebalance_terminal_event(result, now); + msg = terminal_event.message().to_string(); + let mut rebalance_meta = store.rebalance_meta.write().await; + if let Some(meta) = rebalance_meta.as_mut() { + let meta_stopped = meta.stopped_at.is_some(); + if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { + if should_preserve_rebalance_stopped_state( + meta_stopped, + pool_stat.info.status, + &terminal_event, + ) { + info!( + "rebalance_buckets: preserving stopped status for pool {}", + pool_index + ); } else { - msg = format!("Rebalance stopped at {now:?} with err {err:?}"); - RebalStatus::Failed + apply_rebalance_terminal_event( + &mut pool_stat.info.status, + &mut pool_stat.info.end_time, + &mut pool_stat.info.last_error, + terminal_event, + now, + ); } } - }; - - { - info!("rebalance_buckets: save rebalance meta, pool_index: {}, state: {:?}", pool_index, state); - let mut rebalance_meta = store.rebalance_meta.write().await; - - if let Some(rbm) = rebalance_meta.as_mut() { - info!("rebalance_buckets: save rebalance meta2, pool_index: {}, state: {:?}", pool_index, state); - rbm.pool_stats[pool_index].info.status = state; - rbm.pool_stats[pool_index].info.end_time = Some(now); - } } - - } _ = timer.tick() => { let now = OffsetDateTime::now_utc(); @@ -637,14 +889,18 @@ impl ECStore { } if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await { - error!("{} err: {:?}", msg, err); + let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}")); + error!("{} err: {:?}", msg, wrapped); + if quit { + return Err(wrapped); + } } else { info!(msg); } if quit { info!("{}: exiting save_task", msg); - return; + return Ok(()); } timer.reset(); @@ -652,29 +908,56 @@ impl ECStore { }); info!("Pool {} rebalancing is started", pool_index); + let mut final_result: Result<()> = Ok(()); loop { if rx.is_cancelled() { info!("Pool {} rebalancing is stopped", pool_index); - done_tx.send(Err(Error::other("rebalance stopped canceled"))).await.ok(); + let err = Error::OperationCanceled; + final_result = Err(resolve_rebalance_terminal_error( + err.clone(), + send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, + )); break; } - if let Some(bucket) = self.next_rebal_bucket(pool_index).await? { + let next_bucket = match self.next_rebal_bucket(pool_index).await { + Ok(bucket) => bucket, + Err(err) => { + error!("next_rebal_bucket failed for pool {}: {:?}", pool_index, err); + final_result = Err(resolve_rebalance_terminal_error( + err.clone(), + send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, + )); + break; + } + }; + + if let Some(bucket) = next_bucket { info!("Rebalancing bucket: start {}", bucket); - if let Err(err) = self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index).await { - if err.to_string().contains("not initialized") { - info!("rebalance_bucket: rebalance not initialized, continue"); - continue; - } + if let Err(err) = resolve_rebalance_bucket_result( + self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index).await, + pool_index, + &bucket, + ) { error!("Error rebalancing bucket {}: {:?}", bucket, err); - done_tx.send(Err(err)).await.ok(); + final_result = Err(resolve_rebalance_terminal_error( + err.clone(), + send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, + )); break; } info!("Rebalance bucket: done {} ", bucket); - self.bucket_rebalance_done(pool_index, bucket).await?; + if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await { + error!("bucket_rebalance_done failed for pool {}: {:?}", pool_index, err); + final_result = Err(resolve_rebalance_terminal_error( + err.clone(), + send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, + )); + break; + } } else { info!("Rebalance bucket: no bucket to rebalance"); break; @@ -683,10 +966,19 @@ impl ECStore { info!("Pool {} rebalancing is done", pool_index); - done_tx.send(Ok(())).await.ok(); - save_task.await.ok(); + if final_result.is_ok() + && let Err(err) = send_rebalance_done_signal(&done_tx, Ok(()), pool_index).await + { + final_result = Err(err); + } + drop(done_tx); + if let Err(err) = resolve_rebalance_save_task_result(pool_index, save_task.await) + && final_result.is_ok() + { + final_result = Err(err); + } info!("Pool {} rebalancing is done2", pool_index); - Ok(()) + final_result } async fn check_if_rebalance_done(&self, pool_index: usize) -> bool { @@ -701,11 +993,19 @@ impl ECStore { return true; } - // Calculate the percentage of free space improvement - let pfi = (pool_stat.init_free_space + pool_stat.bytes) as f64 / pool_stat.init_capacity as f64; - // Mark pool rebalance as done if within 5% of the PercentFreeGoal - if (pfi - meta.percent_free_goal).abs() <= 0.05 { + let pfi = if pool_stat.init_capacity == 0 { + 0.0 + } else { + (pool_stat.init_free_space + pool_stat.bytes) as f64 / pool_stat.init_capacity as f64 + }; + + if rebalance_goal_reached( + pool_stat.init_free_space, + pool_stat.init_capacity, + pool_stat.bytes, + meta.percent_free_goal, + ) { pool_stat.info.status = RebalStatus::Completed; pool_stat.info.end_time = Some(OffsetDateTime::now_utc()); info!("check_if_rebalance_done: pool {} is completed, pfi: {}", pool_index, pfi); @@ -715,7 +1015,489 @@ impl ECStore { false } +} +fn rebalance_goal_reached(init_free_space: u64, init_capacity: u64, bytes: u64, percent_free_goal: f64) -> bool { + if init_capacity == 0 { + return false; + } + + let pfi = (init_free_space + bytes) as f64 / init_capacity as f64; + (pfi - percent_free_goal).abs() <= 0.05 + f64::EPSILON +} + +fn percent_free_ratio(total_free: u64, total_cap: u64) -> f64 { + if total_cap == 0 { + return 0.0; + } + total_free as f64 / total_cap as f64 +} + +fn next_rebal_bucket_from_stat(pool_stat: &RebalanceStats) -> Option { + if pool_stat.buckets.is_empty() { + return None; + } + + first_rebalance_bucket(pool_stat) +} + +fn rebalance_metadata_not_initialized_error(operation: &str) -> Error { + Error::other(format!("failed to {operation}: rebalance metadata not initialized")) +} + +fn invalid_rebalance_pool_index_error(pool_index: usize, pool_count: usize) -> Error { + Error::other(format!("invalid rebalance pool index {pool_index} for {pool_count} pools")) +} + +fn clone_rebalance_pool_stats(meta: Option<&RebalanceMeta>) -> Result> { + let Some(meta) = meta else { + return Err(rebalance_metadata_not_initialized_error("clone rebalance pool stats")); + }; + Ok(meta.pool_stats.clone()) +} + +fn resolve_next_rebalance_bucket(meta: Option<&RebalanceMeta>, pool_index: usize) -> Result> { + let Some(meta) = meta else { + return Err(rebalance_metadata_not_initialized_error("resolve next rebalance bucket")); + }; + + ensure_valid_rebalance_pool_index(meta.pool_stats.len(), pool_index)?; + let Some(pool_stat) = meta.pool_stats.get(pool_index) else { + return Err(invalid_rebalance_pool_index_error(pool_index, meta.pool_stats.len())); + }; + + if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating { + info!("next_rebal_bucket: pool_index: {} completed or not participating", pool_index); + return Ok(None); + } + + if pool_stat.buckets.is_empty() { + info!("next_rebal_bucket: pool_index: {} buckets is empty", pool_index); + return Ok(None); + } + + if let Some(bucket) = next_rebal_bucket_from_stat(pool_stat) { + info!("next_rebal_bucket: pool_index: {} bucket: {}", pool_index, bucket); + return Ok(Some(bucket)); + } + + info!("next_rebal_bucket: pool_index: {} None", pool_index); + Ok(None) +} + +fn mark_rebalance_bucket_done(meta: Option<&mut RebalanceMeta>, pool_index: usize, bucket: &str) -> Result<()> { + let Some(meta) = meta else { + return Err(rebalance_metadata_not_initialized_error("mark rebalance bucket done")); + }; + + ensure_valid_rebalance_pool_index(meta.pool_stats.len(), pool_index)?; + let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else { + return Err(invalid_rebalance_pool_index_error(pool_index, meta.pool_stats.len())); + }; + + info!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets); + + if take_bucket_from_rebalance_queue(pool_stat, bucket) { + info!("bucket_rebalance_done: bucket {} rebalanced", bucket); + Ok(()) + } else { + Err(Error::other(format!( + "failed to mark rebalance bucket done: bucket {bucket} was not queued for pool {pool_index}" + ))) + } +} + +fn take_bucket_from_rebalance_queue(pool_stat: &mut RebalanceStats, bucket: &str) -> bool { + let mut found = false; + pool_stat.buckets.retain(|name| { + if name == bucket { + found = true; + pool_stat.rebalanced_buckets.push(name.clone()); + false + } else { + true + } + }); + + found +} + +fn should_pool_participate(init_free_space: u64, init_capacity: u64, percent_free_goal: f64) -> bool { + init_capacity > 0 && percent_free_ratio(init_free_space, init_capacity) < percent_free_goal +} + +fn resolve_rebalance_worker_result( + set_idx: usize, + worker_result: std::result::Result, tokio::task::JoinError>, +) -> Result<()> { + match worker_result { + Ok(result) => result, + Err(err) => Err(Error::other(format!("rebalance worker {set_idx} task join error: {err}"))), + } +} + +fn resolve_rebalance_save_task_result( + pool_idx: usize, + save_task_result: std::result::Result, tokio::task::JoinError>, +) -> Result<()> { + match save_task_result { + Ok(result) => result.map_err(|err| Error::other(format!("rebalance save_task failed for pool {pool_idx}: {err}"))), + Err(err) => Err(Error::other(format!("rebalance save_task for pool {pool_idx} join error: {err}"))), + } +} + +fn resolve_rebalance_meta_save_result(result: Result<()>, stage: &str) -> Result<()> { + result.map_err(|err| Error::other(format!("rebalance meta save failed during {stage}: {err}"))) +} + +fn resolve_rebalance_meta_load_result(result: Result<()>) -> Result { + match result { + Ok(()) => Ok(true), + Err(Error::ConfigNotFound) => Ok(false), + Err(err) => { + error!("rebalanceMeta: load rebalance meta err {:?}", &err); + Err(Error::other(format!("rebalance metadata load failed during load_rebalance_meta: {err}"))) + } + } +} + +fn resolve_rebalance_stats_update_result(result: Result<()>, pool_idx: usize, bucket: &str, object_name: &str) -> Result<()> { + result.map_err(|err| { + Error::other(format!( + "rebalance stats update failed for pool {pool_idx} bucket {bucket} object {object_name}: {err}" + )) + }) +} + +fn resolve_rebalance_file_info_versions_result( + result: std::result::Result, + bucket: &str, + object_name: &str, +) -> Result +where + E: std::fmt::Display, +{ + result.map_err(|err| Error::other(format!("rebalance file_info_versions failed for {bucket}/{object_name}: {err}"))) +} + +fn resolve_rebalance_entry_cleanup_delete_result(result: Result, bucket: &str, object_name: &str) -> Result<()> { + match result { + Ok(_) => Ok(()), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()), + Err(err) => Err(Error::other(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}"))), + } +} + +fn resolve_rebalance_migrate_result_error( + err: Option, + pool_idx: usize, + bucket: &str, + object_name: &str, + version_id: Option<&str>, +) -> Error { + err.unwrap_or_else(|| { + Error::other(format!( + "rebalance migration reported failure without error for pool {pool_idx} entry {bucket}/{object_name} version {}", + version_id.unwrap_or("none") + )) + }) +} + +fn resolve_load_rebalance_stats_update_result(result: Result<()>) -> Result<()> { + result.map_err(|err| Error::other(format!("rebalance metadata stats refresh failed after load: {err}"))) +} + +async fn send_rebalance_done_signal( + done_tx: &tokio::sync::mpsc::Sender>, + signal: Result<()>, + pool_idx: usize, +) -> Result<()> { + done_tx + .send(signal) + .await + .map_err(|err| Error::other(format!("rebalance done signal send failed for pool {pool_idx}: {err}"))) +} + +fn resolve_rebalance_terminal_error(primary_err: Error, signal_result: Result<()>) -> Error { + match signal_result { + Ok(()) => primary_err, + Err(signal_err) => Error::other(format!("rebalance terminal signal failed after error {primary_err}: {signal_err}")), + } +} + +fn resolve_rebalance_bucket_error(entry_error: Option, worker_error: Option) -> Result<()> { + if let Some(err) = entry_error { + return Err(err); + } + + if let Some(err) = worker_error { + return Err(err); + } + + Ok(()) +} + +fn resolve_rebalance_bucket_result(result: Result<()>, pool_idx: usize, bucket: &str) -> Result<()> { + match result { + Ok(()) => Ok(()), + Err(err) if is_err_operation_canceled(&err) => Err(err), + Err(err) => Err(Error::other(format!("rebalance bucket {bucket} failed for pool {pool_idx}: {err}"))), + } +} + +fn ensure_rebalance_listing_disks_available(has_disks: bool, bucket: &str) -> Result<()> { + if !has_disks { + return Err(Error::other(format!( + "failed to list objects to rebalance for bucket {bucket}: no disks available" + ))); + } + + Ok(()) +} + +fn with_rebalance_entry_context(stage: &str, bucket: &str, object_name: &str, err: Error) -> Error { + Error::other(format!("rebalance entry {stage} failed for {bucket}/{object_name}: {err}")) +} + +fn should_count_rebalance_version_complete(result: &MigrationVersionResult) -> bool { + result.cleanup_ignored || (result.moved && !result.failed) +} + +fn should_cleanup_rebalance_source_entry(rebalanced: usize, total_versions: usize) -> bool { + rebalanced == total_versions +} + +fn should_skip_rebalance_delete_marker(version: &FileInfo, remaining_versions: usize, replication_configured: bool) -> bool { + version.deleted && remaining_versions == 1 && !replication_configured +} + +fn resolve_rebalance_optional_bucket_config_result(bucket: &str, stage: &str, result: Result) -> Result> { + match result { + Ok(config) => Ok(Some(config)), + Err(Error::ConfigNotFound) => Ok(None), + Err(err) => Err(Error::other(format!("rebalance {stage} config load failed for bucket {bucket}: {err}"))), + } +} + +async fn load_rebalance_bucket_configs(bucket: &str) -> Result { + if bucket == crate::disk::RUSTFS_META_BUCKET { + return Ok(RebalanceBucketConfigs::default()); + } + + let _ = resolve_rebalance_optional_bucket_config_result( + bucket, + "versioning", + crate::bucket::versioning_sys::BucketVersioningSys::get(bucket).await, + )?; + + Ok(RebalanceBucketConfigs { + lifecycle_config: crate::global::GLOBAL_LifecycleSys.get(bucket).await, + lock_retention: crate::bucket::object_lock::objectlock_sys::BucketObjectLockSys::get(bucket).await, + replication_config: resolve_rebalance_optional_bucket_config_result( + bucket, + "replication", + crate::bucket::metadata_sys::get_replication_config(bucket).await, + )?, + }) +} + +fn clone_first_arc(values: &[Arc], err_msg: &str) -> Result> { + values.first().cloned().ok_or_else(|| Error::other(err_msg)) +} + +fn clone_arc_by_index(values: &[Arc], idx: usize, err_prefix: &str) -> Result> { + values + .get(idx) + .cloned() + .ok_or_else(|| Error::other(format!("{err_prefix}: {idx}"))) +} + +fn ensure_valid_rebalance_pool_index(pool_count: usize, idx: usize) -> Result<()> { + if idx >= pool_count { + return Err(invalid_rebalance_pool_index_error(idx, pool_count)); + } + + Ok(()) +} + +enum RebalanceTerminalEvent { + Completed { msg: String }, + Stopped { msg: String }, + Failed { msg: String, last_error: String }, + ChannelClosed { msg: String, last_error: String }, +} + +impl RebalanceTerminalEvent { + fn message(&self) -> &str { + match self { + RebalanceTerminalEvent::Completed { msg } + | RebalanceTerminalEvent::Stopped { msg } + | RebalanceTerminalEvent::Failed { msg, .. } + | RebalanceTerminalEvent::ChannelClosed { msg, .. } => msg, + } + } +} + +fn apply_rebalance_terminal_event( + status: &mut RebalStatus, + end_time: &mut Option, + last_error: &mut Option, + terminal_event: RebalanceTerminalEvent, + now: OffsetDateTime, +) { + match terminal_event { + RebalanceTerminalEvent::Completed { .. } => { + *status = RebalStatus::Completed; + *end_time = Some(now); + *last_error = None; + } + RebalanceTerminalEvent::Stopped { .. } => { + *status = RebalStatus::Stopped; + *end_time = Some(now); + *last_error = None; + } + RebalanceTerminalEvent::Failed { last_error: err, .. } + | RebalanceTerminalEvent::ChannelClosed { last_error: err, .. } => { + *status = RebalStatus::Failed; + *end_time = Some(now); + *last_error = Some(err); + } + } +} + +fn classify_rebalance_terminal_event(signal: Option>, now: OffsetDateTime) -> RebalanceTerminalEvent { + match signal { + Some(Ok(())) => RebalanceTerminalEvent::Completed { + msg: format!("Rebalance completed at {now:?}"), + }, + Some(Err(err)) => { + if is_err_operation_canceled(&err) { + RebalanceTerminalEvent::Stopped { + msg: format!("Rebalance stopped at {now:?}"), + } + } else { + RebalanceTerminalEvent::Failed { + msg: format!("Rebalance failed at {now:?} with err {err:?}"), + last_error: err.to_string(), + } + } + } + None => RebalanceTerminalEvent::ChannelClosed { + msg: format!("Rebalance save task channel closed unexpectedly at {now:?}"), + last_error: format!("rebalance save channel closed before terminal event at {now:?}"), + }, + } +} + +fn ensure_rebalance_not_decommissioning(decommission_running: bool) -> bool { + !decommission_running +} + +fn validate_start_rebalance_state(decommission_running: bool, meta_loaded: bool) -> Result<()> { + if !ensure_rebalance_not_decommissioning(decommission_running) { + return Err(Error::DecommissionAlreadyRunning); + } + if !meta_loaded { + return Err(Error::ConfigNotFound); + } + + Ok(()) +} + +fn should_skip_start_rebalance(cancel_attached: bool, in_progress: bool) -> bool { + cancel_attached && in_progress +} + +fn is_rebalance_stopped_terminal_event(terminal_event: &RebalanceTerminalEvent) -> bool { + matches!(terminal_event, RebalanceTerminalEvent::Stopped { .. }) +} + +fn should_preserve_rebalance_stopped_state( + meta_stopped: bool, + status: RebalStatus, + terminal_event: &RebalanceTerminalEvent, +) -> bool { + (meta_stopped || status == RebalStatus::Stopped) && !is_rebalance_stopped_terminal_event(terminal_event) +} + +fn resolve_rebalance_participants(pool_stats: &[RebalanceStats], pool_count: usize) -> Vec { + let mut participants = vec![false; pool_count]; + + for (idx, pool_stat) in pool_stats.iter().enumerate() { + if idx >= participants.len() { + break; + } + + if pool_stat.info.status == RebalStatus::Started { + participants[idx] = pool_stat.participating; + } + } + + participants +} + +fn is_rebalance_actively_running(meta: &RebalanceMeta) -> bool { + meta.cancel.is_some() && is_rebalance_in_progress(meta) +} + +fn should_ignore_rebalance_data_usage_cache(bucket: &str) -> bool { + bucket == crate::disk::RUSTFS_META_BUCKET +} + +fn apply_rebalance_save_option(meta: &mut RebalanceMeta, pool_idx: usize, opt: RebalSaveOpt, now: OffsetDateTime) { + match opt { + RebalSaveOpt::Stats => { + if pool_idx >= meta.pool_stats.len() { + info!("save_rebalance_stats: pool_idx {pool_idx} out of range for pool_stats"); + } + } + RebalSaveOpt::StoppedAt => { + apply_stopped_at(meta, now); + } + } + + meta.last_refreshed_at = Some(now); +} + +fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, stop_time: OffsetDateTime) { + for pool_stat in meta.pool_stats.iter_mut() { + if pool_stat.info.status == RebalStatus::Started { + pool_stat.info.status = RebalStatus::Stopped; + pool_stat.info.end_time.get_or_insert(stop_time); + pool_stat.info.last_error = None; + } + } +} + +fn apply_stopped_at(meta: &mut RebalanceMeta, now: OffsetDateTime) { + meta.stopped_at = Some(now); + mark_started_rebalance_pools_stopped(meta, now); +} + +fn stop_rebalance_state(meta: &mut RebalanceMeta, now: OffsetDateTime) { + if let Some(cancel_tx) = meta.cancel.take() { + cancel_tx.cancel(); + } + + let stop_time = meta.stopped_at.unwrap_or(now); + if meta.stopped_at.is_none() && is_rebalance_in_progress(meta) { + meta.stopped_at = Some(stop_time); + } + + if meta.stopped_at.is_some() { + mark_started_rebalance_pools_stopped(meta, stop_time); + } +} + +fn stop_rebalance_meta_snapshot(meta: Option<&mut RebalanceMeta>, now: OffsetDateTime) -> Option { + meta.map(|meta| { + stop_rebalance_state(meta, now); + meta.last_refreshed_at = Some(now); + meta.clone() + }) +} + +impl ECStore { #[allow(unused_assignments)] #[tracing::instrument(skip(self, set))] async fn rebalance_entry( @@ -724,8 +1506,9 @@ impl ECStore { pool_index: usize, entry: MetaCacheEntry, set: Arc, + bucket_configs: Arc, // wk: Arc, - ) { + ) -> Result<()> { info!("rebalance_entry: start rebalance_entry"); // defer!(|| async { @@ -736,157 +1519,103 @@ impl ECStore { if entry.is_dir() { info!("rebalance_entry: entry is dir, skipping"); - return; + return Ok(()); } if self.check_if_rebalance_done(pool_index).await { info!("rebalance_entry: rebalance done, skipping pool {}", pool_index); - return; + return Ok(()); } - let mut fivs = match entry.file_info_versions(&bucket) { - Ok(fivs) => fivs, - Err(err) => { - error!("rebalance_entry Error getting file info versions: {}", err); - info!("rebalance_entry: Error getting file info versions, skipping"); - return; - } - }; + let mut fivs = + resolve_rebalance_file_info_versions_result(entry.file_info_versions(&bucket), bucket.as_str(), entry.name.as_str())?; fivs.versions.sort_by(|a, b| b.mod_time.cmp(&a.mod_time)); let mut rebalanced: usize = 0; - let expired: usize = 0; + let mut expired: usize = 0; for version in fivs.versions.iter() { - if version.is_remote() { - info!("rebalance_entry Entry {} is remote, skipping", version.name); + if crate::pools::should_skip_lifecycle_for_data_movement( + self.clone(), + &bucket, + version, + bucket_configs.lifecycle_config.as_ref(), + bucket_configs.lock_retention.clone(), + bucket_configs.replication_config.clone(), + true, + &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Rebal, + ) + .await + { + expired += 1; + info!("rebalance_entry {} Entry {} expired by lifecycle, skipping", &bucket, version.name); continue; } - // TODO: filterLifecycle let remaining_versions = fivs.versions.len() - expired; - if version.deleted && remaining_versions == 1 { + if should_skip_rebalance_delete_marker(version, remaining_versions, bucket_configs.replication_config.is_some()) { rebalanced += 1; - info!("rebalance_entry Entry {} is deleted and last version, skipping", version.name); + info!( + "rebalance_entry Entry {} is deleted and last version without replication, skipping", + version.name + ); continue; } + let version_id = version.version_id.map(|v| v.to_string()); + let mut transfer = |src_pool_idx: usize, bucket: String, rd: GetObjectReader| { + let store = self.clone(); + async move { store.rebalance_object(src_pool_idx, bucket, rd).await } + }; + let result = migrate_entry_version( + set.as_ref(), + bucket.clone(), + pool_index, + version, + version_id.clone(), + 3, + should_ignore_rebalance_data_usage_cache(bucket.as_str()), + &mut transfer, + ) + .await; - let mut ignore = false; - let mut failure = false; - let mut error = None; - if version.deleted { - if let Err(err) = set - .delete_object( - &bucket, - &version.name, - ObjectOptions { - versioned: true, - version_id: version_id.clone(), - mod_time: version.mod_time, - src_pool_idx: pool_index, - data_movement: true, - delete_marker: true, - skip_decommissioned: true, - ..Default::default() - }, - ) - .await - { - if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { - ignore = true; - info!("rebalance_entry {} Entry {} is already deleted, skipping", &bucket, version.name); - continue; - } - error = Some(err); - failure = true; - } - - if !failure { - error!("rebalance_entry {} Entry {} deleted successfully", &bucket, &version.name); - let _ = self.update_pool_stats(pool_index, bucket.clone(), version).await; - + if result.ignored { + if should_count_rebalance_version_complete(&result) { rebalanced += 1; - } else { - error!( - "rebalance_entry {} Error deleting entry {}/{:?}: {:?}", - &bucket, &version.name, &version.version_id, error - ); } - - continue; - } - - for _i in 0..3 { - info!("rebalance_entry: get_object_reader, bucket: {}, version: {}", &bucket, &version.name); - let rd = match set - .get_object_reader( - bucket.as_str(), - &encode_dir_object(&version.name), - None, - HeaderMap::new(), - &ObjectOptions { - version_id: version_id.clone(), - no_lock: true, // NoDecryption - ..Default::default() - }, - ) - .await - { - Ok(rd) => rd, - Err(err) => { - if is_err_object_not_found(&err) || is_err_version_not_found(&err) { - ignore = true; - info!( - "rebalance_entry: get_object_reader, bucket: {}, version: {}, ignore", - &bucket, &version.name - ); - break; - } - - failure = true; - error!("rebalance_entry: get_object_reader err {:?}", &err); - continue; - } - }; - - if let Err(err) = self.clone().rebalance_object(pool_index, bucket.clone(), rd).await { - if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { - ignore = true; - info!("rebalance_entry {} Entry {} is already deleted, skipping", &bucket, version.name); - break; - } - - failure = true; - error!("rebalance_entry: rebalance_object err {:?}", &err); - continue; - } - - failure = false; - info!("rebalance_entry {} Entry {} rebalanced successfully", &bucket, &version.name); - break; - } - - if ignore { info!("rebalance_entry {} Entry {} is already deleted, skipping", &bucket, version.name); continue; } - if failure { + if result.failed { + let err = resolve_rebalance_migrate_result_error( + result.error, + pool_index, + bucket.as_str(), + version.name.as_str(), + version_id.as_deref(), + ); error!( "rebalance_entry {} Error rebalancing entry {}/{:?}: {:?}", - &bucket, &version.name, &version.version_id, error + &bucket, &version.name, &version.version_id, err ); - break; + return Err(with_rebalance_entry_context("migrate", bucket.as_str(), version.name.as_str(), err)); } - let _ = self.update_pool_stats(pool_index, bucket.clone(), version).await; - rebalanced += 1; + resolve_rebalance_stats_update_result( + self.update_pool_stats(pool_index, bucket.clone(), version).await, + pool_index, + bucket.as_str(), + version.name.as_str(), + )?; + if should_count_rebalance_version_complete(&result) { + rebalanced += 1; + } } - if rebalanced == fivs.versions.len() { - if let Err(err) = set - .delete_object( + if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len()) { + resolve_rebalance_entry_cleanup_delete_result( + set.delete_object( bucket.as_str(), &encode_dir_object(&entry.name), ObjectOptions { @@ -896,148 +1625,25 @@ impl ECStore { ..Default::default() }, ) - .await - { - error!("rebalance_entry: delete_object err {:?}", &err); - } else { - info!("rebalance_entry {} Entry {} deleted successfully", &bucket, &entry.name); - } - } - } - - #[tracing::instrument(skip(self, rd))] - async fn rebalance_object(self: Arc, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> { - let object_info = rd.object_info.clone(); - - // TODO: check : use size or actual_size ? - let _actual_size = object_info.get_actual_size()?; - - if object_info.is_multipart() { - let res = match self - .new_multipart_upload( - &bucket, - &object_info.name, - &ObjectOptions { - version_id: object_info.version_id.as_ref().map(|v| v.to_string()), - user_defined: object_info.user_defined.clone(), - src_pool_idx: pool_idx, - data_movement: true, - ..Default::default() - }, - ) - .await - { - Ok(res) => res, - Err(err) => { - error!("rebalance_object: new_multipart_upload err {:?}", &err); - return Err(err); - } - }; - - defer!(|| async { - if let Err(err) = self - .abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default()) - .await - { - error!("rebalance_object: abort_multipart_upload err {:?}", &err); - } - }); - - let mut parts = vec![CompletePart::default(); object_info.parts.len()]; - - let mut reader = rd.stream; - - for (i, part) in object_info.parts.iter().enumerate() { - // Read one part from the reader and upload it each time - - let mut chunk = vec![0u8; part.size]; - - reader.read_exact(&mut chunk).await?; - - // Read one part from the reader and upload it each time - let mut data = PutObjReader::from_vec(chunk); - - let pi = match self - .put_object_part( - &bucket, - &object_info.name, - &res.upload_id, - part.number, - &mut data, - &ObjectOptions { - preserve_etag: Some(part.etag.clone()), - ..Default::default() - }, - ) - .await - { - Ok(pi) => pi, - Err(err) => { - error!("rebalance_object: put_object_part err {:?}", &err); - return Err(err); - } - }; - - parts[i] = CompletePart { - part_num: pi.part_num, - etag: pi.etag, - ..Default::default() - }; - } - - if let Err(err) = self - .clone() - .complete_multipart_upload( - &bucket, - &object_info.name, - &res.upload_id, - parts, - &ObjectOptions { - data_movement: true, - mod_time: object_info.mod_time, - ..Default::default() - }, - ) - .await - { - error!("rebalance_object: complete_multipart_upload err {:?}", &err); - return Err(err); - } - - return Ok(()); - } - - let reader = BufReader::new(rd.stream); - let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?; - let mut data = PutObjReader::new(hrd); - - if let Err(err) = self - .put_object( - &bucket, - &object_info.name, - &mut data, - &ObjectOptions { - src_pool_idx: pool_idx, - data_movement: true, - version_id: object_info.version_id.as_ref().map(|v| v.to_string()), - mod_time: object_info.mod_time, - user_defined: object_info.user_defined.clone(), - preserve_etag: object_info.etag.clone(), - - ..Default::default() - }, - ) - .await - { - error!("rebalance_object: put_object err {:?}", &err); - return Err(err); + .await, + bucket.as_str(), + entry.name.as_str(), + )?; + info!("rebalance_entry {} Entry {} deleted successfully", &bucket, &entry.name); } Ok(()) } + #[tracing::instrument(skip(self, rd))] + async fn rebalance_object(self: Arc, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> { + data_movement::migrate_object(self, pool_idx, bucket, rd, "rebalance_object").await + } + #[tracing::instrument(skip(self, rx))] async fn rebalance_bucket(self: &Arc, rx: CancellationToken, bucket: String, pool_index: usize) -> Result<()> { + ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?; + // Placeholder for actual bucket rebalance logic info!("Rebalancing bucket {} in pool {}", bucket, pool_index); @@ -1046,9 +1652,11 @@ impl ECStore { // } - let pool = self.pools[pool_index].clone(); + let pool = clone_arc_by_index(self.pools.as_slice(), pool_index, "invalid rebalance pool index")?; + let bucket_configs = Arc::new(load_rebalance_bucket_configs(&bucket).await?); let mut jobs = Vec::new(); + let entry_error = Arc::new(tokio::sync::Mutex::new(None::)); // let wk = Workers::new(pool.disk_set.len() * 2).map_err(Error::other)?; // wk.clone().take().await; @@ -1056,19 +1664,39 @@ impl ECStore { let rebalance_entry: ListCallback = Arc::new({ let this = Arc::clone(self); let bucket = bucket.clone(); + let entry_error = entry_error.clone(); + let callback_rx = rx.clone(); // let wk = wk.clone(); let set = set.clone(); + let bucket_configs = bucket_configs.clone(); move |entry: MetaCacheEntry| { let this = this.clone(); let bucket = bucket.clone(); + let entry_error = entry_error.clone(); + let callback_rx = callback_rx.clone(); // let wk = wk.clone(); let set = set.clone(); + let bucket_configs = bucket_configs.clone(); Box::pin(async move { + if callback_rx.is_cancelled() { + return; + } + if entry_error.lock().await.is_some() { + return; + } + info!("rebalance_entry: rebalance_entry spawn start"); // wk.take().await; // tokio::spawn(async move { info!("rebalance_entry: rebalance_entry spawn start2"); - this.rebalance_entry(bucket, pool_index, entry, set).await; + if let Err(err) = this.rebalance_entry(bucket, pool_index, entry, set, bucket_configs).await { + error!("rebalance_entry: rebalance entry failed: {err}"); + let mut first_err = entry_error.lock().await; + if first_err.is_none() { + *first_err = Some(err); + callback_rx.cancel(); + } + } info!("rebalance_entry: rebalance_entry spawn done"); // }); }) @@ -1081,64 +1709,56 @@ impl ECStore { // let wk = wk.clone(); let job = tokio::spawn(async move { - if let Err(err) = set.list_objects_to_rebalance(rx, bucket, rebalance_entry).await { + let result = set.list_objects_to_rebalance(rx, bucket, rebalance_entry).await; + if let Err(err) = &result { error!("Rebalance worker {} error: {}", set_idx, err); } else { info!("Rebalance worker {} done", set_idx); } // wk.clone().give().await; + result }); - jobs.push(job); + jobs.push((set_idx, job)); } // wk.wait().await; - for job in jobs { - job.await.unwrap(); + let mut worker_error: Option = None; + for (set_idx, job) in jobs { + if let Err(err) = resolve_rebalance_worker_result(set_idx, job.await) + && worker_error.is_none() + { + worker_error = Some(err); + } } + let entry_error = entry_error.lock().await.clone(); + resolve_rebalance_bucket_error(entry_error, worker_error)?; + info!("rebalance_bucket: rebalance_bucket done"); Ok(()) } #[tracing::instrument(skip(self))] pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> { - // TODO: lock - let mut meta = RebalanceMeta::new(); - if let Err(err) = meta.load(self.pools[0].clone()).await - && err != Error::ConfigNotFound - { - info!("save_rebalance_stats: load err: {:?}", err); - return Err(err); - } - - match opt { - RebalSaveOpt::Stats => { - { - let mut rebalance_meta = self.rebalance_meta.write().await; - if let Some(rbm) = rebalance_meta.as_mut() { - meta.pool_stats[pool_idx] = rbm.pool_stats[pool_idx].clone(); - } - } - - if let Some(pool_stat) = meta.pool_stats.get_mut(pool_idx) { - pool_stat.info.end_time = Some(OffsetDateTime::now_utc()); - } - } - RebalSaveOpt::StoppedAt => { - meta.stopped_at = Some(OffsetDateTime::now_utc()); - } - } - - { + let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; - *rebalance_meta = Some(meta.clone()); - } + let Some(meta) = rebalance_meta.as_mut() else { + return Ok(()); + }; + + let now = OffsetDateTime::now_utc(); + apply_rebalance_save_option(meta, pool_idx, opt, now); + meta.clone() + }; + + let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?; info!( "save_rebalance_stats: save rebalance meta, pool_idx: {}, opt: {:?}, meta: {:?}", - pool_idx, opt, meta + pool_idx, opt, meta_to_save ); - meta.save(self.pools[0].clone()).await?; + let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}"); + resolve_rebalance_meta_save_result(meta_to_save.save(pool).await, stage.as_str())?; Ok(()) } @@ -1155,10 +1775,7 @@ impl SetDisks { info!("list_objects_to_rebalance: start list_objects_to_rebalance"); // Placeholder for actual object listing logic let (disks, _) = self.get_online_disks_with_healing(false).await; - if disks.is_empty() { - info!("list_objects_to_rebalance: no disk available"); - return Err(Error::other("errNoDiskAvailable")); - } + ensure_rebalance_listing_disks_available(!disks.is_empty(), &bucket)?; info!("list_objects_to_rebalance: get online disks with healing"); let listing_quorum = self.set_drive_count.div_ceil(2); @@ -1207,3 +1824,2172 @@ impl SetDisks { Ok(()) } } + +#[cfg(test)] +mod rebalance_unit_tests { + use super::first_rebalance_bucket; + use super::is_rebalance_actively_running; + use super::is_rebalance_conflicting_with_decommission; + use super::is_rebalance_in_progress; + use super::percent_free_ratio; + use super::rebalance_goal_reached; + use super::{ + GetObjectReader, HTTPRangeSpec, MigrationBackend, MigrationVersionResult, ObjectInfo, ObjectOptions, RebalSaveOpt, + RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceTerminalEvent, apply_rebalance_save_option, + apply_rebalance_terminal_event, apply_stopped_at, classify_rebalance_terminal_event, clone_arc_by_index, clone_first_arc, + clone_rebalance_pool_stats, ensure_rebalance_listing_disks_available, ensure_rebalance_not_decommissioning, + ensure_valid_rebalance_pool_index, is_rebalance_stopped_terminal_event, load_rebalance_bucket_configs, + mark_rebalance_bucket_done, migrate_entry_version, next_rebal_bucket_from_stat, rebalance_delete_marker_opts, + rebalance_meta_load_no_data_error, rebalance_meta_load_unknown_format_error, rebalance_meta_load_unknown_version_error, + resolve_load_rebalance_stats_update_result, resolve_next_rebalance_bucket, resolve_rebalance_bucket_error, + resolve_rebalance_bucket_result, resolve_rebalance_entry_cleanup_delete_result, + resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, + resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_participants, + resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error, + resolve_rebalance_worker_result, send_rebalance_done_signal, should_cleanup_rebalance_source_entry, + should_count_rebalance_version_complete, should_ignore_rebalance_data_usage_cache, should_pool_participate, + should_preserve_rebalance_stopped_state, should_skip_rebalance_delete_marker, should_skip_start_rebalance, + stop_rebalance_meta_snapshot, stop_rebalance_state, take_bucket_from_rebalance_queue, validate_start_rebalance_state, + with_rebalance_entry_context, + }; + use crate::data_movement; + use crate::data_usage::DATA_USAGE_CACHE_NAME; + use crate::disk::RUSTFS_META_BUCKET; + use crate::error::{Error, Result}; + use rustfs_filemeta::FileInfo; + use rustfs_filemeta::TRANSITION_COMPLETE; + use rustfs_rio::Index; + use s3s::dto::ReplicationConfiguration; + use std::io::Cursor; + use std::sync::Arc; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + use time::OffsetDateTime; + use tokio::sync::mpsc; + use tokio_util::sync::CancellationToken; + + struct MigrationBackendSpy { + get_object_reader: Mutex>>, + delete_object: Mutex>>, + move_remote: Mutex>>, + get_calls: AtomicUsize, + delete_calls: AtomicUsize, + move_remote_calls: AtomicUsize, + } + + impl MigrationBackendSpy { + fn new( + get_object_reader: Option>, + delete_object: Option>, + move_remote: Option>, + ) -> Self { + Self { + get_object_reader: Mutex::new(get_object_reader), + delete_object: Mutex::new(delete_object), + move_remote: Mutex::new(move_remote), + get_calls: AtomicUsize::new(0), + delete_calls: AtomicUsize::new(0), + move_remote_calls: AtomicUsize::new(0), + } + } + + fn get_calls(&self) -> usize { + self.get_calls.load(Ordering::SeqCst) + } + + fn delete_calls(&self) -> usize { + self.delete_calls.load(Ordering::SeqCst) + } + + fn move_remote_calls(&self) -> usize { + self.move_remote_calls.load(Ordering::SeqCst) + } + + fn make_reader() -> GetObjectReader { + GetObjectReader { + stream: Box::new(Cursor::new(vec![0_u8; 3])), + object_info: ObjectInfo::default(), + } + } + } + + #[async_trait::async_trait] + impl MigrationBackend for MigrationBackendSpy { + async fn get_object_reader_for_migration( + &self, + _bucket: &str, + _object: &str, + _range: Option, + _h: http::HeaderMap, + _opts: &ObjectOptions, + ) -> Result { + self.get_calls.fetch_add(1, Ordering::SeqCst); + if let Some(result) = self.get_object_reader.lock().unwrap().take() { + return result; + } + + Ok(Self::make_reader()) + } + + async fn delete_object_for_migration(&self, _bucket: &str, _object: &str, _opts: ObjectOptions) -> Result { + self.delete_calls.fetch_add(1, Ordering::SeqCst); + if let Some(result) = self.delete_object.lock().unwrap().take() { + return result; + } + + Ok(ObjectInfo::default()) + } + + async fn move_remote_version_for_migration( + &self, + _bucket: &str, + _object: &str, + _fi: &FileInfo, + _opts: &ObjectOptions, + ) -> Result<()> { + self.move_remote_calls.fetch_add(1, Ordering::SeqCst); + if let Some(result) = self.move_remote.lock().unwrap().take() { + return result; + } + + Ok(()) + } + } + + fn version_deleted() -> FileInfo { + let mut version = FileInfo::new("object.bin", 4, 2); + version.name = "object.bin".to_string(); + version.deleted = true; + version + } + + fn version_normal() -> FileInfo { + let mut version = FileInfo::new("object.bin", 4, 2); + version.name = "object.bin".to_string(); + version.size = 64; + version + } + + fn version_remote() -> FileInfo { + let mut version = FileInfo::new("object.bin", 4, 2); + version.name = "object.bin".to_string(); + version.transition_status = TRANSITION_COMPLETE.to_string(); + version + } + + #[test] + fn test_rebalance_delete_marker_opts_preserves_replication_state() { + let mod_time = OffsetDateTime::now_utc(); + let version = FileInfo { + mod_time: Some(mod_time), + replication_state_internal: Some(rustfs_filemeta::ReplicationState { + replica_status: rustfs_filemeta::ReplicationStatusType::Replica, + delete_marker: true, + replicate_decision_str: "existing".to_string(), + ..Default::default() + }), + ..version_deleted() + }; + + let opts = rebalance_delete_marker_opts(&version, Some("version-id".to_string()), 7); + let replication = opts.delete_replication.expect("replication state should be preserved"); + + assert!(opts.versioned); + assert!(opts.data_movement); + assert!(opts.delete_marker); + assert!(opts.skip_decommissioned); + assert_eq!(opts.src_pool_idx, 7); + assert_eq!(opts.version_id.as_deref(), Some("version-id")); + assert_eq!(opts.mod_time, Some(mod_time)); + assert_eq!(replication.replica_status, rustfs_filemeta::ReplicationStatusType::Replica); + assert!(replication.delete_marker); + assert_eq!(replication.replicate_decision_str, "existing"); + } + + #[tokio::test] + async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() { + let backend = MigrationBackendSpy::new(None, Some(Ok(ObjectInfo::default())), Some(Ok(()))); + let version = version_remote(); + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + }; + + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 0, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(!result.ignored); + assert!(!result.cleanup_ignored); + assert!(result.moved); + assert!(!result.failed); + assert!(result.error.is_none()); + assert_eq!(transfer_count.load(Ordering::SeqCst), 0); + assert_eq!(backend.move_remote_calls(), 1); + assert_eq!(backend.get_calls(), 0); + assert_eq!(backend.delete_calls(), 0); + } + + #[tokio::test] + async fn test_migrate_entry_version_remote_not_found_is_cleanup_ignored() { + let backend = MigrationBackendSpy::new( + None, + Some(Ok(ObjectInfo::default())), + Some(Err(Error::ObjectNotFound("bucket".to_string(), "object.bin".to_string()))), + ); + let version = version_remote(); + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + }; + + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 0, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(result.ignored); + assert!(result.cleanup_ignored); + assert!(!result.moved); + assert!(!result.failed); + assert!(result.error.is_none()); + assert_eq!(transfer_count.load(Ordering::SeqCst), 0); + assert_eq!(backend.move_remote_calls(), 1); + assert_eq!(backend.get_calls(), 0); + assert_eq!(backend.delete_calls(), 0); + } + + #[tokio::test] + async fn test_migrate_entry_version_remote_failure_is_reported() { + let backend = MigrationBackendSpy::new(None, Some(Ok(ObjectInfo::default())), Some(Err(Error::SlowDown))); + let version = version_remote(); + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + }; + + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 0, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(!result.ignored); + assert!(!result.cleanup_ignored); + assert!(!result.moved); + assert!(result.failed); + assert!(matches!(result.error, Some(Error::SlowDown))); + assert_eq!(transfer_count.load(Ordering::SeqCst), 0); + assert_eq!(backend.move_remote_calls(), 1); + assert_eq!(backend.get_calls(), 0); + assert_eq!(backend.delete_calls(), 0); + } + + #[tokio::test] + async fn test_migrate_entry_version_deleted_version_calls_delete_and_moved() { + let backend = MigrationBackendSpy::new(None, Some(Ok(ObjectInfo::default())), None); + let version = version_deleted(); + let mut transfer = |_, _, _| async move { Ok(()) }; + + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(!result.ignored); + assert!(!result.cleanup_ignored); + assert!(result.moved); + assert!(!result.failed); + assert!(result.error.is_none()); + assert_eq!(backend.get_calls(), 0); + assert_eq!(backend.delete_calls(), 1); + } + + #[tokio::test] + async fn test_migrate_entry_version_deleted_version_not_found_is_ignored() { + let backend = MigrationBackendSpy::new( + None, + Some(Err(Error::ObjectNotFound("bucket".to_string(), "object.bin".to_string()))), + None, + ); + let version = version_deleted(); + let mut transfer = |_, _, _| async move { Ok(()) }; + + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(result.ignored); + assert!(result.cleanup_ignored); + assert!(!result.moved); + assert!(!result.failed); + assert!(result.error.is_none()); + assert_eq!(backend.delete_calls(), 1); + } + + #[tokio::test] + async fn test_migrate_entry_version_reader_not_found_is_ignored() { + let backend = MigrationBackendSpy::new( + Some(Err(Error::ObjectNotFound("bucket".to_string(), "object.bin".to_string()))), + None, + None, + ); + let version = version_normal(); + let mut transfer = |_, _, _| async move { Ok(()) }; + + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(result.ignored); + assert!(result.cleanup_ignored); + assert!(!result.moved); + assert!(!result.failed); + assert!(result.error.is_none()); + assert_eq!(backend.get_calls(), 1); + assert_eq!(backend.delete_calls(), 0); + } + + #[tokio::test] + async fn test_migrate_entry_version_reader_retries_before_success() { + let backend = MigrationBackendSpy::new(Some(Err(Error::SlowDown)), None, None); + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + }; + + let version = version_normal(); + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(!result.ignored); + assert!(!result.cleanup_ignored); + assert!(result.moved); + assert!(!result.failed); + assert!(result.error.is_none()); + assert_eq!(backend.get_calls(), 2); + assert_eq!(backend.delete_calls(), 0); + assert_eq!(transfer_count.load(Ordering::SeqCst), 1); + } + + struct AlwaysFailGetBackend { + get_calls: AtomicUsize, + } + + impl AlwaysFailGetBackend { + fn new() -> Self { + Self { + get_calls: AtomicUsize::new(0), + } + } + + fn get_calls(&self) -> usize { + self.get_calls.load(Ordering::SeqCst) + } + } + + #[async_trait::async_trait] + impl MigrationBackend for AlwaysFailGetBackend { + async fn get_object_reader_for_migration( + &self, + _bucket: &str, + _object: &str, + _range: Option, + _h: http::HeaderMap, + _opts: &ObjectOptions, + ) -> Result { + self.get_calls.fetch_add(1, Ordering::SeqCst); + Err(Error::SlowDown) + } + + async fn delete_object_for_migration(&self, _bucket: &str, _object: &str, _opts: ObjectOptions) -> Result { + Ok(ObjectInfo::default()) + } + + async fn move_remote_version_for_migration( + &self, + _bucket: &str, + _object: &str, + _fi: &FileInfo, + _opts: &ObjectOptions, + ) -> Result<()> { + Ok(()) + } + } + + #[tokio::test] + async fn test_migrate_entry_version_reader_fails_after_retries() { + let backend = AlwaysFailGetBackend::new(); + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + }; + + let version = version_normal(); + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(!result.ignored); + assert!(!result.cleanup_ignored); + assert!(!result.moved); + assert!(result.failed); + assert!(matches!(result.error, Some(Error::SlowDown))); + assert_eq!(backend.get_calls(), 3); + assert_eq!(transfer_count.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn test_migrate_entry_version_zero_max_attempts_still_attempts_once() { + let backend = AlwaysFailGetBackend::new(); + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + }; + + let version = version_normal(); + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 0, + false, + &mut transfer, + ) + .await; + + assert!(!result.ignored); + assert!(!result.cleanup_ignored); + assert!(!result.moved); + assert!(result.failed); + assert!(matches!(result.error, Some(Error::SlowDown))); + assert_eq!(backend.get_calls(), 1); + assert_eq!(transfer_count.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn test_migrate_entry_version_transfer_retries_before_success() { + let backend = MigrationBackendSpy::new(Some(Ok(MigrationBackendSpy::make_reader())), None, None); + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + let attempt = transfer_count.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + return Err(Error::SlowDown); + } + Ok(()) + } + } + }; + + let version = version_normal(); + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(!result.ignored); + assert!(!result.cleanup_ignored); + assert!(result.moved); + assert!(!result.failed); + assert_eq!(backend.get_calls(), 2); + assert_eq!(transfer_count.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn test_migrate_entry_version_transfer_fails_after_retries() { + let backend = MigrationBackendSpy::new(Some(Ok(MigrationBackendSpy::make_reader())), None, None); + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Err(Error::NotModified) + } + } + }; + + let version = version_normal(); + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 2, + false, + &mut transfer, + ) + .await; + + assert!(result.failed); + assert!(!result.ignored); + assert!(!result.cleanup_ignored); + assert!(!result.moved); + assert!(result.error.is_some()); + assert_eq!(backend.get_calls(), 2); + assert_eq!(transfer_count.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn test_migrate_entry_version_transfer_not_found_is_ignored() { + let backend = MigrationBackendSpy::new(Some(Ok(MigrationBackendSpy::make_reader())), None, None); + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Err(Error::ObjectNotFound("bucket".to_string(), "object.bin".to_string())) + } + } + }; + + let version = version_normal(); + let result = migrate_entry_version( + &backend, + "bucket".to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 3, + false, + &mut transfer, + ) + .await; + + assert!(result.ignored); + assert!(result.cleanup_ignored); + assert!(!result.moved); + assert!(!result.failed); + assert!(result.error.is_none()); + assert_eq!(transfer_count.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn test_migrate_entry_version_ignores_data_usage_cache_when_enabled() { + let backend = MigrationBackendSpy::new(Some(Ok(MigrationBackendSpy::make_reader())), None, None); + let version = { + let mut version = version_normal(); + version.name = format!("{}.{}", DATA_USAGE_CACHE_NAME, version.name); + version + }; + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + }; + + let result = migrate_entry_version( + &backend, + RUSTFS_META_BUCKET.to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 2, + true, + &mut transfer, + ) + .await; + + assert!(result.ignored); + assert!(!result.cleanup_ignored); + assert!(!result.moved); + assert!(!result.failed); + assert!(result.error.is_none()); + assert_eq!(transfer_count.load(Ordering::SeqCst), 0); + assert_eq!(backend.get_calls(), 0); + assert_eq!(backend.delete_calls(), 0); + } + + #[tokio::test] + async fn test_migrate_entry_version_data_usage_cache_moves_when_ignore_disabled() { + let backend = MigrationBackendSpy::new(Some(Ok(MigrationBackendSpy::make_reader())), None, None); + let version = { + let mut version = version_normal(); + version.name = format!("{}.{}", DATA_USAGE_CACHE_NAME, version.name); + version + }; + let transfer_count = Arc::new(AtomicUsize::new(0)); + let mut transfer = { + let transfer_count = transfer_count.clone(); + move |_, _, _| { + let transfer_count = transfer_count.clone(); + async move { + transfer_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + }; + + let result = migrate_entry_version( + &backend, + RUSTFS_META_BUCKET.to_string(), + 1, + &version, + version.version_id.map(|v| v.to_string()), + 2, + false, + &mut transfer, + ) + .await; + + assert!(!result.ignored); + assert!(!result.cleanup_ignored); + assert!(result.moved); + assert!(!result.failed); + assert!(result.error.is_none()); + assert_eq!(transfer_count.load(Ordering::SeqCst), 1); + assert_eq!(backend.get_calls(), 1); + assert_eq!(backend.delete_calls(), 0); + } + + #[test] + fn test_should_ignore_rebalance_data_usage_cache_true_for_meta_bucket() { + assert!(should_ignore_rebalance_data_usage_cache(RUSTFS_META_BUCKET)); + } + + #[test] + fn test_should_ignore_rebalance_data_usage_cache_false_for_regular_bucket() { + assert!(!should_ignore_rebalance_data_usage_cache("bucket-a")); + } + + #[test] + fn test_rebalance_goal_reached_exact_tolerance_bound() { + let init_free_space = 200_u64; + let init_capacity = 1_000_u64; + let goal = 0.45_f64; + + // goal - 0.05 => 200 + 200 = 400 free / 1000 => 0.4 exactly + assert!(rebalance_goal_reached(init_free_space, init_capacity, 200, goal)); + + // one byte above the tolerance boundary should be false + assert!(!rebalance_goal_reached(init_free_space, init_capacity, 199, goal)); + } + + #[test] + fn test_rebalance_goal_reached_within_tolerance() { + let init_free_space = 200_u64; + let init_capacity = 1_000_u64; + let bytes = 250_u64; + let goal = 0.45_f64; + + assert!(rebalance_goal_reached(init_free_space, init_capacity, bytes, goal)); + } + + #[test] + fn test_rebalance_goal_not_reached_outside_tolerance() { + let init_free_space = 100_u64; + let init_capacity = 1_000_u64; + let bytes = 80_u64; + let goal = 0.5_f64; + + assert!(!rebalance_goal_reached(init_free_space, init_capacity, bytes, goal)); + } + + #[test] + fn test_rebalance_goal_zero_capacity_is_false() { + assert!(!rebalance_goal_reached(100, 0, 50, 0.5)); + } + + #[test] + fn test_rebalance_goal_above_one_is_true_when_within_tolerance() { + assert!(rebalance_goal_reached(950, 1_000, 100, 1.0)); + } + + #[test] + fn test_rebalance_goal_below_zero_is_true_when_within_tolerance() { + assert!(rebalance_goal_reached(10, 1_000, 0, -0.01)); + } + + #[test] + fn test_resolve_rebalance_worker_result_passthrough() { + assert!(resolve_rebalance_worker_result(0, Ok(Ok(()))).is_ok()); + + let err = resolve_rebalance_worker_result(0, Ok(Err(Error::OperationCanceled))).unwrap_err(); + assert!(matches!(err, Error::OperationCanceled)); + } + + #[tokio::test] + async fn test_resolve_rebalance_worker_result_join_error_keeps_context() { + let join_error = tokio::spawn(async { + panic!("rebalance worker panic"); + }) + .await + .expect_err("panic task should return JoinError"); + + let err = resolve_rebalance_worker_result(7, Err(join_error)).unwrap_err(); + assert!(err.to_string().contains("rebalance worker 7 task join error")); + } + + #[test] + fn test_resolve_rebalance_save_task_result_passthrough() { + assert!(resolve_rebalance_save_task_result(0, Ok(Ok(()))).is_ok()); + } + + #[test] + fn test_resolve_rebalance_save_task_result_wraps_inner_error_context() { + let err = resolve_rebalance_save_task_result(1, Ok(Err(Error::SlowDown))) + .expect_err("inner save-task error should include pool context"); + assert!(err.to_string().contains("rebalance save_task failed for pool 1")); + } + + #[test] + fn test_resolve_rebalance_meta_save_result_passthrough() { + assert!(resolve_rebalance_meta_save_result(Ok(()), "stop_rebalance").is_ok()); + } + + #[test] + fn test_resolve_rebalance_meta_save_result_wraps_error_context() { + let err = resolve_rebalance_meta_save_result(Err(Error::SlowDown), "init_rebalance_meta") + .expect_err("meta save failure should include stage context"); + let message = err.to_string(); + assert!(message.contains("rebalance meta save failed during init_rebalance_meta")); + assert!(message.contains(Error::SlowDown.to_string().as_str())); + } + + #[test] + fn test_rebalance_meta_load_no_data_error_formats_context() { + let err = rebalance_meta_load_no_data_error(); + let rendered = err.to_string(); + + assert!(rendered.contains("rebalance metadata load failed"), "{rendered}"); + assert!(rendered.contains("payload is too short"), "{rendered}"); + } + + #[test] + fn test_rebalance_meta_load_unknown_format_error_formats_context() { + let err = rebalance_meta_load_unknown_format_error(9); + let rendered = err.to_string(); + + assert!(rendered.contains("rebalance metadata load failed"), "{rendered}"); + assert!(rendered.contains("unknown format 9"), "{rendered}"); + } + + #[test] + fn test_rebalance_meta_load_unknown_version_error_formats_context() { + let err = rebalance_meta_load_unknown_version_error(3); + let rendered = err.to_string(); + + assert!(rendered.contains("rebalance metadata load failed"), "{rendered}"); + assert!(rendered.contains("unknown version 3"), "{rendered}"); + } + + #[test] + fn test_resolve_rebalance_stats_update_result_passthrough() { + assert!(resolve_rebalance_stats_update_result(Ok(()), 0, "bucket", "object").is_ok()); + } + + #[test] + fn test_resolve_rebalance_stats_update_result_wraps_error_context() { + let err = resolve_rebalance_stats_update_result(Err(Error::SlowDown), 2, "bucket-a", "obj.txt") + .expect_err("stats update error should include context"); + assert!( + err.to_string() + .contains("rebalance stats update failed for pool 2 bucket bucket-a object obj.txt") + ); + } + + #[test] + fn test_resolve_load_rebalance_stats_update_result_passthrough() { + assert!(resolve_load_rebalance_stats_update_result(Ok(())).is_ok()); + } + + #[test] + fn test_resolve_load_rebalance_stats_update_result_wraps_error_context() { + let err = resolve_load_rebalance_stats_update_result(Err(Error::SlowDown)) + .expect_err("load-time stats refresh failure should include context"); + assert!(err.to_string().contains("rebalance metadata stats refresh failed after load")); + } + + #[test] + fn test_resolve_rebalance_file_info_versions_result_passthrough() { + let value = resolve_rebalance_file_info_versions_result::(Ok(7), "bucket-a", "obj.txt") + .expect("ok results should pass through"); + assert_eq!(value, 7); + } + + #[test] + fn test_resolve_rebalance_file_info_versions_result_wraps_error_context() { + let err = resolve_rebalance_file_info_versions_result::(Err(Error::SlowDown), "bucket-a", "obj.txt") + .expect_err("errors should be wrapped"); + let message = err.to_string(); + assert!(message.contains("rebalance file_info_versions failed for bucket-a/obj.txt")); + } + + #[test] + fn test_resolve_rebalance_meta_load_result_returns_true_for_loaded_meta() { + assert!(resolve_rebalance_meta_load_result(Ok(())).expect("loaded rebalance metadata should pass through")); + } + + #[test] + fn test_resolve_rebalance_meta_load_result_returns_false_for_missing_meta() { + assert!( + !resolve_rebalance_meta_load_result(Err(Error::ConfigNotFound)) + .expect("missing rebalance metadata should be treated as not started") + ); + } + + #[test] + fn test_resolve_rebalance_meta_load_result_wraps_error_context() { + let err = + resolve_rebalance_meta_load_result(Err(Error::SlowDown)).expect_err("unexpected load failures should be wrapped"); + let message = err.to_string(); + assert!(message.contains("rebalance metadata load failed during load_rebalance_meta")); + } + + #[test] + fn test_resolve_rebalance_entry_cleanup_delete_result_passthrough() { + let result = resolve_rebalance_entry_cleanup_delete_result(Ok(ObjectInfo::default()), "bucket-a", "obj.txt"); + assert!(result.is_ok()); + } + + #[test] + fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_not_found() { + let result = resolve_rebalance_entry_cleanup_delete_result( + Err(Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string())), + "bucket-a", + "obj.txt", + ); + assert!(result.is_ok()); + } + + #[test] + fn test_resolve_rebalance_entry_cleanup_delete_result_wraps_error_context() { + let err = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown), "bucket-a", "obj.txt") + .expect_err("unexpected cleanup errors should be wrapped"); + let message = err.to_string(); + assert!(message.contains("rebalance cleanup delete failed for bucket-a/obj.txt")); + } + + #[test] + fn test_resolve_rebalance_migrate_result_error_preserves_inner_error() { + let err = resolve_rebalance_migrate_result_error(Some(Error::SlowDown), 2, "bucket-a", "obj.txt", Some("vid-1")); + assert!(matches!(err, Error::SlowDown)); + } + + #[test] + fn test_resolve_rebalance_migrate_result_error_wraps_missing_error_context() { + let err = resolve_rebalance_migrate_result_error(None, 2, "bucket-a", "obj.txt", Some("vid-1")); + let message = err.to_string(); + assert!( + message + .contains("rebalance migration reported failure without error for pool 2 entry bucket-a/obj.txt version vid-1"), + "{message}" + ); + } + + #[test] + fn test_resolve_rebalance_bucket_result_passthrough() { + assert!(resolve_rebalance_bucket_result(Ok(()), 2, "bucket-a").is_ok()); + } + + #[test] + fn test_resolve_rebalance_bucket_result_preserves_operation_canceled() { + let err = resolve_rebalance_bucket_result(Err(Error::OperationCanceled), 2, "bucket-a") + .expect_err("operation canceled should be preserved"); + assert!(matches!(err, Error::OperationCanceled)); + } + + #[test] + fn test_resolve_rebalance_bucket_result_wraps_not_initialized_with_context() { + let err = resolve_rebalance_bucket_result(Err(Error::other("errServerNotInitialized")), 2, "bucket-a") + .expect_err("not initialized should be surfaced with context"); + let message = err.to_string(); + assert!(message.contains("rebalance bucket bucket-a failed for pool 2")); + assert!(message.contains("errServerNotInitialized")); + } + + #[test] + fn test_rebalance_listing_disks_available_rejects_empty_set() { + let err = ensure_rebalance_listing_disks_available(false, "bucket-a") + .expect_err("missing online disks should be reported with bucket context"); + assert!( + err.to_string() + .contains("failed to list objects to rebalance for bucket bucket-a: no disks available") + ); + } + + #[test] + fn test_rebalance_listing_disks_available_allows_online_disks() { + assert!(ensure_rebalance_listing_disks_available(true, "bucket-a").is_ok()); + } + + #[test] + fn test_with_rebalance_entry_context_formats_stage_bucket_and_object() { + let err = with_rebalance_entry_context("migrate", "bucket-a", "obj.txt", Error::SlowDown); + let message = err.to_string(); + assert!(message.contains("rebalance entry migrate failed for bucket-a/obj.txt")); + assert!(message.contains("Please reduce your request rate")); + } + + #[test] + fn test_should_count_rebalance_version_complete_for_cleanup_safe_ignored_result() { + let result = MigrationVersionResult { + ignored: true, + cleanup_ignored: true, + ..Default::default() + }; + assert!(should_count_rebalance_version_complete(&result)); + } + + #[test] + fn test_should_count_rebalance_version_complete_rejects_skip_only_ignored_result() { + let result = MigrationVersionResult { + ignored: true, + cleanup_ignored: false, + ..Default::default() + }; + assert!(!should_count_rebalance_version_complete(&result)); + } + + #[test] + fn test_should_count_rebalance_version_complete_for_moved_result() { + let result = MigrationVersionResult { + moved: true, + ..Default::default() + }; + assert!(should_count_rebalance_version_complete(&result)); + } + + #[test] + fn test_should_count_rebalance_version_complete_rejects_failed_result() { + let result = MigrationVersionResult { + moved: true, + failed: true, + ..Default::default() + }; + assert!(!should_count_rebalance_version_complete(&result)); + } + + #[test] + fn test_should_count_rebalance_version_complete_rejects_incomplete_result() { + assert!(!should_count_rebalance_version_complete(&MigrationVersionResult::default())); + } + + #[test] + fn test_should_skip_rebalance_delete_marker_when_last_remaining_without_replication() { + assert!(should_skip_rebalance_delete_marker(&version_deleted(), 1, false)); + } + + #[test] + fn test_should_skip_rebalance_delete_marker_rejects_configured_replication() { + assert!(!should_skip_rebalance_delete_marker(&version_deleted(), 1, true)); + } + + #[test] + fn test_should_skip_rebalance_delete_marker_rejects_non_deleted_versions() { + assert!(!should_skip_rebalance_delete_marker(&version_normal(), 1, false)); + } + + #[test] + fn test_should_skip_rebalance_delete_marker_rejects_multiple_remaining_versions() { + assert!(!should_skip_rebalance_delete_marker(&version_deleted(), 2, false)); + } + + #[test] + fn test_should_cleanup_rebalance_source_entry_accepts_all_versions_completed() { + assert!(should_cleanup_rebalance_source_entry(3, 3)); + } + + #[test] + fn test_should_cleanup_rebalance_source_entry_rejects_versions_only_expired_by_lifecycle() { + assert!(!should_cleanup_rebalance_source_entry(2, 3)); + } + + #[test] + fn test_resolve_rebalance_optional_bucket_config_result_passthrough() { + let result = resolve_rebalance_optional_bucket_config_result( + "bucket-a", + "replication", + Ok((ReplicationConfiguration::default(), OffsetDateTime::UNIX_EPOCH)), + ) + .expect("bucket config should pass through"); + assert!(result.is_some()); + } + + #[test] + fn test_resolve_rebalance_optional_bucket_config_result_returns_none_for_missing_config() { + let result = resolve_rebalance_optional_bucket_config_result::<()>("bucket-a", "versioning", Err(Error::ConfigNotFound)) + .expect("missing bucket config should map to None"); + assert!(result.is_none()); + } + + #[test] + fn test_resolve_rebalance_optional_bucket_config_result_wraps_other_errors() { + let err = resolve_rebalance_optional_bucket_config_result::<()>("bucket-a", "replication", Err(Error::SlowDown)) + .expect_err("unexpected bucket config errors should be wrapped with context"); + assert!( + err.to_string() + .contains("rebalance replication config load failed for bucket bucket-a") + ); + } + + #[tokio::test] + async fn test_load_rebalance_bucket_configs_skips_meta_bucket_lookup() { + let configs = load_rebalance_bucket_configs(RUSTFS_META_BUCKET) + .await + .expect("meta bucket config loading should short-circuit"); + assert!(configs.lifecycle_config.is_none()); + assert!(configs.lock_retention.is_none()); + assert!(configs.replication_config.is_none()); + } + + #[tokio::test] + async fn test_resolve_rebalance_save_task_result_join_error_keeps_context() { + let join_error = tokio::spawn(async { + panic!("rebalance save task panic"); + }) + .await + .expect_err("panic task should return JoinError"); + + let err = resolve_rebalance_save_task_result(3, Err(join_error)).unwrap_err(); + assert!(err.to_string().contains("rebalance save_task for pool 3 join error")); + } + + #[tokio::test] + async fn test_send_rebalance_done_signal_sends_message() { + let (tx, mut rx) = mpsc::channel(1); + + send_rebalance_done_signal(&tx, Ok(()), 2) + .await + .expect("send should succeed when receiver is active"); + + let received = rx.recv().await.expect("receiver should get signal"); + assert!(received.is_ok()); + } + + #[tokio::test] + async fn test_send_rebalance_done_signal_reports_closed_channel() { + let (tx, rx) = mpsc::channel(1); + drop(rx); + + let err = send_rebalance_done_signal(&tx, Ok(()), 5) + .await + .expect_err("send should fail when receiver is closed"); + assert!(err.to_string().contains("rebalance done signal send failed for pool 5")); + } + + #[test] + fn test_resolve_rebalance_terminal_error_keeps_primary_when_signal_ok() { + let err = resolve_rebalance_terminal_error(Error::SlowDown, Ok(())); + assert!(matches!(err, Error::SlowDown)); + } + + #[test] + fn test_resolve_rebalance_terminal_error_wraps_signal_failure_context() { + let err = resolve_rebalance_terminal_error(Error::SlowDown, Err(Error::OperationCanceled)); + assert!(err.to_string().contains("rebalance terminal signal failed after error")); + } + + #[test] + fn test_resolve_rebalance_bucket_error_prefers_entry_error() { + let err = resolve_rebalance_bucket_error(Some(Error::OperationCanceled), Some(Error::SlowDown)).unwrap_err(); + assert!(matches!(err, Error::OperationCanceled)); + } + + #[test] + fn test_resolve_rebalance_bucket_error_uses_worker_error_when_entry_ok() { + let err = resolve_rebalance_bucket_error(None, Some(Error::SlowDown)).unwrap_err(); + assert!(matches!(err, Error::SlowDown)); + } + + #[test] + fn test_resolve_rebalance_bucket_error_is_ok_when_no_errors() { + assert!(resolve_rebalance_bucket_error(None, None).is_ok()); + } + + #[test] + fn test_ensure_valid_rebalance_pool_index_allows_in_range() { + assert!(ensure_valid_rebalance_pool_index(3, 2).is_ok()); + } + + #[test] + fn test_ensure_valid_rebalance_pool_index_rejects_out_of_range() { + let err = ensure_valid_rebalance_pool_index(2, 2).expect_err("out of range index should fail"); + assert!(err.to_string().contains("invalid rebalance pool index")); + } + + #[test] + fn test_clone_first_arc_returns_first_value() { + let values = vec![Arc::new(7_u8), Arc::new(9_u8)]; + let first = clone_first_arc(values.as_slice(), "empty values").expect("first value should be returned"); + assert_eq!(*first, 7_u8); + } + + #[test] + fn test_clone_first_arc_rejects_empty_values() { + let values: Vec> = Vec::new(); + let err = clone_first_arc(values.as_slice(), "empty values").expect_err("empty values should fail"); + assert!(err.to_string().contains("empty values")); + } + + #[test] + fn test_clone_arc_by_index_returns_value() { + let values = vec![Arc::new(7_u8), Arc::new(9_u8)]; + let value = + clone_arc_by_index(values.as_slice(), 1, "invalid rebalance pool index").expect("index within bounds should work"); + assert_eq!(*value, 9_u8); + } + + #[test] + fn test_clone_arc_by_index_rejects_out_of_range() { + let values = vec![Arc::new(7_u8)]; + let err = + clone_arc_by_index(values.as_slice(), 2, "invalid rebalance pool index").expect_err("out of range index should fail"); + assert!(err.to_string().contains("invalid rebalance pool index: 2")); + } + + #[test] + fn test_classify_rebalance_terminal_event_completed() { + let now = OffsetDateTime::now_utc(); + match classify_rebalance_terminal_event(Some(Ok(())), now) { + RebalanceTerminalEvent::Completed { msg } => assert!(msg.contains("Rebalance completed")), + _ => panic!("expected completed terminal event"), + } + } + + #[test] + fn test_classify_rebalance_terminal_event_stopped() { + let now = OffsetDateTime::now_utc(); + match classify_rebalance_terminal_event(Some(Err(Error::OperationCanceled)), now) { + RebalanceTerminalEvent::Stopped { msg } => assert!(msg.contains("Rebalance stopped")), + _ => panic!("expected stopped terminal event"), + } + } + + #[test] + fn test_classify_rebalance_terminal_event_failed() { + let now = OffsetDateTime::now_utc(); + match classify_rebalance_terminal_event(Some(Err(Error::SlowDown)), now) { + RebalanceTerminalEvent::Failed { msg, last_error } => { + assert!(msg.contains("Rebalance failed")); + assert!(msg.contains("with err")); + assert_eq!(last_error, Error::SlowDown.to_string()); + } + _ => panic!("expected failed terminal event"), + } + } + + #[test] + fn test_classify_rebalance_terminal_event_channel_closed() { + let now = OffsetDateTime::now_utc(); + match classify_rebalance_terminal_event(None, now) { + RebalanceTerminalEvent::ChannelClosed { msg, last_error } => { + assert!(msg.contains("channel closed")); + assert!(last_error.contains("before terminal event")); + assert!(msg.contains("at")); + assert!(last_error.contains("at")); + } + _ => panic!("expected channel closed terminal event"), + } + } + + #[test] + fn test_apply_rebalance_terminal_event_channel_closed_marks_failed() { + let now = OffsetDateTime::now_utc(); + let mut status = RebalStatus::Started; + let mut end_time = None; + let mut last_error = None; + + apply_rebalance_terminal_event( + &mut status, + &mut end_time, + &mut last_error, + RebalanceTerminalEvent::ChannelClosed { + msg: "channel closed".to_string(), + last_error: "rebalance save channel closed before terminal event".to_string(), + }, + now, + ); + + assert_eq!(status, RebalStatus::Failed); + assert_eq!(end_time, Some(now)); + assert_eq!(last_error.as_deref(), Some("rebalance save channel closed before terminal event")); + } + + #[test] + fn test_apply_rebalance_terminal_event_stopped_clears_error() { + let now = OffsetDateTime::now_utc(); + let mut status = RebalStatus::Started; + let mut end_time = None; + let mut last_error = Some("old-error".to_string()); + + apply_rebalance_terminal_event( + &mut status, + &mut end_time, + &mut last_error, + RebalanceTerminalEvent::Stopped { + msg: "rebalance stopped".to_string(), + }, + now, + ); + + assert_eq!(status, RebalStatus::Stopped); + assert_eq!(end_time, Some(now)); + assert_eq!(last_error, None); + } + + #[test] + fn test_is_rebalance_stopped_terminal_event_only_matches_stopped_variant() { + let stopped = RebalanceTerminalEvent::Stopped { + msg: "stopped".to_string(), + }; + let completed = RebalanceTerminalEvent::Completed { + msg: "completed".to_string(), + }; + + assert!(is_rebalance_stopped_terminal_event(&stopped)); + assert!(!is_rebalance_stopped_terminal_event(&completed)); + } + + #[test] + fn test_should_preserve_rebalance_stopped_state_when_meta_marked_stopped() { + let event = RebalanceTerminalEvent::Completed { + msg: "completed".to_string(), + }; + + assert!(should_preserve_rebalance_stopped_state(true, RebalStatus::Started, &event)); + } + + #[test] + fn test_should_preserve_rebalance_stopped_state_when_pool_already_stopped() { + let event = RebalanceTerminalEvent::Failed { + msg: "failed".to_string(), + last_error: "boom".to_string(), + }; + + assert!(should_preserve_rebalance_stopped_state(false, RebalStatus::Stopped, &event)); + } + + #[test] + fn test_should_preserve_rebalance_stopped_state_allows_stopped_terminal_update() { + let event = RebalanceTerminalEvent::Stopped { + msg: "stopped".to_string(), + }; + + assert!(!should_preserve_rebalance_stopped_state(true, RebalStatus::Started, &event)); + } + + #[test] + fn test_ensure_rebalance_not_decommissioning_rejects_running_decommission() { + assert!(!ensure_rebalance_not_decommissioning(true)); + } + + #[test] + fn test_ensure_rebalance_not_decommissioning_allows_idle_decommission() { + assert!(ensure_rebalance_not_decommissioning(false)); + } + + #[test] + fn test_validate_start_rebalance_state_rejects_running_decommission() { + let err = validate_start_rebalance_state(true, true).expect_err("running decommission should block rebalance start"); + assert!(matches!(err, Error::DecommissionAlreadyRunning)); + } + + #[test] + fn test_validate_start_rebalance_state_rejects_missing_meta() { + let err = validate_start_rebalance_state(false, false).expect_err("missing rebalance meta should fail"); + assert!(matches!(err, Error::ConfigNotFound)); + } + + #[test] + fn test_validate_start_rebalance_state_allows_loaded_meta() { + validate_start_rebalance_state(false, true).expect("loaded rebalance meta should allow start"); + } + + #[test] + fn test_percent_free_ratio_zero_capacity_is_zero() { + assert_eq!(percent_free_ratio(100, 0), 0.0); + } + + #[test] + fn test_percent_free_ratio_normal_case() { + assert_eq!(percent_free_ratio(250, 1_000), 0.25); + } + + #[test] + fn test_should_pool_participate_false_when_capacity_zero() { + assert!(!should_pool_participate(0, 0, 0.2)); + } + + #[test] + fn test_should_pool_participate_true_when_ratio_below_goal() { + assert!(should_pool_participate(200, 1_000, 0.3)); + } + + #[test] + fn test_should_pool_participate_false_when_ratio_meets_goal() { + assert!(!should_pool_participate(300, 1_000, 0.3)); + } + + #[test] + fn test_should_skip_start_rebalance_only_when_running_and_cancel_attached() { + assert!(should_skip_start_rebalance(true, true)); + assert!(!should_skip_start_rebalance(true, false)); + assert!(!should_skip_start_rebalance(false, true)); + assert!(!should_skip_start_rebalance(false, false)); + } + + #[test] + fn test_new_multipart_abort_flag_defaults_to_abort_enabled() { + let flag = data_movement::new_multipart_abort_flag(); + assert!(data_movement::should_abort_multipart_upload(&flag)); + } + + #[test] + fn test_mark_multipart_upload_completed_disables_abort_cleanup() { + let flag = data_movement::new_multipart_abort_flag(); + data_movement::mark_multipart_upload_completed(&flag); + assert!(!data_movement::should_abort_multipart_upload(&flag)); + } + + #[test] + fn test_decode_part_index_returns_none_when_absent() { + assert!(data_movement::decode_part_index(None).is_none()); + } + + #[test] + fn test_decode_part_index_returns_none_for_invalid_payload() { + let invalid = bytes::Bytes::from_static(b"not-a-valid-index"); + assert!(data_movement::decode_part_index(Some(&invalid)).is_none()); + } + + #[test] + fn test_decode_part_index_returns_some_for_valid_payload() { + let mut index = Index::new(); + index.add(0, 0).expect("first index entry should be accepted"); + index + .add(2_097_152, 2_097_152) + .expect("second index entry should advance totals"); + + let encoded = index.into_vec(); + let decoded = data_movement::decode_part_index(Some(&encoded)).expect("valid index payload should decode"); + + assert_eq!(decoded.total_uncompressed, 2_097_152); + assert_eq!(decoded.total_compressed, 2_097_152); + } + + #[test] + fn test_resolve_rebalance_participants_respects_runtime_pool_count() { + let now = OffsetDateTime::now_utc(); + let stats = vec![ + RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(now), + ..Default::default() + }, + ..Default::default() + }, + RebalanceStats { + participating: false, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(now), + ..Default::default() + }, + ..Default::default() + }, + RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(now), + ..Default::default() + }, + ..Default::default() + }, + ]; + + let participants = resolve_rebalance_participants(stats.as_slice(), 2); + assert_eq!(participants, vec![true, false]); + } + + #[test] + fn test_resolve_rebalance_participants_requires_started_status() { + let now = OffsetDateTime::now_utc(); + let stats = vec![ + RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Completed, + start_time: Some(now), + ..Default::default() + }, + ..Default::default() + }, + RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(now), + ..Default::default() + }, + ..Default::default() + }, + ]; + + let participants = resolve_rebalance_participants(stats.as_slice(), 2); + assert_eq!(participants, vec![false, true]); + } + + #[test] + fn test_is_rebalance_actively_running_requires_cancel_and_started_state() { + let now = OffsetDateTime::now_utc(); + let mut meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + start_time: Some(now), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + assert!(!is_rebalance_actively_running(&meta)); + meta.cancel = Some(CancellationToken::new()); + assert!(is_rebalance_actively_running(&meta)); + } + + #[test] + fn test_is_rebalance_in_progress_only_started_participants() { + let now = OffsetDateTime::now_utc(); + let meta = RebalanceMeta { + stopped_at: None, + pool_stats: vec![ + RebalanceStats { + participating: true, + info: RebalanceInfo { + start_time: Some(now), + status: RebalStatus::Completed, + ..Default::default() + }, + ..Default::default() + }, + RebalanceStats { + participating: true, + info: RebalanceInfo { + start_time: Some(now), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }, + ], + ..Default::default() + }; + + assert!(is_rebalance_in_progress(&meta)); + } + + #[test] + fn test_is_rebalance_conflicting_with_decommission_true_when_in_progress() { + let now = OffsetDateTime::now_utc(); + let meta = RebalanceMeta { + stopped_at: None, + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + start_time: Some(now), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + assert!(is_rebalance_conflicting_with_decommission(&meta)); + } + + #[test] + fn test_is_rebalance_conflicting_with_decommission_false_when_stopped() { + let now = OffsetDateTime::now_utc(); + let meta = RebalanceMeta { + stopped_at: Some(now), + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + start_time: Some(now), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + assert!(!is_rebalance_conflicting_with_decommission(&meta)); + } + + #[test] + fn test_is_rebalance_in_progress_stopped_takes_precedence() { + let now = OffsetDateTime::now_utc(); + let meta = RebalanceMeta { + stopped_at: Some(now), + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + start_time: Some(now), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + assert!(!is_rebalance_in_progress(&meta)); + } + + #[test] + fn test_first_rebalance_bucket_returns_first_name() { + let pool_stat = RebalanceStats { + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + ..Default::default() + }; + + assert_eq!(first_rebalance_bucket(&pool_stat), Some("bucket-a".to_string())); + } + + #[test] + fn test_first_rebalance_bucket_returns_none_when_empty() { + let pool_stat = RebalanceStats::default(); + + assert_eq!(first_rebalance_bucket(&pool_stat), None); + } + + #[test] + fn test_next_rebal_bucket_from_stat_respects_empty_queue() { + let pool_stat = RebalanceStats { + buckets: vec![], + ..Default::default() + }; + + assert_eq!(next_rebal_bucket_from_stat(&pool_stat), None); + } + + #[test] + fn test_next_rebal_bucket_from_stat_returns_first_bucket() { + let now = OffsetDateTime::now_utc(); + let pool_stat = RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(now), + ..Default::default() + }, + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + ..Default::default() + }; + + assert_eq!(next_rebal_bucket_from_stat(&pool_stat), Some("bucket-a".to_string())); + } + + #[test] + fn test_clone_rebalance_pool_stats_rejects_missing_meta() { + let err = clone_rebalance_pool_stats(None).expect_err("missing rebalance meta should fail"); + assert!( + err.to_string() + .contains("failed to clone rebalance pool stats: rebalance metadata not initialized") + ); + } + + #[test] + fn test_clone_rebalance_pool_stats_clones_entries() { + let meta = RebalanceMeta { + pool_stats: vec![RebalanceStats::default()], + ..Default::default() + }; + + let stats = clone_rebalance_pool_stats(Some(&meta)).expect("metadata should clone pool stats"); + assert_eq!(stats.len(), 1); + } + + #[test] + fn test_resolve_next_rebalance_bucket_rejects_missing_meta() { + let err = resolve_next_rebalance_bucket(None, 0).expect_err("missing meta should fail"); + assert!( + err.to_string() + .contains("failed to resolve next rebalance bucket: rebalance metadata not initialized") + ); + } + + #[test] + fn test_resolve_next_rebalance_bucket_rejects_invalid_pool_index() { + let meta = RebalanceMeta { + pool_stats: vec![RebalanceStats::default()], + ..Default::default() + }; + + let err = resolve_next_rebalance_bucket(Some(&meta), 3).expect_err("invalid pool index should fail"); + assert!(err.to_string().contains("invalid rebalance pool index 3 for 1 pools")); + } + + #[test] + fn test_resolve_next_rebalance_bucket_returns_none_for_completed_pool() { + let meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Completed, + ..Default::default() + }, + buckets: vec!["bucket-a".to_string()], + ..Default::default() + }], + ..Default::default() + }; + + let next = resolve_next_rebalance_bucket(Some(&meta), 0).expect("completed pool should return none"); + assert!(next.is_none()); + } + + #[test] + fn test_resolve_next_rebalance_bucket_returns_first_bucket_for_active_pool() { + let now = OffsetDateTime::now_utc(); + let meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(now), + ..Default::default() + }, + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + ..Default::default() + }], + ..Default::default() + }; + + let next = resolve_next_rebalance_bucket(Some(&meta), 0).expect("active pool should return first bucket"); + assert_eq!(next.as_deref(), Some("bucket-a")); + } + + #[test] + fn test_take_bucket_from_rebalance_queue_moves_bucket_and_keeps_remaining() { + let mut pool_stat = RebalanceStats { + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-a".to_string()], + rebalanced_buckets: Vec::new(), + ..Default::default() + }; + + assert!(take_bucket_from_rebalance_queue(&mut pool_stat, "bucket-a")); + assert_eq!(pool_stat.buckets, vec!["bucket-b".to_string()]); + assert_eq!(pool_stat.rebalanced_buckets, vec!["bucket-a".to_string(), "bucket-a".to_string()]); + } + + #[test] + fn test_take_bucket_from_rebalance_queue_no_match_keeps_queue() { + let mut pool_stat = RebalanceStats { + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + rebalanced_buckets: Vec::new(), + ..Default::default() + }; + + assert!(!take_bucket_from_rebalance_queue(&mut pool_stat, "bucket-c")); + assert_eq!(pool_stat.buckets, vec!["bucket-a".to_string(), "bucket-b".to_string()]); + assert!(pool_stat.rebalanced_buckets.is_empty()); + } + + #[test] + fn test_mark_rebalance_bucket_done_rejects_missing_meta() { + let err = mark_rebalance_bucket_done(None, 0, "bucket-a").expect_err("missing meta should fail"); + assert!( + err.to_string() + .contains("failed to mark rebalance bucket done: rebalance metadata not initialized") + ); + } + + #[test] + fn test_mark_rebalance_bucket_done_rejects_invalid_pool_index() { + let mut meta = RebalanceMeta { + pool_stats: vec![RebalanceStats::default()], + ..Default::default() + }; + + let err = mark_rebalance_bucket_done(Some(&mut meta), 3, "bucket-a").expect_err("invalid pool index should fail"); + assert!(err.to_string().contains("invalid rebalance pool index 3 for 1 pools")); + } + + #[test] + fn test_mark_rebalance_bucket_done_rejects_missing_bucket() { + let mut meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + buckets: vec!["bucket-a".to_string()], + ..Default::default() + }], + ..Default::default() + }; + + let err = mark_rebalance_bucket_done(Some(&mut meta), 0, "bucket-x").expect_err("missing bucket should fail"); + assert!( + err.to_string() + .contains("failed to mark rebalance bucket done: bucket bucket-x was not queued for pool 0") + ); + } + + #[test] + fn test_mark_rebalance_bucket_done_marks_bucket_as_rebalanced() { + let mut meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + rebalanced_buckets: Vec::new(), + ..Default::default() + }], + ..Default::default() + }; + + mark_rebalance_bucket_done(Some(&mut meta), 0, "bucket-a").expect("bucket in queue should be marked done"); + assert_eq!(meta.pool_stats[0].buckets, vec!["bucket-b".to_string()]); + assert_eq!(meta.pool_stats[0].rebalanced_buckets, vec!["bucket-a".to_string()]); + } + + #[test] + fn test_apply_stopped_at_transitions_started_pools_only() { + let now = OffsetDateTime::now_utc(); + let mut meta = RebalanceMeta { + pool_stats: vec![ + RebalanceStats { + info: RebalanceInfo { + status: RebalStatus::Started, + end_time: None, + last_error: Some("old".to_string()), + ..Default::default() + }, + ..Default::default() + }, + RebalanceStats { + info: RebalanceInfo { + status: RebalStatus::Failed, + end_time: Some(now), + last_error: Some("failed".to_string()), + ..Default::default() + }, + ..Default::default() + }, + ], + ..Default::default() + }; + + apply_stopped_at(&mut meta, now); + + assert_eq!(meta.stopped_at, Some(now)); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); + assert_eq!(meta.pool_stats[0].info.end_time, Some(now)); + assert_eq!(meta.pool_stats[0].info.last_error, None); + + assert_eq!(meta.pool_stats[1].info.status, RebalStatus::Failed); + assert_eq!(meta.pool_stats[1].info.end_time, Some(now)); + assert_eq!(meta.pool_stats[1].info.last_error.as_deref(), Some("failed")); + } + + #[test] + fn test_stop_rebalance_state_cancels_token_and_marks_stopped_when_in_progress() { + let now = OffsetDateTime::from_unix_timestamp(10_000).unwrap(); + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let mut meta = RebalanceMeta { + cancel: Some(cancel), + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + stop_rebalance_state(&mut meta, now); + + assert!(cancel_clone.is_cancelled()); + assert!(meta.cancel.is_none()); + assert_eq!(meta.stopped_at, Some(now)); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); + assert_eq!(meta.pool_stats[0].info.end_time, Some(now)); + } + + #[test] + fn test_stop_rebalance_state_clears_token_without_forcing_stopped_when_not_in_progress() { + let now = OffsetDateTime::from_unix_timestamp(20_000).unwrap(); + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let mut meta = RebalanceMeta { + cancel: Some(cancel), + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Completed, + end_time: Some(now), + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + stop_rebalance_state(&mut meta, now); + + assert!(cancel_clone.is_cancelled()); + assert!(meta.cancel.is_none()); + assert_eq!(meta.stopped_at, None); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Completed); + } + + #[test] + fn test_stop_rebalance_state_normalizes_started_pool_when_stopped_at_already_set() { + let stopped_at = OffsetDateTime::from_unix_timestamp(30_000).unwrap(); + let now = OffsetDateTime::from_unix_timestamp(40_000).unwrap(); + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let mut meta = RebalanceMeta { + cancel: Some(cancel), + stopped_at: Some(stopped_at), + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + last_error: Some("stale".to_string()), + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + stop_rebalance_state(&mut meta, now); + + assert!(cancel_clone.is_cancelled()); + assert!(meta.cancel.is_none()); + assert_eq!(meta.stopped_at, Some(stopped_at)); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); + assert_eq!(meta.pool_stats[0].info.end_time, Some(stopped_at)); + assert_eq!(meta.pool_stats[0].info.last_error, None); + } + + #[test] + fn test_stop_rebalance_meta_snapshot_returns_none_when_meta_missing() { + let now = OffsetDateTime::from_unix_timestamp(50_000).unwrap(); + assert!(stop_rebalance_meta_snapshot(None, now).is_none()); + } + + #[test] + fn test_stop_rebalance_meta_snapshot_stops_meta_and_returns_snapshot() { + let now = OffsetDateTime::from_unix_timestamp(60_000).unwrap(); + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let mut meta = RebalanceMeta { + cancel: Some(cancel), + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + let snapshot = stop_rebalance_meta_snapshot(Some(&mut meta), now).expect("snapshot should be returned for present meta"); + + assert!(cancel_clone.is_cancelled()); + assert!(meta.cancel.is_none()); + assert_eq!(meta.stopped_at, Some(now)); + assert_eq!(meta.last_refreshed_at, Some(now)); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); + + assert!(snapshot.cancel.is_none()); + assert_eq!(snapshot.stopped_at, Some(now)); + assert_eq!(snapshot.last_refreshed_at, Some(now)); + assert_eq!(snapshot.pool_stats[0].info.status, RebalStatus::Stopped); + assert_eq!(snapshot.pool_stats[0].info.end_time, Some(now)); + } + + #[test] + fn test_apply_rebalance_save_option_stats_keeps_pool_status_and_updates_refresh() { + let now = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let later = OffsetDateTime::from_unix_timestamp(2_000).unwrap(); + let mut meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(now), + ..Default::default() + }, + buckets: vec!["bucket-a".to_string()], + ..Default::default() + }], + last_refreshed_at: Some(now), + stopped_at: None, + ..Default::default() + }; + + apply_rebalance_save_option(&mut meta, 0, RebalSaveOpt::Stats, later); + + assert_eq!(meta.last_refreshed_at, Some(later)); + assert_eq!(meta.stopped_at, None); + assert_eq!(meta.pool_stats.len(), 1); + assert!(meta.pool_stats[0].participating); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started); + assert_eq!(meta.pool_stats[0].info.start_time, Some(now)); + assert_eq!(meta.pool_stats[0].buckets, vec!["bucket-a".to_string()]); + } + + #[test] + fn test_apply_rebalance_save_option_stopped_at_updates_refresh_and_statuses() { + let now = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let mut meta = RebalanceMeta { + pool_stats: vec![ + RebalanceStats { + info: RebalanceInfo { + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }, + RebalanceStats { + info: RebalanceInfo { + status: RebalStatus::Failed, + last_error: Some("previous failure".to_string()), + ..Default::default() + }, + ..Default::default() + }, + ], + ..Default::default() + }; + + apply_rebalance_save_option(&mut meta, 9_000, RebalSaveOpt::StoppedAt, now); + + assert_eq!(meta.stopped_at, Some(now)); + assert_eq!(meta.last_refreshed_at, Some(now)); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); + assert_eq!(meta.pool_stats[0].info.end_time, Some(now)); + assert!(meta.pool_stats[0].info.last_error.is_none()); + assert_eq!(meta.pool_stats[1].info.status, RebalStatus::Failed); + assert_eq!(meta.pool_stats[1].info.last_error.as_deref(), Some("previous failure")); + } + + #[test] + fn test_rebalance_stats_update_counts_and_bytes_growth() { + let mut stat = RebalanceStats { + bucket: "bucket-a".to_string(), + object: "obj-previous".to_string(), + num_objects: 3, + num_versions: 5, + bytes: 120, + ..Default::default() + }; + + let mut latest = FileInfo::new("object-1", 4, 2); + latest.name = "object-1".to_string(); + latest.is_latest = true; + latest.size = 300; + latest.deleted = false; + latest.mod_time = Some(OffsetDateTime::UNIX_EPOCH); + latest.version_id = None; + + let mut historical = FileInfo::new("object-1", 4, 2); + historical.name = "object-1".to_string(); + historical.is_latest = false; + historical.size = 128; + historical.deleted = false; + historical.mod_time = Some(OffsetDateTime::UNIX_EPOCH); + historical.version_id = None; + + let mut tombstone = FileInfo::new("object-1", 4, 2); + tombstone.name = "object-1".to_string(); + tombstone.is_latest = false; + tombstone.size = 64; + tombstone.deleted = true; + tombstone.mod_time = Some(OffsetDateTime::UNIX_EPOCH); + tombstone.version_id = None; + + stat.update("bucket-b".to_string(), &latest); + stat.update("bucket-b".to_string(), &historical); + stat.update("bucket-b".to_string(), &tombstone); + + assert_eq!(stat.bucket, "bucket-b"); + assert_eq!(stat.object, "object-1"); + assert_eq!(stat.num_objects, 4); + assert_eq!(stat.num_versions, 8); + let expected_bytes = 120_u64 + + (latest.size * (latest.erasure.data_blocks + latest.erasure.parity_blocks) as i64 + / latest.erasure.data_blocks as i64) as u64 + + (historical.size * (historical.erasure.data_blocks + historical.erasure.parity_blocks) as i64 + / historical.erasure.data_blocks as i64) as u64; + assert_eq!(stat.bytes, expected_bytes); + } + + #[test] + fn test_rebalance_stats_update_ignores_invalid_data_blocks() { + let mut stat = RebalanceStats { + bucket: "bucket-a".to_string(), + object: "obj-previous".to_string(), + num_objects: 1, + num_versions: 2, + bytes: 77, + ..Default::default() + }; + + let mut invalid = FileInfo::new("object-invalid", 0, 2); + invalid.name = "object-invalid".to_string(); + invalid.is_latest = true; + invalid.size = 256; + invalid.deleted = false; + invalid.mod_time = Some(OffsetDateTime::UNIX_EPOCH); + invalid.version_id = None; + + stat.update("bucket-z".to_string(), &invalid); + + assert_eq!(stat.bucket, "bucket-z"); + assert_eq!(stat.object, "object-invalid"); + assert_eq!(stat.num_objects, 2); + assert_eq!(stat.num_versions, 3); + assert_eq!(stat.bytes, 77); + } + + #[test] + fn test_rebalance_goal_reached_tolerance_and_regression() { + let init_free_space = 150_u64; + let init_capacity = 800_u64; + let goal = 0.35_f64; + + assert!(!rebalance_goal_reached(init_free_space, init_capacity, 0, goal)); + assert!(rebalance_goal_reached(init_free_space, init_capacity, 90, goal)); + assert!(!rebalance_goal_reached(init_free_space, init_capacity, 89, goal)); + } +} diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 4695f2233..dbfad0789 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -246,6 +246,19 @@ fn build_tiered_decommission_file_info( (updated, write_quorum) } +fn resolve_tiered_decommission_write_quorum_result( + errs: &[Option], + write_quorum: usize, + bucket: &str, + object: &str, +) -> Result<()> { + if let Some(err) = reduce_write_quorum_errs(errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + return Err(to_object_err(err.into(), vec![bucket, object])); + } + + Ok(()) +} + #[derive(Clone, Debug)] pub struct SetDisks { pub locker_owner: String, @@ -1200,10 +1213,7 @@ impl ObjectOperations for SetDisks { } } - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - return Err(err.into()); - } - Ok(()) + resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object) } #[tracing::instrument(skip(self))] @@ -2158,11 +2168,7 @@ impl SetDisks { } } - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - return Err(to_object_err(err.into(), vec![bucket, object])); - } - - Ok(()) + resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object) } } @@ -4463,6 +4469,31 @@ mod tests { assert_ne!(updated.erasure.distribution, original.erasure.distribution); } + #[test] + fn test_resolve_tiered_decommission_write_quorum_result_allows_successful_quorum() { + let errs = vec![None, None, Some(DiskError::DiskNotFound), None]; + + let result = resolve_tiered_decommission_write_quorum_result(&errs, 3, "bucket", "object"); + + assert!(result.is_ok()); + } + + #[test] + fn test_resolve_tiered_decommission_write_quorum_result_wraps_object_context() { + let errs = vec![ + Some(DiskError::DiskNotFound), + Some(DiskError::DiskNotFound), + Some(DiskError::DiskNotFound), + Some(DiskError::DiskNotFound), + ]; + + let err = resolve_tiered_decommission_write_quorum_result(&errs, 3, "bucket", "object").expect_err("expected error"); + let rendered = err.to_string(); + + assert!(rendered.contains("bucket"), "{rendered}"); + assert!(rendered.contains("object"), "{rendered}"); + } + #[test] fn test_should_prevent_write() { let oi = ObjectInfo { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 8c508f041..e01b7bdb6 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -13,8 +13,84 @@ // limitations under the License. use super::*; +use crate::error::is_err_decommission_running; use crate::global::is_first_cluster_node_local; +fn should_resume_local_decommission(endpoints: &EndpointServerPools, idx: usize) -> Result { + let pool = endpoints.as_ref().get(idx).ok_or_else(|| { + Error::other(format!( + "store init failed to resolve decommission resume pool index {idx} from current endpoints" + )) + })?; + let endpoint = pool.endpoints.as_ref().first().ok_or_else(|| { + Error::other(format!( + "store init failed to resolve decommission resume pool index {idx}: no endpoints available" + )) + })?; + + Ok(endpoint.is_local) +} + +const LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES: usize = 6; +const LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(60 * 3); +const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30); + +fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool { + matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES +} + +async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool { + tokio::select! { + _ = rx.cancelled() => false, + _ = tokio::time::sleep(delay) => true, + } +} + +fn resolve_store_init_stage_result(result: Result<()>, stage: &str) -> Result<()> { + result.map_err(|err| Error::other(format!("store init failed during {stage}: {err}"))) +} + +async fn resume_local_decommission_after_init(store: Arc, rx: CancellationToken, pool_indices: Vec) { + for attempt in 0..=LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES { + if rx.is_cancelled() { + return; + } + + match store.decommission(rx.clone(), pool_indices.clone()).await { + Ok(()) => return, + Err(err) if is_err_decommission_running(&err) => { + if let Err(spawn_err) = store + .spawn_decommission_routines(store.clone(), rx.clone(), pool_indices.clone()) + .await + { + error!( + "store init failed to resume decommission workers for pools {:?}: {}", + pool_indices, spawn_err + ); + } + return; + } + Err(err) if should_retry_local_decommission_resume(&err, attempt) => { + warn!( + "store init decommission resume missing config for pools {:?}, retry {}/{}: {}", + pool_indices, + attempt + 1, + LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES + 1, + err + ); + tokio::select! { + _ = rx.cancelled() => return, + _ = tokio::time::sleep(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY) => {} + } + } + Err(err) => { + error!("store init failed to resume decommission for pools {:?}: {}", pool_indices, err); + return; + } + } + } +} + impl ECStore { #[allow(clippy::new_ret_no_self)] #[instrument(level = "debug", skip(endpoint_pools))] @@ -87,7 +163,7 @@ impl ECStore { Ok(fm) => break Ok(fm), // Wrap the final error if we are giving up Err(e) if times >= 10 => { - break Err(Error::other(format!("can not get formats after {} retries, last error: {e}", times))); + break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}"))); } // Retrying so just drop the error Err(_) => {} @@ -118,10 +194,10 @@ impl ECStore { } if deployment_id != Some(fm.id) { - return Err(Error::other("deployment_id not same in one pool")); + return Err(Error::other("store init failed: deployment IDs do not match across pools")); } - if deployment_id.is_some() && deployment_id.unwrap().is_nil() { + if deployment_id.is_some_and(|id| id.is_nil()) { deployment_id = Some(Uuid::new_v4()); } @@ -152,7 +228,7 @@ impl ECStore { let decommission_cancelers = RwLock::new(vec![None; pools.len()]); let ec = Arc::new(ECStore { - id: deployment_id.unwrap(), + id: deployment_id.ok_or_else(|| Error::other("store init failed: deployment id is not initialized"))?, disk_map, pools, peer_sys, @@ -177,7 +253,7 @@ impl ECStore { sleep(Duration::from_secs(wait_sec)).await; if exit_count > 10 { - return Err(Error::other("ec init failed")); + return Err(Error::other("store init failed: init retry budget exhausted")); } exit_count += 1; @@ -197,13 +273,25 @@ impl ECStore { pub async fn init(self: &Arc, rx: CancellationToken) -> Result<()> { GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await; - if self.load_rebalance_meta().await.is_ok() { - self.start_rebalance().await; + resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?; + if self.rebalance_meta.read().await.is_some() { + resolve_store_init_stage_result(self.start_rebalance().await, "start_rebalance")?; } let mut meta = PoolMeta::default(); - meta.load(self.pools[0].clone(), self.pools.clone()).await?; + resolve_store_init_stage_result( + meta.load( + self.pools + .first() + .cloned() + .ok_or_else(|| Error::other("store init failed: no storage pools available"))?, + self.pools.clone(), + ) + .await, + "load_pool_meta", + )?; let update = meta.validate(self.pools.clone())?; + let endpoints = get_global_endpoints(); let should_persist_pool_meta = is_first_cluster_node_local().await; if !update { @@ -213,8 +301,10 @@ impl ECStore { } } else { let new_meta = PoolMeta::new(&self.pools, &meta); + // Only one local node should persist validated pool metadata here; otherwise + // distributed startup can race on the same lock and replay the prior init bug. if should_persist_pool_meta { - new_meta.save(self.pools.clone()).await?; + resolve_store_init_stage_result(new_meta.save(self.pools.clone()).await, "save_validated_pool_meta")?; } { let mut pool_meta = self.pool_meta.write().await; @@ -225,14 +315,12 @@ impl ECStore { let pools = meta.return_resumable_pools(); let mut pool_indices = Vec::with_capacity(pools.len()); - let endpoints = get_global_endpoints(); - for p in pools.iter() { if let Some(idx) = endpoints.get_pool_idx(&p.cmd_line) { pool_indices.push(idx); } else { return Err(Error::other(format!( - "unexpected state present for decommission status pool({}) not found", + "store init failed to resolve resumable decommission pool `{}` from current endpoints", p.cmd_line ))); } @@ -240,25 +328,14 @@ impl ECStore { if !pool_indices.is_empty() { let idx = pool_indices[0]; - if endpoints.as_ref()[idx].endpoints.as_ref()[0].is_local { + if should_resume_local_decommission(&endpoints, idx)? { let store = self.clone(); tokio::spawn(async move { - // wait 3 minutes for cluster init - tokio::time::sleep(Duration::from_secs(60 * 3)).await; - - if let Err(err) = store.decommission(rx.clone(), pool_indices.clone()).await { - if err == StorageError::DecommissionAlreadyRunning { - for i in pool_indices.iter() { - store.do_decommission_in_routine(rx.clone(), *i).await; - } - return; - } - - error!("store init decommission err: {}", err); - - // TODO: check config err + if !wait_for_local_decommission_resume_delay(&rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await { + return; } + resume_local_decommission_after_init(store, rx, pool_indices).await; }); } } @@ -284,3 +361,106 @@ impl ECStore { self.pools.len() == 1 } } + +#[cfg(test)] +mod tests { + use super::{ + LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, resolve_store_init_stage_result, should_resume_local_decommission, + should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay, + }; + use crate::{ + disk::endpoint::Endpoint, + endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, + error::StorageError, + }; + use std::time::Duration; + use tokio_util::sync::CancellationToken; + + #[test] + fn test_should_resume_local_decommission_respects_local_flag() { + let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("endpoint should parse"); + local_endpoint.is_local = true; + let endpoints = EndpointServerPools::from(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 1, + endpoints: Endpoints::from(vec![local_endpoint]), + cmd_line: "pool-0".to_string(), + platform: String::new(), + }]); + + assert!(should_resume_local_decommission(&endpoints, 0).expect("local endpoint should resume")); + } + + #[test] + fn test_should_resume_local_decommission_rejects_unresolvable_pool() { + let endpoints = EndpointServerPools::default(); + let err = should_resume_local_decommission(&endpoints, 0).expect_err("missing pool should error"); + assert_eq!( + err.to_string(), + "Io error: store init failed to resolve decommission resume pool index 0 from current endpoints" + ); + } + + #[test] + fn test_should_resume_local_decommission_rejects_missing_endpoint() { + let endpoints = EndpointServerPools::from(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 1, + endpoints: Endpoints::from(Vec::::new()), + cmd_line: "pool-0".to_string(), + platform: String::new(), + }]); + let err = should_resume_local_decommission(&endpoints, 0).expect_err("missing endpoint should error"); + assert_eq!( + err.to_string(), + "Io error: store init failed to resolve decommission resume pool index 0: no endpoints available" + ); + } + + #[test] + fn test_should_retry_local_decommission_resume_accepts_config_not_found_before_retry_limit() { + assert!(should_retry_local_decommission_resume(&StorageError::ConfigNotFound, 0)); + } + + #[test] + fn test_should_retry_local_decommission_resume_rejects_config_not_found_at_retry_limit() { + assert!(!should_retry_local_decommission_resume( + &StorageError::ConfigNotFound, + LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES + )); + } + + #[test] + fn test_should_retry_local_decommission_resume_rejects_non_config_errors() { + assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0)); + } + + #[test] + fn test_resolve_store_init_stage_result_passthrough_ok() { + resolve_store_init_stage_result(Ok(()), "load_rebalance_meta").expect("successful stage should pass through"); + } + + #[test] + fn test_resolve_store_init_stage_result_wraps_error_context() { + let err = resolve_store_init_stage_result(Err(StorageError::SlowDown), "start_rebalance") + .expect_err("failed stage should be wrapped"); + let err_message = err.to_string(); + assert!(err_message.contains("store init failed during start_rebalance")); + assert!(err_message.contains(&StorageError::SlowDown.to_string())); + } + + #[tokio::test] + async fn test_wait_for_local_decommission_resume_delay_returns_true_after_delay() { + let rx = CancellationToken::new(); + assert!(wait_for_local_decommission_resume_delay(&rx, Duration::from_millis(1)).await); + } + + #[tokio::test] + async fn test_wait_for_local_decommission_resume_delay_returns_false_when_cancelled() { + let rx = CancellationToken::new(); + rx.cancel(); + assert!(!wait_for_local_decommission_resume_delay(&rx, Duration::from_secs(1)).await); + } +} diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index a964556ea..e648e0e02 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -40,7 +40,97 @@ fn select_data_movement_target_pool( } } +fn latest_object_access_delete_marker_error( + bucket: &str, + object: &str, + info: &ObjectInfo, + opts: &ObjectOptions, +) -> Option { + if !info.delete_marker { + return None; + } + + Some(if opts.version_id.is_none() || opts.delete_marker { + to_object_err(StorageError::FileNotFound, vec![bucket, object]) + } else { + to_object_err(StorageError::MethodNotAllowed, vec![bucket, object]) + }) +} + +fn resolve_latest_object_access( + bucket: &str, + object: &str, + info: ObjectInfo, + idx: usize, + opts: &ObjectOptions, +) -> Result<(ObjectInfo, usize)> { + if let Some(err) = latest_object_access_delete_marker_error(bucket, object, &info, opts) { + return Err(err); + } + + Ok((info, idx)) +} + +fn version_aware_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions { + let mut lookup_opts = opts.clone(); + lookup_opts.no_lock = no_lock; + if lookup_opts.version_id.is_some() { + lookup_opts.metadata_chg = true; + } + + lookup_opts +} + +fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions { + let mut lookup_opts = version_aware_lookup_opts(opts, no_lock); + lookup_opts.skip_decommissioned = true; + lookup_opts.skip_rebalancing = true; + + lookup_opts +} + impl ECStore { + async fn get_latest_accessible_object_info_with_idx( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + ) -> Result<(ObjectInfo, usize)> { + let (info, idx) = self.get_latest_object_info_with_idx(bucket, object, opts).await?; + resolve_latest_object_access(bucket, object, info, idx, opts) + } + + pub(super) async fn select_data_movement_pool_idx( + &self, + bucket: &str, + object: &str, + size: i64, + opts: &ObjectOptions, + no_lock: bool, + ) -> Result { + match self + .get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock)) + .await + { + Ok((pinfo, _)) => Ok(pinfo.index), + Err(err) => { + if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { + return Err(err); + } + + self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull) + } + } + } + + fn resolve_decommission_target_pool_idx_result(result: Result, bucket: &str, object: &str) -> Result { + result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}"))) + } + + fn resolve_decommission_tiered_object_result(result: Result<()>, bucket: &str, object: &str) -> Result<()> { + result.map_err(|err| Error::other(format!("failed to decommission tiered object for {bucket}/{object}: {err}"))) + } + #[instrument(skip(self, fi, opts))] pub(crate) async fn decommission_tiered_object( &self, @@ -54,10 +144,26 @@ impl ECStore { let object = encode_dir_object(object); if self.single_pool() { - return Err(Error::other(format!("error decommissioning {bucket}/{object}"))); + return Self::resolve_decommission_tiered_object_result( + Err(Error::other("single pool deployments cannot decommission tiered objects")), + bucket, + &object, + ); } - let idx = self.get_pool_idx_no_lock(bucket, &object, fi.size).await?; + let idx = if opts.data_movement && opts.version_id.is_some() { + Self::resolve_decommission_target_pool_idx_result( + self.select_data_movement_pool_idx(bucket, &object, fi.size, opts, true).await, + bucket, + &object, + )? + } else { + Self::resolve_decommission_target_pool_idx_result( + self.get_pool_idx_no_lock(bucket, &object, fi.size).await, + bucket, + &object, + )? + }; if opts.data_movement && idx == opts.src_pool_idx { return Err(StorageError::DataMovementOverwriteErr( bucket.to_owned(), @@ -66,10 +172,14 @@ impl ECStore { )); } - self.pools[idx] - .get_disks_by_key(&object) - .decommission_tiered_object(bucket, &object, fi, opts) - .await + Self::resolve_decommission_tiered_object_result( + self.pools[idx] + .get_disks_by_key(&object) + .decommission_tiered_object(bucket, &object, fi, opts) + .await, + bucket, + &object, + ) } #[instrument(level = "debug", skip(self))] @@ -95,9 +205,9 @@ impl ECStore { opts.no_lock = true; - // TODO: check if DeleteMarker - let (_oi, idx) = self.get_latest_object_info_with_idx(bucket, &object, &opts).await?; - + let (_, idx) = self + .get_latest_accessible_object_info_with_idx(bucket, &object, &opts) + .await?; self.pools[idx] .get_object_reader(bucket, object.as_str(), range, h, &opts) .await @@ -119,7 +229,12 @@ impl ECStore { return self.pools[0].put_object(bucket, object.as_str(), data, opts).await; } - let idx = self.get_pool_idx(bucket, &object, data.size()).await?; + let idx = if opts.data_movement && opts.version_id.is_some() { + self.select_data_movement_pool_idx(bucket, &object, data.size(), opts, false) + .await? + } else { + self.get_pool_idx(bucket, &object, data.size()).await? + }; if opts.data_movement && idx == opts.src_pool_idx { return Err(StorageError::DataMovementOverwriteErr( @@ -144,8 +259,9 @@ impl ECStore { // TODO: nslock - let (info, _) = self.get_latest_object_info_with_idx(bucket, object.as_str(), opts).await?; - + let (info, _) = self + .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts) + .await?; opts.precondition_check(&info)?; Ok(info) } @@ -172,7 +288,11 @@ impl ECStore { // TODO: nslock - let pool_idx = self.get_pool_idx_no_lock(src_bucket, &src_object, src_info.size).await?; + let pool_idx = self + .get_pool_info_existing_with_opts(src_bucket, &src_object, &version_aware_lookup_opts(src_opts, true)) + .await? + .0 + .index; if cp_src_dst_same { if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id) @@ -233,8 +353,7 @@ impl ECStore { let object = encode_dir_object(object); let object = object.as_str(); - let mut gopts = opts.clone(); - gopts.no_lock = true; + let gopts = version_aware_lookup_opts(&opts, true); if opts.data_movement { let existing_pool_idx = self @@ -505,8 +624,12 @@ impl ECStore { return Ok(()); } - let idx = self - .get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default()) + let opts = ObjectOptions { + version_id: Some(version_id.to_string()), + ..Default::default() + }; + let (_, idx) = self + .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts) .await?; let _ = self.pools[idx].add_partial(bucket, object.as_str(), version_id).await; @@ -522,7 +645,7 @@ impl ECStore { //opts.skip_decommissioned = true; //opts.no_lock = true; - let idx = self.get_pool_idx_existing_with_opts(bucket, &object, opts).await?; + let (_, idx) = self.get_latest_accessible_object_info_with_idx(bucket, &object, opts).await?; self.pools[idx].transition_object(bucket, &object, opts).await } @@ -541,7 +664,9 @@ impl ECStore { //opts.skip_decommissioned = true; //opts.nolock = true; - let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?; + let (_, idx) = self + .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts) + .await?; self.pools[idx] .clone() @@ -564,7 +689,9 @@ impl ECStore { let mut opts = opts.clone(); opts.metadata_chg = true; - let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), &opts).await?; + let (_, idx) = self + .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts) + .await?; self.pools[idx].put_object_metadata(bucket, object.as_str(), &opts).await } @@ -577,8 +704,7 @@ impl ECStore { return self.pools[0].get_object_tags(bucket, object.as_str(), opts).await; } - let (oi, _) = self.get_latest_object_info_with_idx(bucket, &object, opts).await?; - + let (oi, _) = self.get_latest_accessible_object_info_with_idx(bucket, &object, opts).await?; Ok(oi.user_tags) } @@ -596,7 +722,9 @@ impl ECStore { return self.pools[0].put_object_tags(bucket, object.as_str(), tags, opts).await; } - let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?; + let (_, idx) = self + .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts) + .await?; self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await } @@ -629,7 +757,9 @@ impl ECStore { return self.pools[0].delete_object_tags(bucket, object.as_str(), opts).await; } - let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?; + let (_, idx) = self + .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts) + .await?; self.pools[idx].delete_object_tags(bucket, object.as_str(), opts).await } @@ -665,4 +795,193 @@ mod tests { let target = select_data_movement_target_pool(Ok(0), 1, false).unwrap(); assert_eq!(target, Some(0)); } + + #[test] + fn latest_object_access_delete_marker_error_returns_none_for_live_object() { + let info = ObjectInfo::default(); + let opts = ObjectOptions::default(); + + assert!(latest_object_access_delete_marker_error("bucket", "object", &info, &opts).is_none()); + } + + #[test] + fn latest_object_access_delete_marker_error_returns_not_found_without_version_id() { + let info = ObjectInfo { + delete_marker: true, + ..Default::default() + }; + let opts = ObjectOptions::default(); + + let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts) + .expect("delete marker should stop latest-object reads"); + + assert!(crate::error::is_err_object_not_found(&err)); + } + + #[test] + fn latest_object_access_delete_marker_error_returns_method_not_allowed_for_version_read() { + let info = ObjectInfo { + delete_marker: true, + ..Default::default() + }; + let opts = ObjectOptions { + version_id: Some("vid-1".to_string()), + ..Default::default() + }; + + let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts) + .expect("delete marker version reads should be rejected"); + + assert!(matches!(err, Error::MethodNotAllowed)); + } + + #[test] + fn latest_object_access_delete_marker_error_returns_not_found_for_delete_marker_lookup() { + let info = ObjectInfo { + delete_marker: true, + ..Default::default() + }; + let opts = ObjectOptions { + version_id: Some("vid-1".to_string()), + delete_marker: true, + ..Default::default() + }; + + let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts) + .expect("delete marker lookup should keep not-found semantics"); + + assert!(crate::error::is_err_object_not_found(&err)); + } + + #[test] + fn resolve_latest_object_access_returns_live_object_and_pool_idx() { + let info = ObjectInfo::default(); + let opts = ObjectOptions::default(); + + let (resolved, idx) = resolve_latest_object_access("bucket", "object", info, 7, &opts).unwrap(); + + assert_eq!(idx, 7); + assert!(!resolved.delete_marker); + } + + #[test] + fn resolve_latest_object_access_rejects_delete_marker_without_version_id() { + let info = ObjectInfo { + delete_marker: true, + ..Default::default() + }; + let opts = ObjectOptions::default(); + + let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err(); + + assert!(crate::error::is_err_object_not_found(&err)); + } + + #[test] + fn resolve_latest_object_access_rejects_delete_marker_version_read() { + let info = ObjectInfo { + delete_marker: true, + ..Default::default() + }; + let opts = ObjectOptions { + version_id: Some("vid-1".to_string()), + ..Default::default() + }; + + let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err(); + + assert!(matches!(err, Error::MethodNotAllowed)); + } + + #[test] + fn resolve_decommission_target_pool_idx_result_passthrough_ok() { + let idx = ECStore::resolve_decommission_target_pool_idx_result(Ok(3), "bucket", "object").unwrap(); + + assert_eq!(idx, 3); + } + + #[test] + fn resolve_decommission_target_pool_idx_result_wraps_error_context() { + let err = ECStore::resolve_decommission_target_pool_idx_result(Err(Error::other("boom")), "bucket", "object") + .expect_err("expected contextual error"); + let rendered = err.to_string(); + + assert!(rendered.contains("failed to select decommission target pool"), "{rendered}"); + assert!(rendered.contains("bucket"), "{rendered}"); + assert!(rendered.contains("object"), "{rendered}"); + assert!(rendered.contains("boom"), "{rendered}"); + } + + #[test] + fn resolve_decommission_tiered_object_result_passthrough_ok() { + ECStore::resolve_decommission_tiered_object_result(Ok(()), "bucket", "object") + .expect("successful decommission result should pass through"); + } + + #[test] + fn resolve_decommission_tiered_object_result_wraps_error_context() { + let err = ECStore::resolve_decommission_tiered_object_result(Err(Error::other("boom")), "bucket", "object") + .expect_err("expected contextual error"); + let rendered = err.to_string(); + + assert!(rendered.contains("failed to decommission tiered object"), "{rendered}"); + assert!(rendered.contains("bucket"), "{rendered}"); + assert!(rendered.contains("object"), "{rendered}"); + assert!(rendered.contains("boom"), "{rendered}"); + } + + #[test] + fn version_aware_lookup_opts_enables_version_aware_lookup() { + let opts = ObjectOptions { + version_id: Some("vid-1".to_string()), + ..Default::default() + }; + + let lookup_opts = version_aware_lookup_opts(&opts, true); + + assert!(lookup_opts.no_lock); + assert!(lookup_opts.metadata_chg); + assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1")); + } + + #[test] + fn version_aware_lookup_opts_keeps_latest_lookup_for_unversioned_requests() { + let lookup_opts = version_aware_lookup_opts(&ObjectOptions::default(), true); + + assert!(lookup_opts.no_lock); + assert!(!lookup_opts.metadata_chg); + assert!(lookup_opts.version_id.is_none()); + } + + #[test] + fn data_movement_pool_lookup_opts_enables_version_aware_lookup_and_skip_flags() { + let opts = ObjectOptions { + version_id: Some("vid-1".to_string()), + ..Default::default() + }; + + let lookup_opts = data_movement_pool_lookup_opts(&opts, false); + + assert!(!lookup_opts.no_lock); + assert!(lookup_opts.metadata_chg); + assert!(lookup_opts.skip_decommissioned); + assert!(lookup_opts.skip_rebalancing); + assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1")); + } + + #[test] + fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() { + let lookup_opts = data_movement_pool_lookup_opts( + &ObjectOptions { + version_id: Some("vid-1".to_string()), + ..Default::default() + }, + true, + ); + + assert!(lookup_opts.no_lock); + assert!(lookup_opts.metadata_chg); + assert!(lookup_opts.skip_decommissioned); + assert!(lookup_opts.skip_rebalancing); + } } diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index a437cd811..2779a125d 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -13,7 +13,133 @@ // limitations under the License. use super::*; -use crate::bucket::utils::is_meta_bucketname; + +struct LatestObjectInfoCandidate { + info: Option, + idx: usize, + err: Option, +} + +fn pool_lookup_not_found_error(bucket: &str, object: &str, opts: &ObjectOptions) -> Error { + let object = decode_dir_object(object); + + if let Some(version_id) = &opts.version_id { + StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), version_id.clone()) + } else { + StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned()) + } +} + +fn resolve_store_rebalance_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> { + result.map_err(|err| Error::other(format!("store rebalance pool meta reload failed during {stage}: {err}"))) +} + +fn resolve_rebalance_delete_from_all_pools_result(result: Result, bucket: &str, object: &str) -> Result { + result.map_err(|err| Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}"))) +} + +fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, pool_count: usize) -> Error { + Error::other(format!( + "failed to resolve rebalance disk set: pool index {pool_idx}, set index {set_idx}, pool count {pool_count}", + )) +} + +fn resolve_latest_object_info_candidates( + mut candidates: Vec, + bucket: &str, + object: &str, + opts: &ObjectOptions, +) -> Result<(ObjectInfo, usize)> { + candidates.sort_by(|a, b| { + let a_mod = if let Some(info) = &a.info { + info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) + } else { + OffsetDateTime::UNIX_EPOCH + }; + + let b_mod = if let Some(info) = &b.info { + info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) + } else { + OffsetDateTime::UNIX_EPOCH + }; + + if a_mod == b_mod { + return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less }; + } + + b_mod.cmp(&a_mod) + }); + + for candidate in candidates { + if let Some(info) = candidate.info { + return Ok((info, candidate.idx)); + } + + if let Some(err) = candidate.err + && !is_err_object_not_found(&err) + && !is_err_version_not_found(&err) + { + return Err(err); + } + } + + Err(pool_lookup_not_found_error(bucket, object, opts)) +} + +async fn build_server_pools_available_space( + bucket: &str, + size: i64, + n_sets: &[usize], + infos: &[Vec>], +) -> ServerPoolsAvailableSpace { + let mut server_pools = vec![PoolAvailableSpace::default(); infos.len()]; + + for (i, zinfo) in infos.iter().enumerate() { + if zinfo.is_empty() { + server_pools[i] = PoolAvailableSpace { + index: i, + ..Default::default() + }; + + continue; + } + + if !is_meta_bucketname(bucket) && !has_space_for(zinfo, size).await.unwrap_or_default() { + server_pools[i] = PoolAvailableSpace { + index: i, + ..Default::default() + }; + + continue; + } + + let mut available = 0; + let mut max_used_pct = 0; + for disk in zinfo.iter().flatten() { + if disk.total == 0 { + continue; + } + + available += disk.total - disk.used; + + let pct_used = disk.used * 100 / disk.total; + + if pct_used > max_used_pct { + max_used_pct = pct_used; + } + } + + available *= n_sets[i] as u64; + + server_pools[i] = PoolAvailableSpace { + index: i, + available, + max_used_pct, + } + } + + ServerPoolsAvailableSpace(server_pools) +} impl ECStore { #[instrument(level = "debug", skip(self))] @@ -72,7 +198,7 @@ impl ECStore { Ok(()) } - async fn get_available_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Option { + pub(super) async fn get_available_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Option { // // Return a random one first let mut server_pools = self.get_server_pools_available_space(bucket, object, size).await; @@ -102,67 +228,26 @@ impl ECStore { async fn get_server_pools_available_space(&self, bucket: &str, object: &str, size: i64) -> ServerPoolsAvailableSpace { let mut n_sets = vec![0; self.pools.len()]; let mut infos = vec![Vec::new(); self.pools.len()]; - - // TODO: add concurrency - for (idx, pool) in self.pools.iter().enumerate() { + let pool_inputs = join_all(self.pools.iter().enumerate().map(|(idx, pool)| async move { if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await { - continue; + return (idx, 0, Vec::new()); } - n_sets[idx] = pool.set_count; + let disk_infos = match pool.get_disks_by_key(object).get_disks(0, 0).await { + Ok(disks) => get_disk_infos(&disks).await, + Err(_) => Vec::new(), + }; - if let Ok(disks) = pool.get_disks_by_key(object).get_disks(0, 0).await { - let disk_infos = get_disk_infos(&disks).await; - infos[idx] = disk_infos; - } + (idx, pool.set_count, disk_infos) + })) + .await; + + for (idx, set_count, disk_infos) in pool_inputs { + n_sets[idx] = set_count; + infos[idx] = disk_infos; } - let mut server_pools = vec![PoolAvailableSpace::default(); self.pools.len()]; - for (i, zinfo) in infos.iter().enumerate() { - if zinfo.is_empty() { - server_pools[i] = PoolAvailableSpace { - index: i, - ..Default::default() - }; - - continue; - } - - if !is_meta_bucketname(bucket) && !has_space_for(zinfo, size).await.unwrap_or_default() { - server_pools[i] = PoolAvailableSpace { - index: i, - ..Default::default() - }; - - continue; - } - - let mut available = 0; - let mut max_used_pct = 0; - for disk in zinfo.iter().flatten() { - if disk.total == 0 { - continue; - } - - available += disk.total - disk.used; - - let pct_used = disk.used * 100 / disk.total; - - if pct_used > max_used_pct { - max_used_pct = pct_used; - } - } - - available *= n_sets[i] as u64; - - server_pools[i] = PoolAvailableSpace { - index: i, - available, - max_used_pct, - } - } - - ServerPoolsAvailableSpace(server_pools) + build_server_pools_available_space(bucket, size, &n_sets, &infos).await } pub(super) async fn is_suspended(&self, idx: usize) -> bool { @@ -349,7 +434,7 @@ impl ECStore { return Ok((def_pool, Vec::new())); } - Err(Error::ObjectNotFound(bucket.to_owned(), object.to_owned())) + Err(pool_lookup_not_found_error(bucket, object, opts)) } async fn pools_with_object(&self, pools: &[PoolObjInfo], opts: &ObjectOptions) -> Vec { @@ -393,27 +478,20 @@ impl ECStore { } let results = join_all(futures).await; - - struct IndexRes { - res: Option, - idx: usize, - err: Option, - } - - let mut idx_res = Vec::with_capacity(self.pools.len()); + let mut candidates = Vec::with_capacity(self.pools.len()); for (idx, result) in results.into_iter().enumerate() { match result { Ok(res) => { - idx_res.push(IndexRes { - res: Some(res), + candidates.push(LatestObjectInfoCandidate { + info: Some(res), idx, err: None, }); } Err(e) => { - idx_res.push(IndexRes { - res: None, + candidates.push(LatestObjectInfoCandidate { + info: None, idx, err: Some(e), }); @@ -421,53 +499,10 @@ impl ECStore { } } - // TODO: test order - idx_res.sort_by(|a, b| { - let a_mod = if let Some(o1) = &a.res { - o1.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) - } else { - OffsetDateTime::UNIX_EPOCH - }; - - let b_mod = if let Some(o2) = &b.res { - o2.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) - } else { - OffsetDateTime::UNIX_EPOCH - }; - - if a_mod == b_mod { - return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less }; - } - - b_mod.cmp(&a_mod) - }); - - for res in idx_res.into_iter() { - if let Some(obj) = res.res { - return Ok((obj, res.idx)); - } - - if let Some(err) = res.err - && !is_err_object_not_found(&err) - && !is_err_version_not_found(&err) - { - return Err(err); - } - - // TODO: delete marker - } - - let object = decode_dir_object(object); - - if opts.version_id.is_none() { - Err(StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned())) - } else { - Err(StorageError::VersionNotFound( - bucket.to_owned(), - object.to_owned(), - opts.version_id.clone().unwrap_or_default(), - )) - } + // Delete markers are returned as latest object infos here. Higher-level + // access paths are responsible for translating them into read/write + // semantics such as object-not-found or method-not-allowed. + resolve_latest_object_info_candidates(candidates, bucket, object, opts) } pub(super) async fn delete_object_from_all_pools( @@ -505,15 +540,18 @@ impl ECStore { } if let Some(e) = &derrs[0] { - return Err(e.clone()); + return resolve_rebalance_delete_from_all_pools_result(Err(e.clone()), bucket, object); } - Ok(objs[0].as_ref().unwrap().clone()) + resolve_rebalance_delete_from_all_pools_result(Ok(objs[0].as_ref().unwrap().clone()), bucket, object) } pub async fn reload_pool_meta(&self) -> Result<()> { let mut meta = PoolMeta::default(); - meta.load(self.pools[0].clone(), self.pools.clone()).await?; + resolve_store_rebalance_pool_meta_reload_result( + meta.load(self.pools[0].clone(), self.pools.clone()).await, + "reload_pool_meta", + )?; let mut pool_meta = self.pool_meta.write().await; *pool_meta = meta; @@ -670,7 +708,7 @@ impl ECStore { if pool_idx < self.pools.len() && set_idx < self.pools[pool_idx].disk_set.len() { self.pools[pool_idx].disk_set[set_idx].get_disks(0, 0).await } else { - Err(Error::other(format!("pool idx {pool_idx}, set idx {set_idx}, not found"))) + Err(rebalance_disk_set_lookup_error(pool_idx, set_idx, self.pools.len())) } } @@ -699,3 +737,216 @@ impl ECStore { Err(Error::DiskNotFound) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::disk::DiskInfo; + + fn object_info_with_mod_time(unix_ts: i64, delete_marker: bool) -> ObjectInfo { + ObjectInfo { + mod_time: Some(OffsetDateTime::from_unix_timestamp(unix_ts).unwrap()), + delete_marker, + ..Default::default() + } + } + + fn disk_info(total: u64, used: u64, free: u64) -> DiskInfo { + DiskInfo { + total, + used, + free, + free_inodes: 1_024, + ..Default::default() + } + } + + #[test] + fn resolve_latest_object_info_candidates_returns_latest_delete_marker() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_mod_time(10, false)), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_mod_time(20, true)), + idx: 1, + err: None, + }, + ]; + + let (info, idx) = + resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()).unwrap(); + + assert_eq!(idx, 1); + assert!(info.delete_marker); + } + + #[test] + fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_mod_time(10, false)), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_mod_time(10, false)), + idx: 1, + err: None, + }, + ]; + + let (_, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()).unwrap(); + + assert_eq!(idx, 1); + } + + #[test] + fn resolve_latest_object_info_candidates_returns_non_not_found_error() { + let err = resolve_latest_object_info_candidates( + vec![LatestObjectInfoCandidate { + info: None, + idx: 0, + err: Some(Error::ErasureReadQuorum), + }], + "bucket", + "object", + &ObjectOptions::default(), + ) + .unwrap_err(); + + assert_eq!(err, Error::ErasureReadQuorum); + } + + #[test] + fn resolve_latest_object_info_candidates_returns_version_not_found_for_versioned_lookups() { + let err = resolve_latest_object_info_candidates( + vec![LatestObjectInfoCandidate { + info: None, + idx: 0, + err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())), + }], + "bucket", + "object", + &ObjectOptions { + version_id: Some("vid-1".to_string()), + ..Default::default() + }, + ) + .unwrap_err(); + + assert_eq!( + err, + Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string()) + ); + } + + #[test] + fn pool_lookup_not_found_error_returns_object_not_found_for_latest_lookup() { + let err = pool_lookup_not_found_error("bucket", "object", &ObjectOptions::default()); + + assert_eq!(err, Error::ObjectNotFound("bucket".to_string(), "object".to_string())); + } + + #[test] + fn pool_lookup_not_found_error_returns_version_not_found_for_versioned_lookup() { + let err = pool_lookup_not_found_error( + "bucket", + "object", + &ObjectOptions { + version_id: Some("vid-1".to_string()), + ..Default::default() + }, + ); + + assert_eq!( + err, + Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string()) + ); + } + + #[test] + fn resolve_store_rebalance_pool_meta_reload_result_passthrough_ok() { + resolve_store_rebalance_pool_meta_reload_result(Ok(()), "reload_pool_meta") + .expect("successful pool meta reload should pass through"); + } + + #[test] + fn resolve_store_rebalance_pool_meta_reload_result_wraps_error_context() { + let err = resolve_store_rebalance_pool_meta_reload_result(Err(Error::SlowDown), "reload_pool_meta") + .expect_err("failed pool meta reload should be wrapped"); + let err_message = err.to_string(); + assert!(err_message.contains("store rebalance pool meta reload failed during reload_pool_meta")); + assert!(err_message.contains(&Error::SlowDown.to_string())); + } + + #[test] + fn resolve_rebalance_delete_from_all_pools_result_passthrough_ok() { + let info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + ..Default::default() + }; + + let resolved = resolve_rebalance_delete_from_all_pools_result(Ok(info.clone()), "bucket", "object") + .expect("successful rebalance delete should pass through"); + + assert_eq!(resolved.bucket, info.bucket); + assert_eq!(resolved.name, info.name); + } + + #[test] + fn resolve_rebalance_delete_from_all_pools_result_wraps_object_context() { + let err = resolve_rebalance_delete_from_all_pools_result(Err(Error::SlowDown), "bucket", "object") + .expect_err("failed rebalance delete should be wrapped"); + let rendered = err.to_string(); + + assert!(rendered.contains("failed to delete rebalance source object bucket/object"), "{rendered}"); + assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}"); + } + + #[test] + fn rebalance_disk_set_lookup_error_formats_pool_and_set_context() { + let err = rebalance_disk_set_lookup_error(2, 7, 3); + + assert!( + err.to_string() + .contains("failed to resolve rebalance disk set: pool index 2, set index 7, pool count 3") + ); + } + + #[tokio::test] + async fn build_server_pools_available_space_returns_zero_for_empty_pool_info() { + let spaces = build_server_pools_available_space("bucket-a", 64, &[1], &[Vec::new()]).await; + + assert_eq!(spaces.0.len(), 1); + assert_eq!(spaces.0[0].index, 0); + assert_eq!(spaces.0[0].available, 0); + assert_eq!(spaces.0[0].max_used_pct, 0); + } + + #[tokio::test] + async fn build_server_pools_available_space_computes_available_capacity_and_max_used_pct() { + let infos = vec![vec![Some(disk_info(1_000, 100, 900)), Some(disk_info(1_000, 200, 800))]]; + + let spaces = build_server_pools_available_space("bucket-a", 64, &[2], &infos).await; + + assert_eq!(spaces.0.len(), 1); + assert_eq!(spaces.0[0].index, 0); + assert_eq!(spaces.0[0].available, 3_400); + assert_eq!(spaces.0[0].max_used_pct, 20); + } + + #[tokio::test] + async fn build_server_pools_available_space_skips_capacity_guard_for_meta_bucket() { + let infos = vec![vec![Some(disk_info(10, 9, 1)), Some(disk_info(10, 9, 1))]]; + + let spaces = build_server_pools_available_space(crate::disk::RUSTFS_META_BUCKET, 1_024, &[1], &infos).await; + + assert_eq!(spaces.0.len(), 1); + assert_eq!(spaces.0[0].available, 2); + assert_eq!(spaces.0[0].max_used_pct, 90); + } +} diff --git a/crates/ecstore/src/tier/tier.rs b/crates/ecstore/src/tier/tier.rs index ace4f9e4e..8cf6e6062 100644 --- a/crates/ecstore/src/tier/tier.rs +++ b/crates/ecstore/src/tier/tier.rs @@ -49,7 +49,7 @@ use crate::{ StorageAPI, config::com::{CONFIG_PREFIX, read_config}, disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}, - global::{get_global_endpoints, is_first_cluster_node_local}, + global::is_first_cluster_node_local, store::ECStore, store_api::{ObjectIO as _, ObjectOptions, PutObjReader}, }; @@ -1006,7 +1006,7 @@ impl TierConfigMgr { #[tracing::instrument(level = "debug", name = "tier_save", skip(self))] pub async fn save(&self) -> std::result::Result<(), std::io::Error> { let Some(api) = new_object_layer_fn() else { - return Err(std::io::Error::other("errServerNotInitialized")); + return Err(tier_config_not_initialized_error("save tiering config")); }; //let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?; @@ -1231,6 +1231,10 @@ pub fn is_err_config_not_found(err: &StorageError) -> bool { matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::BucketNotFound(_)) || err == &StorageError::ConfigNotFound } +fn tier_config_not_initialized_error(operation: &str) -> std::io::Error { + std::io::Error::other(format!("failed to {operation}: object layer not initialized")) +} + #[cfg(test)] mod tests { use super::*; @@ -1335,4 +1339,13 @@ mod tests { Some("bucket-a") ); } + + #[test] + fn test_tier_config_not_initialized_error_formats_operation_context() { + let err = tier_config_not_initialized_error("save tiering config"); + let rendered = err.to_string(); + + assert!(rendered.contains("failed to save tiering config"), "{rendered}"); + assert!(rendered.contains("object layer not initialized"), "{rendered}"); + } } diff --git a/crates/filemeta/src/metacache.rs b/crates/filemeta/src/metacache.rs index bfb20f806..0cc274e65 100644 --- a/crates/filemeta/src/metacache.rs +++ b/crates/filemeta/src/metacache.rs @@ -399,7 +399,13 @@ impl MetaCacheEntries { return None; } - let metadata = match cached.marshal_msg() { + let merged_cached = FileMeta { + meta_ver: cached.meta_ver, + versions, + ..Default::default() + }; + + let metadata = match merged_cached.marshal_msg() { Ok(meta) => meta, Err(e) => { warn!("decommission_pool: entries resolve entry marshal_msg {:?}", e); @@ -411,11 +417,7 @@ impl MetaCacheEntries { // Create a new merged result. let new_selected = MetaCacheEntry { name: selected.name.clone(), - cached: Some(FileMeta { - meta_ver: cached.meta_ver, - versions, - ..Default::default() - }), + cached: Some(merged_cached), reusable: true, metadata, }; @@ -871,7 +873,12 @@ impl Cache { #[cfg(test)] mod tests { use super::*; + use crate::test_data::create_real_xlmeta; + use crate::{FileMetaVersion, MetaDeleteMarker}; + use std::collections::HashMap; use std::io::Cursor; + use time::OffsetDateTime; + use uuid::Uuid; #[tokio::test] async fn test_writer() { @@ -900,4 +907,61 @@ mod tests { assert_eq!(objs, nobjs); } + + #[test] + fn test_resolve_rebuilds_metadata_from_merged_versions() { + let base_metadata = create_real_xlmeta().expect("base xl.meta"); + let base = FileMeta::load(&base_metadata).expect("load base xl.meta"); + + let extra_version = FileMetaVersion { + version_type: VersionType::Delete, + object: None, + delete_marker: Some(MetaDeleteMarker { + version_id: Some(Uuid::from_u128(0x22222222333344445555666666666666)), + mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp")), + meta_sys: HashMap::new(), + }), + write_version: 99, + uses_legacy_checksum: false, + }; + + let extra_shallow = FileMetaShallowVersion::try_from(extra_version).expect("build shallow delete version"); + + let mut extended = base.clone(); + extended.versions.insert(0, extra_shallow); + + let base_versions = base.versions.len(); + let extended_versions = extended.versions.len(); + let extended_metadata = extended.marshal_msg().expect("serialize extended xl.meta"); + + let resolved = MetaCacheEntries(vec![ + Some(MetaCacheEntry { + name: "bucket/object".to_string(), + metadata: extended_metadata, + cached: Some(extended), + reusable: false, + }), + Some(MetaCacheEntry { + name: "bucket/object".to_string(), + metadata: base_metadata, + cached: Some(base), + reusable: false, + }), + ]) + .resolve(MetadataResolutionParams { + obj_quorum: 2, + requested_versions: extended_versions, + strict: true, + ..Default::default() + }) + .expect("merged entry should resolve"); + + let cached = resolved.cached.expect("resolved entry should keep merged cached metadata"); + let decoded = FileMeta::load(&resolved.metadata).expect("resolved metadata should decode"); + + assert_eq!(cached.versions.len(), base_versions); + assert_eq!(decoded.versions.len(), base_versions); + assert_eq!(decoded.versions, cached.versions); + assert_ne!(extended_versions, cached.versions.len()); + } } diff --git a/crates/rio/src/compress_index.rs b/crates/rio/src/compress_index.rs index b1f7e7ea7..2c8e5139b 100644 --- a/crates/rio/src/compress_index.rs +++ b/crates/rio/src/compress_index.rs @@ -18,6 +18,7 @@ use std::io::{self, Read, Seek, SeekFrom}; const S2_INDEX_HEADER: &[u8] = b"s2idx\x00"; const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s"; +const LEGACY_INDEX_HEADER_PADDING: &[u8] = &[0, 0, 0]; const MAX_INDEX_ENTRIES: usize = 1 << 16; const MIN_INDEX_DIST: i64 = 1 << 20; // const MIN_INDEX_DIST: i64 = 0; @@ -76,10 +77,14 @@ impl Index { } fn alloc_infos(&mut self, n: usize) { - if n > MAX_INDEX_ENTRIES { - panic!("n > MAX_INDEX_ENTRIES"); - } - self.info = Vec::with_capacity(n); + debug_assert!(n <= MAX_INDEX_ENTRIES, "n > MAX_INDEX_ENTRIES"); + self.info = vec![ + IndexInfo { + compressed_offset: 0, + uncompressed_offset: 0, + }; + n + ]; } pub fn add(&mut self, compressed_offset: i64, uncompressed_offset: i64) -> io::Result<()> { @@ -217,9 +222,8 @@ impl Index { self.reduce(); let init_size = b.len(); - // Add skippable header - b.extend_from_slice(&[0x50, 0x2A, 0x4D, 0x18]); // ChunkTypeIndex - b.extend_from_slice(&[0, 0, 0]); // Placeholder for chunk length + // Add skippable header (1-byte marker + 24-bit length placeholder) + b.extend_from_slice(&[0x50, 0x2A, 0x4D, 0x18]); // length is written back into bytes 1..=3 // Add header b.extend_from_slice(S2_INDEX_HEADER); @@ -295,7 +299,7 @@ impl Index { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small")); } - if b[0] != 0x50 || b[1] != 0x2A || b[2] != 0x4D || b[3] != 0x18 { + if b[0] != 0x50 { return Err(io::Error::other("invalid chunk type")); } @@ -306,6 +310,10 @@ impl Index { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small")); } + if b.starts_with(LEGACY_INDEX_HEADER_PADDING) { + b = &b[LEGACY_INDEX_HEADER_PADDING.len()..]; + } + if !b.starts_with(S2_INDEX_HEADER) { return Err(io::Error::other("invalid header")); } @@ -687,4 +695,72 @@ mod tests { Ok(()) } + + #[test] + fn test_index_into_vec_round_trip_via_load() -> io::Result<()> { + let mut source = Index::new(); + source.add(100, 1_000)?; + source.add(300, 1_000 + MIN_INDEX_DIST)?; + + let encoded = source.clone().into_vec(); + + let mut decoded = Index::new(); + let rest = decoded.load(encoded.as_ref())?; + + assert!(rest.is_empty()); + assert_eq!(decoded.total_uncompressed, source.total_uncompressed); + assert_eq!(decoded.total_compressed, source.total_compressed); + assert_eq!(decoded.info.len(), source.info.len()); + assert_eq!(decoded.info[0].compressed_offset, source.info[0].compressed_offset); + assert_eq!(decoded.info[0].uncompressed_offset, source.info[0].uncompressed_offset); + assert_eq!(decoded.info[1].uncompressed_offset, source.info[1].uncompressed_offset); + assert!(decoded.info[1].compressed_offset > decoded.info[0].compressed_offset); + + Ok(()) + } + + #[test] + fn test_index_load_rejects_invalid_chunk_type_marker() -> io::Result<()> { + let mut source = Index::new(); + source.add(100, 1_000)?; + source.add(300, 1_000 + MIN_INDEX_DIST)?; + let mut encoded = source.into_vec().to_vec(); + + encoded[0] = 0x51; + + let mut decoded = Index::new(); + let err = decoded + .load(encoded.as_slice()) + .expect_err("invalid marker should be rejected"); + assert_eq!(err.kind(), io::ErrorKind::Other); + assert_eq!(err.to_string(), "invalid chunk type"); + + Ok(()) + } + + #[test] + fn test_index_load_accepts_legacy_zero_padded_header() -> io::Result<()> { + let mut source = Index::new(); + source.add(100, 1_000)?; + source.add(300, 1_000 + MIN_INDEX_DIST)?; + + let mut encoded = source.clone().into_vec().to_vec(); + let chunk_len = (encoded[1] as usize) | ((encoded[2] as usize) << 8) | ((encoded[3] as usize) << 16); + let legacy_chunk_len = chunk_len + LEGACY_INDEX_HEADER_PADDING.len(); + + encoded[1] = legacy_chunk_len as u8; + encoded[2] = (legacy_chunk_len >> 8) as u8; + encoded[3] = (legacy_chunk_len >> 16) as u8; + encoded.splice(4..4, LEGACY_INDEX_HEADER_PADDING.iter().copied()); + + let mut decoded = Index::new(); + let rest = decoded.load(encoded.as_slice())?; + + assert!(rest.is_empty()); + assert_eq!(decoded.total_uncompressed, source.total_uncompressed); + assert_eq!(decoded.total_compressed, source.total_compressed); + assert_eq!(decoded.info.len(), source.info.len()); + + Ok(()) + } } diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 86186db2d..39ef43ecd 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -22,6 +22,7 @@ use rustfs_common::internode_metrics::global_internode_metrics; use rustfs_utils::get_env_opt_str; use std::io::IoSlice; use std::io::{self, Error}; +use std::net::IpAddr; use std::ops::Not as _; use std::pin::Pin; use std::sync::LazyLock; @@ -91,25 +92,48 @@ fn load_optional_mtls_identity_from_tls_path() -> Option { } } -fn get_http_client() -> Client { - // Reuse the HTTP connection pool in the global `reqwest::Client` instance - // TODO: interact with load balancing? - static CLIENT: LazyLock = LazyLock::new(|| { - let mut builder = Client::builder() - .connect_timeout(std::time::Duration::from_secs(5)) - .tcp_keepalive(std::time::Duration::from_secs(10)) - .http2_keep_alive_interval(std::time::Duration::from_secs(5)) - .http2_keep_alive_timeout(std::time::Duration::from_secs(3)) - .http2_keep_alive_while_idle(true); +fn build_http_client(disable_proxy: bool) -> Client { + let mut builder = Client::builder() + .connect_timeout(std::time::Duration::from_secs(5)) + .tcp_keepalive(std::time::Duration::from_secs(10)) + .http2_keep_alive_interval(std::time::Duration::from_secs(5)) + .http2_keep_alive_timeout(std::time::Duration::from_secs(3)) + .http2_keep_alive_while_idle(true); - // HTTPS root trust + optional mTLS identity from RUSTFS_TLS_PATH - builder = load_ca_roots_from_tls_path(builder); - if let Some(id) = load_optional_mtls_identity_from_tls_path() { - builder = builder.identity(id); - } + if disable_proxy { + builder = builder.no_proxy(); + } + + builder = load_ca_roots_from_tls_path(builder); + if let Some(id) = load_optional_mtls_identity_from_tls_path() { + builder = builder.identity(id); + } + + builder.build().expect("Failed to create global HTTP client") +} + +fn should_bypass_proxy_for_url(url: &str) -> bool { + let Some(host) = reqwest::Url::parse(url) + .ok() + .and_then(|url| url.host_str().map(str::to_owned)) + else { + return false; + }; + let host = host.trim_matches(['[', ']']); + + host.eq_ignore_ascii_case("localhost") || host.parse::().is_ok_and(|addr| addr.is_loopback()) +} + +fn get_http_client(url: &str) -> Client { + // Reuse HTTP connection pools while keeping loopback traffic away from + // system proxies so local RPC/tests do not leak to proxy listeners. + static CLIENT: LazyLock = LazyLock::new(|| build_http_client(false)); + static LOCAL_CLIENT: LazyLock = LazyLock::new(|| build_http_client(true)); + + if should_bypass_proxy_for_url(url) { + return LOCAL_CLIENT.clone(); + } - builder.build().expect("Failed to create global HTTP client") - }); CLIENT.clone() } @@ -138,7 +162,7 @@ impl HttpReader { _read_buf_size: usize, ) -> io::Result { let track_internode_metrics = is_internode_rpc_url(&url); - let client = get_http_client(); + let client = get_http_client(&url); let mut request: RequestBuilder = client.request(method.clone(), url.clone()).headers(headers.clone()); if let Some(body) = body { request = request.body(body); @@ -302,7 +326,7 @@ impl HttpWriter { // "[HttpWriter::spawn] sending HTTP request: url={url_clone}, method={method_clone:?}, headers={headers_clone:?}" // ); - let client = get_http_client(); + let client = get_http_client(&url_clone); let request = client .request(method_clone, url_clone.clone()) .headers(headers_clone.clone()) @@ -664,4 +688,14 @@ mod tests { handle.abort(); } + + #[test] + fn loopback_urls_bypass_proxy_selection() { + assert!(should_bypass_proxy_for_url("http://127.0.0.1:9000/stream")); + assert!(should_bypass_proxy_for_url("http://localhost:9000/stream")); + assert!(should_bypass_proxy_for_url("http://[::1]:9000/stream")); + assert!(!should_bypass_proxy_for_url("http://192.168.1.10:9000/stream")); + assert!(!should_bypass_proxy_for_url("http://example.com/stream")); + assert!(!should_bypass_proxy_for_url("not-a-url")); + } } diff --git a/rustfs/src/admin/handlers/pools.rs b/rustfs/src/admin/handlers/pools.rs index 34f5d9930..5a98e55bf 100644 --- a/rustfs/src/admin/handlers/pools.rs +++ b/rustfs/src/admin/handlers/pools.rs @@ -35,10 +35,85 @@ use crate::{ use hyper::Method; use rustfs_ecstore::new_object_layer_fn; +use std::collections::HashSet; + fn endpoints_from_context() -> Option { resolve_endpoints_handle() } +fn validate_start_decommission_guards(decommission_running: bool, rebalance_running: bool) -> s3s::S3Result<()> { + if decommission_running { + return Err(s3_error!(InvalidRequest, "DecommissionAlreadyRunning")); + } + + if rebalance_running { + return Err(S3Error::with_message( + S3ErrorCode::OperationAborted, + "Decommission cannot be started, rebalance is already in progress".to_string(), + )); + } + + Ok(()) +} + +fn contextualize_admin_pool_api_error( + err: crate::error::ApiError, + operation: &str, + pool_context: impl std::fmt::Display, +) -> crate::error::ApiError { + crate::error::ApiError { + code: err.code, + message: format!("admin {operation} failed for {pool_context}: {}", err.message), + source: err.source, + } +} + +fn decommission_admin_not_initialized_error(operation: &str) -> S3Error { + S3Error::with_message(S3ErrorCode::InternalError, format!("Failed to {operation}: object layer not initialized")) +} + +fn pool_admin_missing_credentials_error(operation: &str) -> S3Error { + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("Failed to {operation}: missing credentials")) +} + +fn pool_admin_query_parse_error(operation: &str) -> S3Error { + S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid query parameters")) +} + +fn pool_admin_pool_parse_error(operation: &str, pool: &str) -> S3Error { + S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid pool `{pool}`")) +} + +fn pool_admin_pool_not_found_error(operation: &str, pool: &str) -> S3Error { + S3Error::with_message( + S3ErrorCode::InvalidArgument, + format!("Failed to {operation}: pool `{pool}` was not found"), + ) +} + +fn pool_admin_pool_index_error(operation: &str, idx: usize, pool_count: usize) -> S3Error { + S3Error::with_message( + S3ErrorCode::InvalidArgument, + format!("Failed to {operation}: pool index {idx} is out of range for {pool_count} pools"), + ) +} + +fn parse_pool_idx_by_id(pool: &str, endpoint_count: usize) -> Option { + let idx = pool.parse::().ok()?; + (idx < endpoint_count).then_some(idx) +} + +fn dedup_indices(indices: &[usize]) -> Vec { + let mut seen = HashSet::with_capacity(indices.len()); + let mut output = Vec::with_capacity(indices.len()); + for idx in indices { + if seen.insert(*idx) { + output.push(*idx); + } + } + output +} + pub fn register_pool_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( Method::GET, @@ -77,7 +152,7 @@ impl Operation for ListPools { warn!("handle ListPools"); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(pool_admin_missing_credentials_error("list pools")); }; let (cred, owner) = @@ -127,7 +202,7 @@ impl Operation for StatusPool { warn!("handle StatusPool"); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(pool_admin_missing_credentials_error("load pool status")); }; let (cred, owner) = @@ -149,7 +224,7 @@ impl Operation for StatusPool { let query = { if let Some(query) = req.uri.query() { let input: StatusPoolQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("load pool status"))?; input } else { StatusPoolQuery::default() @@ -185,7 +260,7 @@ impl Operation for StartDecommission { warn!("handle StartDecommission"); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(pool_admin_missing_credentials_error("start decommission")); }; let (cred, owner) = @@ -210,27 +285,15 @@ impl Operation for StartDecommission { } let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + return Err(decommission_admin_not_initialized_error("start decommission")); }; - if store.is_decommission_running().await { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - "DecommissionAlreadyRunning".to_string(), - )); - } - - if store.is_rebalance_started().await { - return Err(S3Error::with_message( - S3ErrorCode::OperationAborted, - "Decommission cannot be started, rebalance is already in progress".to_string(), - )); - } + validate_start_decommission_guards(store.is_decommission_running().await, store.is_rebalance_started().await)?; let query = { if let Some(query) = req.uri.query() { let input: StatusPoolQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("start decommission"))?; input } else { StatusPoolQuery::default() @@ -239,40 +302,38 @@ impl Operation for StartDecommission { let is_byid = query.by_id.as_str() == "true"; let pools: Vec<&str> = query.pool.split(",").collect(); - let mut pools_indices = Vec::with_capacity(pools.len()); + let mut parsed_indices = Vec::with_capacity(pools.len()); let ctx = CancellationToken::new(); for pool in pools.iter() { let idx = { if is_byid { - pool.parse::() - .map_err(|_e| s3_error!(InvalidArgument, "pool parse failed"))? + parse_pool_idx_by_id(pool, endpoints.as_ref().len()) + .ok_or_else(|| pool_admin_pool_parse_error("start decommission", pool))? } else { let Some(idx) = endpoints.get_pool_idx(pool) else { - return Err(s3_error!(InvalidArgument, "pool parse failed")); + return Err(pool_admin_pool_parse_error("start decommission", pool)); }; idx } }; - let mut has_found = None; - for (i, pool) in store.pools.iter().enumerate() { - if i == idx { - has_found = Some(pool.clone()); - break; - } + if idx >= store.pools.len() { + return Err(pool_admin_pool_index_error("start decommission", idx, store.pools.len())); } - let Some(_p) = has_found else { - return Err(s3_error!(InvalidArgument)); - }; - - pools_indices.push(idx); + parsed_indices.push(idx); } + let pools_indices = dedup_indices(&parsed_indices); if !pools_indices.is_empty() { - store.decommission(ctx.clone(), pools_indices).await.map_err(ApiError::from)?; + let pool_context = format!("pools {:?}", &pools_indices); + store + .decommission(ctx.clone(), pools_indices) + .await + .map_err(ApiError::from) + .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?; } Ok(S3Response::new((StatusCode::OK, Body::default()))) @@ -289,7 +350,7 @@ impl Operation for CancelDecommission { warn!("handle CancelDecommission"); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(pool_admin_missing_credentials_error("cancel decommission")); }; let (cred, owner) = @@ -316,7 +377,7 @@ impl Operation for CancelDecommission { let query = { if let Some(query) = req.uri.query() { let input: StatusPoolQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("cancel decommission"))?; input } else { StatusPoolQuery::default() @@ -327,8 +388,7 @@ impl Operation for CancelDecommission { let has_idx = { if is_byid { - let a = query.pool.parse::().unwrap_or_default(); - if a < endpoints.as_ref().len() { Some(a) } else { None } + parse_pool_idx_by_id(&query.pool, endpoints.as_ref().len()) } else { endpoints.get_pool_idx(&query.pool) } @@ -336,15 +396,181 @@ impl Operation for CancelDecommission { let Some(idx) = has_idx else { warn!("specified pool {} not found, please specify a valid pool", &query.pool); - return Err(s3_error!(InvalidArgument)); + return Err(pool_admin_pool_not_found_error("cancel decommission", &query.pool)); }; let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + return Err(decommission_admin_not_initialized_error("cancel decommission")); }; - store.decommission_cancel(idx).await.map_err(ApiError::from)?; + store + .decommission_cancel(idx) + .await + .map_err(ApiError::from) + .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?; Ok(S3Response::new((StatusCode::OK, Body::default()))) } } + +#[cfg(test)] +mod pools_handler_tests { + use super::{ + contextualize_admin_pool_api_error, decommission_admin_not_initialized_error, dedup_indices, parse_pool_idx_by_id, + pool_admin_missing_credentials_error, pool_admin_pool_index_error, pool_admin_pool_not_found_error, + pool_admin_pool_parse_error, pool_admin_query_parse_error, validate_start_decommission_guards, + }; + + #[test] + fn test_parse_pool_idx_by_id_rejects_non_numeric() { + assert_eq!(parse_pool_idx_by_id("invalid", 4), None); + } + + #[test] + fn test_parse_pool_idx_by_id_rejects_out_of_range() { + assert_eq!(parse_pool_idx_by_id("4", 4), None); + } + + #[test] + fn test_parse_pool_idx_by_id_rejects_empty_pool_count() { + assert_eq!(parse_pool_idx_by_id("0", 0), None); + } + + #[test] + fn test_parse_pool_idx_by_id_accepts_valid_index() { + assert_eq!(parse_pool_idx_by_id("2", 4), Some(2)); + } + + #[test] + fn test_validate_start_decommission_guards_rejects_decommission_running() { + let err = validate_start_decommission_guards(true, false).expect_err("decommission running should be rejected"); + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("DecommissionAlreadyRunning")); + } + + #[test] + fn test_validate_start_decommission_guards_rejects_rebalance_running() { + let err = validate_start_decommission_guards(false, true).expect_err("rebalance running should be rejected"); + assert_eq!(err.code(), &s3s::S3ErrorCode::OperationAborted); + assert_eq!(err.message(), Some("Decommission cannot be started, rebalance is already in progress")); + } + + #[test] + fn test_validate_start_decommission_guards_prefers_decommission_over_rebalance() { + let err = validate_start_decommission_guards(true, true).expect_err("decommission should be checked before rebalance"); + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("DecommissionAlreadyRunning")); + } + + #[test] + fn test_validate_start_decommission_guards_allows_when_idle() { + assert!(validate_start_decommission_guards(false, false).is_ok()); + } + + #[test] + fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() { + let err = crate::error::ApiError { + code: s3s::S3ErrorCode::InvalidRequest, + message: "decommission already running".to_string(), + source: None, + }; + + let err = contextualize_admin_pool_api_error(err, "start decommission", "pools [1, 3]"); + + assert_eq!(err.code, s3s::S3ErrorCode::InvalidRequest); + assert_eq!( + err.message, + "admin start decommission failed for pools [1, 3]: decommission already running" + ); + } + + #[test] + fn test_contextualize_admin_pool_api_error_preserves_source() { + let err = contextualize_admin_pool_api_error( + crate::error::ApiError::other(std::io::Error::other("boom")), + "cancel decommission", + "pool 2", + ); + + assert!(err.message.contains("admin cancel decommission failed for pool 2")); + assert!(err.source.is_some()); + } + + #[test] + fn test_decommission_admin_not_initialized_error_formats_start_context() { + let err = decommission_admin_not_initialized_error("start decommission"); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError); + assert_eq!(err.message(), Some("Failed to start decommission: object layer not initialized")); + } + + #[test] + fn test_decommission_admin_not_initialized_error_formats_cancel_context() { + let err = decommission_admin_not_initialized_error("cancel decommission"); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError); + assert_eq!(err.message(), Some("Failed to cancel decommission: object layer not initialized")); + } + + #[test] + fn test_pool_admin_missing_credentials_error_formats_list_context() { + let err = pool_admin_missing_credentials_error("list pools"); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("Failed to list pools: missing credentials")); + } + + #[test] + fn test_pool_admin_missing_credentials_error_formats_decommission_context() { + let err = pool_admin_missing_credentials_error("start decommission"); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("Failed to start decommission: missing credentials")); + } + + #[test] + fn test_pool_admin_query_parse_error_formats_status_context() { + let err = pool_admin_query_parse_error("load pool status"); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument); + assert_eq!(err.message(), Some("Failed to load pool status: invalid query parameters")); + } + + #[test] + fn test_pool_admin_pool_parse_error_formats_pool_context() { + let err = pool_admin_pool_parse_error("start decommission", "pool-x"); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument); + assert_eq!(err.message(), Some("Failed to start decommission: invalid pool `pool-x`")); + } + + #[test] + fn test_pool_admin_pool_index_error_formats_range_context() { + let err = pool_admin_pool_index_error("start decommission", 4, 2); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument); + assert_eq!( + err.message(), + Some("Failed to start decommission: pool index 4 is out of range for 2 pools") + ); + } + + #[test] + fn test_pool_admin_pool_not_found_error_formats_cancel_context() { + let err = pool_admin_pool_not_found_error("cancel decommission", "pool-x"); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument); + assert_eq!(err.message(), Some("Failed to cancel decommission: pool `pool-x` was not found")); + } + + #[test] + fn test_dedup_indices_removes_duplicates_preserving_order() { + assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]); + } + + #[test] + fn test_dedup_indices_handles_empty_input() { + let empty: Vec = Vec::new(); + assert!(dedup_indices(&empty).is_empty()); + } +} diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index 5a97ac73f..9ddf4711c 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -20,7 +20,7 @@ use crate::{ auth::{check_key_valid, get_session_token}, server::{ADMIN_PREFIX, RemoteAddr}, }; -use http::{HeaderMap, StatusCode}; +use http::{HeaderMap, HeaderValue, StatusCode}; use hyper::Method; use matchit::Params; use rustfs_ecstore::rebalance::RebalanceMeta; @@ -79,6 +79,8 @@ pub struct RebalPoolProgress { pub num_versions: u64, #[serde(rename = "bytes")] pub bytes: u64, + #[serde(rename = "remainingBuckets")] + pub remaining_buckets: usize, #[serde(rename = "bucket")] pub bucket: String, #[serde(rename = "object")] @@ -96,7 +98,9 @@ pub struct RebalancePoolStatus { #[serde(rename = "status")] pub status: String, // Active if rebalance is running, empty otherwise #[serde(rename = "used")] - pub used: f64, // Percentage used space + pub used: f64, // Fraction of used space in range 0.0..=1.0 + #[serde(rename = "lastError")] + pub last_error: Option, // Last rebalance error message for this pool #[serde(rename = "progress")] pub progress: Option, // None when rebalance is not running } @@ -110,6 +114,106 @@ pub struct RebalanceAdminStatus { pub stopped_at: Option, // Optional timestamp when rebalance was stopped } +fn calculate_rebalance_progress( + now: OffsetDateTime, + start_time: Option, + terminal_time: Option, + bytes: u64, + target_bytes: f64, +) -> Option<(u64, u64)> { + let start = start_time?; + let reference = terminal_time.unwrap_or(now); + let elapsed_secs = (reference - start).whole_seconds().max(0) as u64; + + if terminal_time.is_some() { + return Some((elapsed_secs, 0)); + } + + if !target_bytes.is_finite() || bytes == 0 || target_bytes <= bytes as f64 { + return Some((elapsed_secs, 0)); + } + + let remaining = target_bytes - bytes as f64; + if remaining <= 0.0 { + return Some((elapsed_secs, 0)); + } + + let eta_secs_f64 = remaining * elapsed_secs as f64 / bytes as f64; + let eta_secs = Duration::try_from_secs_f64(eta_secs_f64).map_or(0, |duration| duration.as_secs()); + Some((elapsed_secs, eta_secs)) +} + +fn build_rebalance_pool_progress( + now: OffsetDateTime, + stop_time: Option, + percent_free_goal: f64, + ps: &rustfs_ecstore::rebalance::RebalanceStats, +) -> Option { + let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64; + let terminal_time = ps.info.end_time.or(stop_time); + let (elapsed, eta) = calculate_rebalance_progress(now, ps.info.start_time, terminal_time, ps.bytes, total_bytes_to_rebal)?; + + Some(RebalPoolProgress { + num_objects: ps.num_objects, + num_versions: ps.num_versions, + bytes: ps.bytes, + remaining_buckets: rebalance_remaining_buckets(ps.buckets.len(), ps.rebalanced_buckets.len()), + bucket: ps.bucket.clone(), + object: ps.object.clone(), + elapsed, + eta, + }) +} + +fn rebalance_used_pct(total: u64, available: u64) -> f64 { + if total == 0 { + return 0.0; + } + + let bounded_available = available.min(total); + (total - bounded_available) as f64 / total as f64 +} + +fn rebalance_remaining_buckets(buckets: usize, rebalanced_buckets: usize) -> usize { + buckets.saturating_sub(rebalanced_buckets) +} + +fn rebalance_pool_used(disk_stats: &[DiskStat], idx: usize) -> f64 { + let (total_space, available_space) = disk_stats + .get(idx) + .map(|stat| (stat.total_space, stat.available_space)) + .unwrap_or((0, 0)); + rebalance_used_pct(total_space, available_space) +} + +fn build_rebalance_pool_statuses( + now: OffsetDateTime, + stop_time: Option, + percent_free_goal: f64, + pool_stats: &[rustfs_ecstore::rebalance::RebalanceStats], + disk_stats: &[DiskStat], +) -> Vec { + pool_stats + .iter() + .enumerate() + .map(|(i, ps)| { + let mut status = RebalancePoolStatus { + id: i, + status: ps.info.status.to_string(), + used: rebalance_pool_used(disk_stats, i), + last_error: ps.info.last_error.clone(), + progress: None, + }; + + if ps.participating { + status.progress = build_rebalance_pool_progress(now, stop_time, percent_free_goal, ps); + } + + status + }) + .collect() +} + pub struct RebalanceStart {} #[async_trait::async_trait] @@ -119,7 +223,7 @@ impl Operation for RebalanceStart { warn!("handle RebalanceStart"); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "Failed to start rebalance: missing credentials")); }; let (cred, owner) = @@ -136,7 +240,7 @@ impl Operation for RebalanceStart { .await?; let Some(store) = new_object_layer_fn() else { - return Err(s3_error!(InternalError, "Not init")); + return Err(s3_error!(InternalError, "Failed to start rebalance: object layer not initialized")); }; if store.pools.len() == 1 { @@ -150,38 +254,44 @@ impl Operation for RebalanceStart { )); } - if store.is_rebalance_started().await { + if store.is_rebalance_conflicting_with_decommission().await { return Err(s3_error!(OperationAborted, "Rebalance already in progress")); } let bucket_infos = store .list_bucket(&BucketOptions::default()) .await - .map_err(|e| s3_error!(InternalError, "Failed to list buckets: {}", e))?; + .map_err(|e| s3_error!(InternalError, "Failed to list buckets for rebalance: {}", e))?; let buckets: Vec = bucket_infos.into_iter().map(|bucket| bucket.name).collect(); let id = match store.init_rebalance_meta(buckets).await { Ok(id) => id, Err(e) => { - return Err(s3_error!(InternalError, "Failed to init rebalance meta: {}", e)); + return Err(s3_error!(InternalError, "Failed to initialize rebalance metadata: {}", e)); } }; - store.start_rebalance().await; + store + .start_rebalance() + .await + .map_err(|e| s3_error!(InternalError, "Failed to start rebalance: {}", e))?; warn!("Rebalance started with id: {}", id); if let Some(notification_sys) = get_global_notification_sys() { warn!("RebalanceStart Loading rebalance meta start"); - notification_sys.load_rebalance_meta(true).await; + if let Err(err) = notification_sys.load_rebalance_meta(true).await { + warn!("rebalance start propagation failed after local state update: {err}"); + } warn!("RebalanceStart Loading rebalance meta done"); } let resp = RebalanceResp { id }; - let data = serde_json::to_string(&resp).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; + let data = serde_json::to_string(&resp) + .map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance start response: {}", e))?; let mut header = HeaderMap::new(); - header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) } @@ -197,7 +307,7 @@ impl Operation for RebalanceStatus { warn!("handle RebalanceStatus"); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "Failed to load rebalance status: missing credentials")); }; let (cred, owner) = @@ -214,16 +324,26 @@ impl Operation for RebalanceStatus { .await?; let Some(store) = new_object_layer_fn() else { - return Err(s3_error!(InternalError, "Not init")); + return Err(s3_error!(InternalError, "Failed to load rebalance status: object layer not initialized")); }; + if store.pools.is_empty() { + return Err(s3_error!(InternalError, "Failed to load rebalance status: no storage pools available")); + } + + let first_pool = store + .pools + .first() + .cloned() + .ok_or_else(|| s3_error!(InternalError, "Failed to load rebalance status: no storage pools available"))?; + let mut meta = RebalanceMeta::new(); - if let Err(err) = meta.load(store.pools[0].clone()).await { + if let Err(err) = meta.load(first_pool).await { if err == StorageError::ConfigNotFound { return Err(s3_error!(NoSuchResource, "Pool rebalance is not started")); } - return Err(s3_error!(InternalError, "Failed to load rebalance meta: {}", err)); + return Err(s3_error!(InternalError, "Failed to load rebalance metadata from pool 0: {}", err)); } // Compute disk usage percentage @@ -238,68 +358,18 @@ impl Operation for RebalanceStatus { disk_stats[disk.pool_index as usize].total_space += disk.total_space; } - let mut stop_time = meta.stopped_at; - let mut admin_status = RebalanceAdminStatus { + let stop_time = meta.stopped_at; + let now = OffsetDateTime::now_utc(); + let admin_status = RebalanceAdminStatus { id: meta.id.clone(), stopped_at: meta.stopped_at, - pools: vec![RebalancePoolStatus::default(); meta.pool_stats.len()], + pools: build_rebalance_pool_statuses(now, stop_time, meta.percent_free_goal, &meta.pool_stats, &disk_stats), }; - for (i, ps) in meta.pool_stats.iter().enumerate() { - admin_status.pools[i] = RebalancePoolStatus { - id: i, - status: ps.info.status.to_string(), - used: (disk_stats[i].total_space - disk_stats[i].available_space) as f64 / disk_stats[i].total_space as f64, - progress: None, - }; - - if !ps.participating { - continue; - } - - // Calculate total bytes to be rebalanced - let total_bytes_to_rebal = ps.init_capacity as f64 * meta.percent_free_goal - ps.init_free_space as f64; - - let mut elapsed = if let Some(start_time) = ps.info.start_time { - let now = OffsetDateTime::now_utc(); - now - start_time - } else { - return Err(s3_error!(InternalError, "Start time is not available")); - }; - - let mut eta = if ps.bytes > 0 { - Duration::from_secs_f64(total_bytes_to_rebal * elapsed.as_seconds_f64() / ps.bytes as f64) - } else { - Duration::ZERO - }; - - if ps.info.end_time.is_some() { - stop_time = ps.info.end_time; - } - - if let Some(stopped_at) = stop_time { - if let Some(start_time) = ps.info.start_time { - elapsed = stopped_at - start_time; - } - - eta = Duration::ZERO; - } - - admin_status.pools[i].progress = Some(RebalPoolProgress { - num_objects: ps.num_objects, - num_versions: ps.num_versions, - bytes: ps.bytes, - bucket: ps.bucket.clone(), - object: ps.object.clone(), - elapsed: elapsed.whole_seconds() as u64, - eta: eta.as_secs(), - }); - } - - let data = - serde_json::to_string(&admin_status).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; + let data = serde_json::to_string(&admin_status) + .map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance status response: {}", e))?; let mut header = HeaderMap::new(); - header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) } @@ -315,7 +385,7 @@ impl Operation for RebalanceStop { warn!("handle RebalanceStop"); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "Failed to stop rebalance: missing credentials")); }; let (cred, owner) = @@ -332,28 +402,42 @@ impl Operation for RebalanceStop { .await?; let Some(store) = new_object_layer_fn() else { - return Err(s3_error!(InternalError, "Not init")); + return Err(s3_error!(InternalError, "Failed to stop rebalance: object layer not initialized")); }; - if let Some(notification_sys) = get_global_notification_sys() { - notification_sys.stop_rebalance().await; + if !store.is_rebalance_conflicting_with_decommission().await { + return Err(s3_error!(NoSuchResource, "Pool rebalance is not started")); } - store - .save_rebalance_stats(0, RebalSaveOpt::StoppedAt) - .await - .map_err(|e| s3_error!(InternalError, "Failed to stop rebalance: {}", e))?; + if let Some(notification_sys) = get_global_notification_sys() { + notification_sys + .stop_rebalance() + .await + .map_err(|e| s3_error!(InternalError, "Failed to stop rebalance via notification system: {}", e))?; + } else { + store + .stop_rebalance() + .await + .map_err(|e| s3_error!(InternalError, "Failed to stop rebalance: {}", e))?; + + store + .save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt) + .await + .map_err(|e| s3_error!(InternalError, "Failed to persist rebalance stop metadata: {}", e))?; + } warn!("handle RebalanceStop save_rebalance_stats done "); if let Some(notification_sys) = get_global_notification_sys() { warn!("handle RebalanceStop notification_sys load_rebalance_meta"); - notification_sys.load_rebalance_meta(false).await; + if let Err(err) = notification_sys.load_rebalance_meta(false).await { + warn!("rebalance stop propagation failed after local state update: {err}"); + } warn!("handle RebalanceStop notification_sys load_rebalance_meta done"); } let mut header = HeaderMap::new(); - header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); - header.insert(CONTENT_LENGTH, "0".parse().unwrap()); + header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + header.insert(CONTENT_LENGTH, HeaderValue::from_static("0")); Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) } } @@ -389,3 +473,372 @@ mod offsetdatetime_rfc3339 { } } } + +#[cfg(test)] +mod rebalance_handler_tests { + use super::build_rebalance_pool_progress; + use super::calculate_rebalance_progress; + use super::{ + RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used, + rebalance_remaining_buckets, rebalance_used_pct, + }; + use rustfs_ecstore::rebalance::{DiskStat, RebalStatus, RebalanceInfo, RebalanceStats}; + use time::OffsetDateTime; + + #[test] + fn test_calculate_rebalance_progress_running() { + let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let now = OffsetDateTime::from_unix_timestamp(1_050).unwrap(); + + let (elapsed, eta) = calculate_rebalance_progress(now, Some(start), None, 100, 200.0).unwrap(); + + assert_eq!(elapsed, 50); + assert_eq!(eta, 50); + } + + #[test] + fn test_calculate_rebalance_progress_stopped_by_end_time() { + let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let terminal = OffsetDateTime::from_unix_timestamp(1_120).unwrap(); + + let (elapsed, eta) = calculate_rebalance_progress( + OffsetDateTime::from_unix_timestamp(1_200).unwrap(), + Some(start), + Some(terminal), + 100, + 200.0, + ) + .unwrap(); + + assert_eq!(elapsed, 120); + assert_eq!(eta, 0); + } + + #[test] + fn test_calculate_rebalance_progress_invalid_target_is_zero_eta() { + let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let now = OffsetDateTime::from_unix_timestamp(1_010).unwrap(); + + let (elapsed, eta) = calculate_rebalance_progress(now, Some(start), None, 100, f64::NAN).unwrap(); + + assert_eq!(elapsed, 10); + assert_eq!(eta, 0); + } + + #[test] + fn test_calculate_rebalance_progress_negative_target_is_zero_eta() { + let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let now = OffsetDateTime::from_unix_timestamp(1_010).unwrap(); + + let (elapsed, eta) = calculate_rebalance_progress(now, Some(start), None, 100, -10.0).unwrap(); + + assert_eq!(elapsed, 10); + assert_eq!(eta, 0); + } + + #[test] + fn test_calculate_rebalance_progress_overflow_eta_is_zero() { + let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let now = OffsetDateTime::from_unix_timestamp(1_010).unwrap(); + + let (elapsed, eta) = calculate_rebalance_progress(now, Some(start), None, 1, f64::MAX).unwrap(); + + assert_eq!(elapsed, 10); + assert_eq!(eta, 0); + } + + #[test] + fn test_calculate_rebalance_progress_no_start_time() { + assert!( + calculate_rebalance_progress(OffsetDateTime::from_unix_timestamp(1_000).unwrap(), None, None, 1, 100.0).is_none() + ); + } + + #[test] + fn test_build_rebalance_pool_progress_returns_none_without_start_time() { + let ps = RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: None, + ..Default::default() + }, + ..Default::default() + }; + + let progress = build_rebalance_pool_progress(OffsetDateTime::from_unix_timestamp(1_000).unwrap(), None, 0.3, &ps); + assert!(progress.is_none()); + } + + #[test] + fn test_build_rebalance_pool_progress_maps_fields_and_eta() { + let ps = RebalanceStats { + init_capacity: 1_000, + init_free_space: 200, + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-c".to_string()], + rebalanced_buckets: vec!["bucket-a".to_string()], + bucket: "bucket-b".to_string(), + object: "obj-1".to_string(), + num_objects: 3, + num_versions: 5, + bytes: 100, + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()), + ..Default::default() + }, + }; + + let progress = build_rebalance_pool_progress(OffsetDateTime::from_unix_timestamp(1_050).unwrap(), None, 0.3, &ps) + .expect("progress should be generated"); + assert_eq!(progress.num_objects, 3); + assert_eq!(progress.num_versions, 5); + assert_eq!(progress.bytes, 100); + assert_eq!(progress.remaining_buckets, 2); + assert_eq!(progress.bucket, "bucket-b"); + assert_eq!(progress.object, "obj-1"); + assert_eq!(progress.elapsed, 50); + assert_eq!(progress.eta, 0); + } + + #[test] + fn test_build_rebalance_pool_progress_stopped_uses_stop_time() { + let ps = RebalanceStats { + init_capacity: 1_000, + init_free_space: 200, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()), + ..Default::default() + }, + participating: true, + ..Default::default() + }; + + let stop_time = OffsetDateTime::from_unix_timestamp(1_200).unwrap(); + let progress = build_rebalance_pool_progress(stop_time, Some(stop_time), 0.3, &ps).expect("progress should be generated"); + + assert_eq!(progress.elapsed, 200); + assert_eq!(progress.eta, 0); + } + + #[test] + fn test_build_rebalance_pool_progress_prefers_info_end_time_over_stop_time() { + let ps = RebalanceStats { + init_capacity: 1_000, + init_free_space: 200, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()), + end_time: Some(OffsetDateTime::from_unix_timestamp(1_180).unwrap()), + ..Default::default() + }, + participating: true, + ..Default::default() + }; + + let progress = build_rebalance_pool_progress( + OffsetDateTime::from_unix_timestamp(1_300).unwrap(), + Some(OffsetDateTime::from_unix_timestamp(1_250).unwrap()), + 0.3, + &ps, + ) + .expect("progress should be generated"); + + assert_eq!(progress.elapsed, 180); + assert_eq!(progress.eta, 0); + } + + #[test] + fn test_rebalance_used_pct_normal_and_zero_total() { + assert_eq!(rebalance_used_pct(1_000, 650), 0.35); + assert_eq!(rebalance_used_pct(0, 0), 0.0); + } + + #[test] + fn test_rebalance_used_pct_clamps_available_over_total() { + assert_eq!(rebalance_used_pct(1_000, 1_500), 0.0); + } + + #[test] + fn test_rebalance_remaining_buckets_is_saturating_sub() { + assert_eq!(rebalance_remaining_buckets(10, 7), 3); + assert_eq!(rebalance_remaining_buckets(3, 10), 0); + } + + #[test] + fn test_rebalance_pool_used_defaults_to_zero_when_disk_stat_missing() { + let disk_stats: Vec = vec![]; + assert_eq!(rebalance_pool_used(&disk_stats, 0), 0.0); + } + + #[test] + fn test_build_rebalance_pool_statuses_tracks_progress_for_participants() { + let pool_stats = vec![ + RebalanceStats { + participating: true, + init_capacity: 1_000, + init_free_space: 200, + num_objects: 2, + num_versions: 2, + bytes: 100, + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + rebalanced_buckets: vec!["bucket-a".to_string()], + bucket: "bucket-b".to_string(), + object: "obj-2".to_string(), + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()), + ..Default::default() + }, + }, + RebalanceStats { + participating: false, + info: RebalanceInfo { + status: RebalStatus::Completed, + start_time: Some(OffsetDateTime::from_unix_timestamp(1_000).unwrap()), + ..Default::default() + }, + ..Default::default() + }, + ]; + + let disk_stats = vec![ + DiskStat { + total_space: 1_000, + available_space: 500, + }, + DiskStat { + total_space: 0, + available_space: 0, + }, + ]; + + let statuses = build_rebalance_pool_statuses( + OffsetDateTime::from_unix_timestamp(1_050).unwrap(), + None, + 0.3, + &pool_stats, + &disk_stats, + ); + + assert_eq!(statuses.len(), 2); + + let active = &statuses[0]; + assert_eq!(active.id, 0); + assert_eq!(active.status, "Started"); + assert_eq!(active.used, 0.5); + assert_eq!(active.progress.as_ref().unwrap().bucket, "bucket-b"); + assert_eq!(active.progress.as_ref().unwrap().object, "obj-2"); + assert_eq!(active.progress.as_ref().unwrap().remaining_buckets, 1); + + let inactive = &statuses[1]; + assert_eq!(inactive.id, 1); + assert_eq!(inactive.status, "Completed"); + assert_eq!(inactive.used, 0.0); + assert!(inactive.progress.is_none()); + } + + #[test] + fn test_build_rebalance_pool_statuses_uses_zero_used_for_missing_disk_stats() { + let pool_stats = vec![ + RebalanceStats { + participating: false, + info: RebalanceInfo { + status: RebalStatus::Completed, + ..Default::default() + }, + ..Default::default() + }, + RebalanceStats { + participating: true, + init_capacity: 2_000, + init_free_space: 400, + num_objects: 1, + num_versions: 1, + bytes: 10, + buckets: vec!["bucket-a".to_string()], + rebalanced_buckets: vec![], + bucket: "bucket-a".to_string(), + object: "obj".to_string(), + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(OffsetDateTime::from_unix_timestamp(2_000).unwrap()), + ..Default::default() + }, + }, + ]; + + let statuses = + build_rebalance_pool_statuses(OffsetDateTime::from_unix_timestamp(2_010).unwrap(), None, 0.3, &pool_stats, &[]); + + assert_eq!(statuses[1].used, 0.0); + assert!(statuses[1].progress.is_some()); + } + + #[test] + fn test_build_rebalance_pool_statuses_empty_inputs() { + let statuses = build_rebalance_pool_statuses( + OffsetDateTime::from_unix_timestamp(2_000).unwrap(), + None, + 0.3, + &[], + &[DiskStat { + total_space: 1_000, + available_space: 500, + }], + ); + + assert!(statuses.is_empty()); + } + + #[test] + fn test_rebalance_status_serializes_new_fields() { + let status = RebalanceAdminStatus { + id: "id-1".to_string(), + stopped_at: None, + pools: vec![RebalancePoolStatus { + id: 0, + status: "Started".to_string(), + used: 0.5, + last_error: Some("temporary error".to_string()), + progress: Some(RebalPoolProgress { + num_objects: 3, + num_versions: 5, + bytes: 1024, + remaining_buckets: 2, + bucket: "bucket-a".to_string(), + object: "obj".to_string(), + elapsed: 10, + eta: 20, + }), + }], + }; + + let json = serde_json::to_string(&status).unwrap(); + assert!(json.contains("\"remainingBuckets\"")); + assert!(json.contains("\"lastError\"")); + assert!(json.contains("\"stoppedAt\":null")); + } + + #[test] + fn test_rebalance_status_serializes_stopped_at_when_present() { + let stopped = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let status = RebalanceAdminStatus { + id: "id-2".to_string(), + stopped_at: Some(stopped), + pools: vec![RebalancePoolStatus { + id: 0, + status: "Stopped".to_string(), + used: 0.3, + last_error: None, + progress: None, + }], + }; + + let json = serde_json::to_string(&status).unwrap(); + assert!(json.contains("\"stoppedAt\"")); + assert!(json.contains("1970-01-01T00:16:40Z")); + } +} diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 56a52a5e4..14a2d8352 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -217,6 +217,9 @@ impl From for ApiError { StorageError::BucketExists(_) => S3ErrorCode::BucketAlreadyOwnedByYou, StorageError::StorageFull => S3ErrorCode::ServiceUnavailable, StorageError::SlowDown => S3ErrorCode::SlowDown, + StorageError::DecommissionNotStarted => S3ErrorCode::InvalidRequest, + StorageError::DecommissionAlreadyRunning => S3ErrorCode::InvalidRequest, + StorageError::RebalanceAlreadyRunning => S3ErrorCode::InvalidRequest, StorageError::PrefixAccessDenied(_, _) => S3ErrorCode::AccessDenied, StorageError::InvalidUploadIDKeyCombination(_, _) => S3ErrorCode::InvalidArgument, StorageError::MalformedUploadID(_) => S3ErrorCode::InvalidArgument, @@ -410,6 +413,9 @@ mod tests { (StorageError::BucketExists("test".into()), S3ErrorCode::BucketAlreadyOwnedByYou), (StorageError::StorageFull, S3ErrorCode::ServiceUnavailable), (StorageError::SlowDown, S3ErrorCode::SlowDown), + (StorageError::DecommissionNotStarted, S3ErrorCode::InvalidRequest), + (StorageError::DecommissionAlreadyRunning, S3ErrorCode::InvalidRequest), + (StorageError::RebalanceAlreadyRunning, S3ErrorCode::InvalidRequest), (StorageError::PrefixAccessDenied("test".into(), "test".into()), S3ErrorCode::AccessDenied), (StorageError::ObjectNotFound("test".into(), "test".into()), S3ErrorCode::NoSuchKey), (StorageError::ConfigNotFound, S3ErrorCode::NoSuchKey), diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 97e9eb1df..360f65ed4 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -52,6 +52,10 @@ use tracing::{debug, error, info, warn}; type ResponseStream = Pin> + Send>>; +fn background_rebalance_start_error_message(result: rustfs_ecstore::error::Result<()>) -> Option { + result.err().map(|err| format!("start_rebalance failed: {err}")) +} + #[path = "bucket.rs"] mod bucket; #[path = "disk.rs"] @@ -850,7 +854,9 @@ impl Node for NodeService { warn!("start rebalance"); let store = store.clone(); spawn(async move { - store.start_rebalance().await; + if let Some(message) = background_rebalance_start_error_message(store.start_rebalance().await) { + error!("{message}"); + } }); } @@ -1992,6 +1998,20 @@ mod tests { assert!(load_response.error_info.unwrap().contains("errServerNotInitialized")); } + #[test] + fn test_background_rebalance_start_error_message_ignores_success() { + assert!(background_rebalance_start_error_message(Ok(())).is_none()); + } + + #[test] + fn test_background_rebalance_start_error_message_formats_error() { + let message = background_rebalance_start_error_message(Err(rustfs_ecstore::error::Error::other("boom"))) + .expect("background rebalance start failure should be formatted"); + + assert!(message.contains("start_rebalance failed")); + assert!(message.contains("boom")); + } + #[tokio::test] async fn test_load_bucket_metadata_empty_bucket() { let service = create_test_node_service();