From 2f25cf606e5ca814fe992be6327a91e31fe066b3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 22 Jun 2026 12:03:13 +0800 Subject: [PATCH] fix(storage): harden rebalance decommission state (#3515) --- .github/workflows/ci.yml | 11 + .../delete_marker_migration_semantics_test.rs | 165 ++ crates/e2e_test/src/lib.rs | 4 + crates/ecstore/src/api/mod.rs | 4 +- crates/ecstore/src/data_movement.rs | 1189 +++++++- crates/ecstore/src/notification_sys.rs | 232 +- crates/ecstore/src/object_api/readers.rs | 113 + crates/ecstore/src/object_api/types.rs | 1 + crates/ecstore/src/pools.rs | 2518 ++++++++++++++--- crates/ecstore/src/rebalance.rs | 11 +- crates/ecstore/src/rebalance/control.rs | 102 +- crates/ecstore/src/rebalance/entry.rs | 2 +- crates/ecstore/src/rebalance/meta.rs | 217 +- .../src/rebalance/rebalance_unit_tests.rs | 133 +- crates/ecstore/src/rebalance/runtime.rs | 1 + crates/ecstore/src/rebalance/types.rs | 40 + crates/ecstore/src/rpc/peer_rest_client.rs | 20 +- crates/ecstore/src/rpc/peer_s3_client.rs | 31 +- crates/ecstore/src/set_disk.rs | 168 +- crates/ecstore/src/set_disk/read.rs | 48 + crates/ecstore/src/store.rs | 14 +- crates/ecstore/src/store/init.rs | 134 +- crates/ecstore/src/store/object.rs | 328 ++- .../src/generated/proto_gen/node_service.rs | 7 +- crates/protos/src/node.proto | 4 +- .../decommission-compatibility.md | 165 ++ ...lance-decommission-followup-review-plan.md | 1790 ++++++++++++ ...-decommission-implementation-plan-index.md | 76 + ...balance-decommission-phase1-safety-plan.md | 376 +++ ...-decommission-phase2-data-movement-plan.md | 363 +++ ...ance-decommission-phase3-hardening-plan.md | 320 +++ ...commission-post-remediation-review-plan.md | 132 + ...rebalance-decommission-remediation-plan.md | 546 ++++ rustfs/src/admin/handlers/heal.rs | 62 +- rustfs/src/admin/handlers/pools.rs | 652 ++++- rustfs/src/admin/handlers/rebalance.rs | 654 ++++- rustfs/src/admin/handlers/storage_compat.rs | 16 +- rustfs/src/admin/handlers/tier.rs | 71 +- rustfs/src/admin/route_policy.rs | 1 + rustfs/src/admin/route_registration_test.rs | 1 + rustfs/src/admin/storage_compat.rs | 12 + rustfs/src/app/admin_usecase.rs | 161 +- rustfs/src/storage/rpc/node_service.rs | 91 +- 43 files changed, 10151 insertions(+), 835 deletions(-) create mode 100644 crates/e2e_test/src/delete_marker_migration_semantics_test.rs create mode 100644 docs/architecture/decommission-compatibility.md create mode 100644 docs/architecture/rebalance-decommission-followup-review-plan.md create mode 100644 docs/architecture/rebalance-decommission-implementation-plan-index.md create mode 100644 docs/architecture/rebalance-decommission-phase1-safety-plan.md create mode 100644 docs/architecture/rebalance-decommission-phase2-data-movement-plan.md create mode 100644 docs/architecture/rebalance-decommission-phase3-hardening-plan.md create mode 100644 docs/architecture/rebalance-decommission-post-remediation-review-plan.md create mode 100644 docs/architecture/rebalance-decommission-remediation-plan.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 038eeddd7..28fd3d19d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,6 +127,14 @@ jobs: cargo nextest run --all --exclude e2e_test cargo test --all --doc + - name: Run rebalance/decommission migration proofs + run: | + cargo test -p rustfs-ecstore data_movement --lib + cargo test -p rustfs-ecstore rebalance --lib + cargo test -p rustfs-ecstore decommission --lib + cargo test -p rustfs-ecstore source_cleanup --lib + cargo test -p rustfs-ecstore delete_marker --lib + - name: Check code formatting run: cargo fmt --all --check @@ -258,6 +266,9 @@ jobs: - name: Setup Rust toolchain for s3s-e2e installation uses: dtolnay/rust-toolchain@stable + - name: Run delete-marker migration proof + run: cargo test -p e2e_test delete_marker_migration_semantics -- --nocapture --test-threads=1 + - name: Install s3s-e2e test tool uses: taiki-e/cache-cargo-install-action@v2 with: diff --git a/crates/e2e_test/src/delete_marker_migration_semantics_test.rs b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs new file mode 100644 index 000000000..aac225d7a --- /dev/null +++ b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs @@ -0,0 +1,165 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[cfg(test)] +mod tests { + use crate::common::{RustFSTestEnvironment, init_logging}; + use aws_sdk_s3::Client; + use aws_sdk_s3::primitives::ByteStream; + use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; + use serial_test::serial; + + async fn create_versioned_bucket(client: &Client, bucket: &str) { + client + .create_bucket() + .bucket(bucket) + .send() + .await + .expect("create versioning proof bucket"); + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await + .expect("enable versioning for proof bucket"); + } + + async fn assert_current_get_is_delete_marker_not_found(client: &Client, bucket: &str, key: &str) { + let err = client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect_err("current get should fail when latest version is a delete marker"); + let service_err = err.into_service_error(); + let code = service_err.meta().code(); + assert!( + matches!(code, Some("NoSuchKey") | Some("NotFound")), + "current get should expose delete-marker not-found semantics, got {service_err:?}" + ); + } + + #[tokio::test] + #[serial] + async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() { + init_logging(); + let mut env = RustFSTestEnvironment::new().await.expect("create test environment"); + env.start_rustfs_server(vec![]).await.expect("start RustFS"); + let client = env.create_s3_client(); + let bucket = "delete-marker-migration-proof-a"; + let key = "only-delete-marker.txt"; + + create_versioned_bucket(&client, bucket).await; + + let delete_marker = client + .delete_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("create delete marker for absent object"); + let delete_marker_version_id = delete_marker + .version_id() + .expect("MinIO-compatible delete marker should have a version id"); + + let listed = client + .list_object_versions() + .bucket(bucket) + .prefix(key) + .send() + .await + .expect("list delete marker only object"); + let versions = listed.versions(); + let markers = listed.delete_markers(); + + assert!(versions.is_empty(), "only-delete-marker case must not report data versions"); + assert_eq!(markers.len(), 1, "only-delete-marker case must report the delete marker"); + assert_eq!(markers[0].version_id(), Some(delete_marker_version_id)); + assert_eq!(markers[0].is_latest(), Some(true)); + assert_current_get_is_delete_marker_not_found(&client, bucket, key).await; + } + + #[tokio::test] + #[serial] + async fn test_versioning_delete_marker_plus_history_remains_visible_for_migration_proof() { + init_logging(); + let mut env = RustFSTestEnvironment::new().await.expect("create test environment"); + env.start_rustfs_server(vec![]).await.expect("start RustFS"); + let client = env.create_s3_client(); + let bucket = "delete-marker-migration-proof-b"; + let key = "delete-marker-with-history.txt"; + let body = b"historical version body"; + + create_versioned_bucket(&client, bucket).await; + + let put = client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(body)) + .send() + .await + .expect("put historical version"); + let data_version_id = put.version_id().expect("put should return data version id"); + + let delete_marker = client + .delete_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("create delete marker over historical version"); + let delete_marker_version_id = delete_marker.version_id().expect("delete marker should have a version id"); + + let listed = client + .list_object_versions() + .bucket(bucket) + .prefix(key) + .send() + .await + .expect("list delete marker plus historical version"); + let versions = listed.versions(); + let markers = listed.delete_markers(); + + assert_eq!(versions.len(), 1, "history case must report the historical data version"); + assert_eq!(markers.len(), 1, "history case must report the latest delete marker"); + assert_eq!(versions[0].version_id(), Some(data_version_id)); + assert_eq!(versions[0].is_latest(), Some(false)); + assert_eq!(markers[0].version_id(), Some(delete_marker_version_id)); + assert_eq!(markers[0].is_latest(), Some(true)); + assert_current_get_is_delete_marker_not_found(&client, bucket, key).await; + + let historical = client + .get_object() + .bucket(bucket) + .key(key) + .version_id(data_version_id) + .send() + .await + .expect("historical version get should succeed"); + let bytes = historical + .body + .collect() + .await + .expect("collect historical version body") + .into_bytes(); + assert_eq!(bytes.as_ref(), body); + } +} diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index 4a3265e00..bbf09f87f 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -83,6 +83,10 @@ mod delete_objects_versioning_test; #[cfg(test)] mod delete_object_no_content_length_test; +// Delete-marker visibility baseline for data-movement migration proof. +#[cfg(test)] +mod delete_marker_migration_semantics_test; + // Regression test for Issue #2252: ListObjectVersions misses newest version after put -> delete -> put #[cfg(test)] mod list_object_versions_regression_test; diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 9f7bab95f..e3bd98ed4 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -140,7 +140,9 @@ pub mod object { pub mod rebalance { pub use crate::rebalance::{ - DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, RebalanceStats, + DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, + RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record, + encode_rebalance_stop_propagation_record, }; } diff --git a/crates/ecstore/src/data_movement.rs b/crates/ecstore/src/data_movement.rs index 37af7edcd..ea686e913 100644 --- a/crates/ecstore/src/data_movement.rs +++ b/crates/ecstore/src/data_movement.rs @@ -14,20 +14,26 @@ use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; +use crate::set_disk::SetDisks; use crate::store::ECStore; use bytes::Bytes; -use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; +use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo}; +use rustfs_rio::{ChecksumType, EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; use rustfs_storage_api::{CompletePart, MultipartOperations as _, ObjectIO as _, ObjectOperations as _}; +use rustfs_utils::http::AMZ_OBJECT_TAGGING; use rustfs_utils::path::encode_dir_object; -use std::io::Cursor; +use std::collections::{BTreeMap, HashMap}; use std::pin::Pin; use std::sync::{ - Arc, + Arc, Mutex, atomic::{AtomicBool, Ordering}, }; use std::task::{Context, Poll}; -use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf}; -use tracing::error; +use time::format_description::well_known::Rfc3339; +use tokio::io::{AsyncRead, BufReader, ReadBuf}; +use tracing::{error, info}; + +type SharedDataMovementStream = Arc>>; pub struct IndexedDataMovementReader { inner: R, @@ -66,20 +72,107 @@ pub fn decode_part_index(index: Option<&Bytes>) -> Option { } } -pub fn put_obj_reader_from_chunk(chunk: Vec, size: i64, actual_size: i64, index: Option) -> Result { - use sha2::{Digest, Sha256}; +struct DataMovementPartReader { + inner: SharedDataMovementStream, + remaining: u64, +} - let sha256hex = if !chunk.is_empty() { - Some(hex_simd::encode_to_string(Sha256::digest(&chunk), hex_simd::AsciiCase::Lower)) - } else { - None - }; +impl DataMovementPartReader { + fn new(inner: SharedDataMovementStream, size: u64) -> Self { + Self { inner, remaining: size } + } +} - let reader = IndexedDataMovementReader::new(Cursor::new(chunk), index); - let hash_reader = HashReader::from_stream(reader, size, actual_size, None, sha256hex, false)?; +impl AsyncRead for DataMovementPartReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.remaining == 0 || buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + + let allowed = buf.remaining().min(usize::try_from(self.remaining).unwrap_or(usize::MAX)); + let target = buf.initialize_unfilled_to(allowed); + let mut limited_buf = ReadBuf::new(target); + + let poll = { + let mut inner = self + .inner + .lock() + .map_err(|_| std::io::Error::other("data movement stream lock poisoned"))?; + Pin::new(&mut **inner).poll_read(cx, &mut limited_buf) + }; + + if let Poll::Ready(Ok(())) = &poll { + let read = limited_buf.filled().len(); + buf.advance(read); + self.remaining = self.remaining.saturating_sub(u64::try_from(read).unwrap_or(u64::MAX)); + } + + poll + } +} + +impl EtagResolvable for DataMovementPartReader {} + +impl HashReaderDetector for DataMovementPartReader {} + +fn put_obj_reader_from_part_stream( + stream: SharedDataMovementStream, + size: i64, + actual_size: i64, + index: Option, +) -> Result { + let limit = u64::try_from(size).map_err(|_| Error::other("part size overflow"))?; + let reader = IndexedDataMovementReader::new(DataMovementPartReader::new(stream, limit), index); + let hash_reader = HashReader::from_reader(reader, size, actual_size, None, None, false)?; Ok(PutObjReader::new(hash_reader)) } +fn data_movement_object_checksum_type(object_info: &ObjectInfo) -> Option { + let checksum = object_info.checksum.as_ref()?; + let (checksums, _) = rustfs_rio::read_checksums(checksum.as_ref(), 0); + rustfs_rio::BASE_CHECKSUM_TYPES + .iter() + .copied() + .find(|checksum_type| checksums.contains_key(checksum_type.to_string().as_str())) +} + +fn data_movement_multipart_checksum_type(object_info: &ObjectInfo) -> Option { + let checksum = object_info.user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)?; + let checksum_type = object_info + .user_defined + .get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE) + .map(String::as_str) + .unwrap_or_default(); + let checksum_type = ChecksumType::from_string_with_obj_type(checksum, checksum_type); + checksum_type.is_set().then_some(checksum_type) +} + +fn add_data_movement_calculated_checksum(data: &mut PutObjReader, checksum_type: Option) -> Result<()> { + if let Some(checksum_type) = checksum_type { + data.stream.add_calculated_checksum(checksum_type).map_err(Error::from)?; + } + Ok(()) +} + +fn data_movement_part_checksum(part: &ObjectPartInfo, checksum_type: ChecksumType) -> Option { + part.checksums + .as_ref() + .and_then(|checksums| checksums.get(checksum_type.to_string().as_str())) + .cloned() +} + +fn data_movement_complete_part(part_num: usize, etag: Option, source_part: &ObjectPartInfo) -> CompletePart { + CompletePart { + part_num, + etag, + checksum_crc32: data_movement_part_checksum(source_part, ChecksumType::CRC32), + checksum_crc32c: data_movement_part_checksum(source_part, ChecksumType::CRC32C), + checksum_sha1: data_movement_part_checksum(source_part, ChecksumType::SHA1), + checksum_sha256: data_movement_part_checksum(source_part, ChecksumType::SHA256), + checksum_crc64nvme: data_movement_part_checksum(source_part, ChecksumType::CRC64_NVME), + } +} + pub fn new_multipart_abort_flag() -> Arc { Arc::new(AtomicBool::new(true)) } @@ -96,7 +189,7 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz ObjectOptions { versioned: object_info.version_id.is_some(), version_id: object_info.version_id.as_ref().map(|v| v.to_string()), - user_defined: (*object_info.user_defined).clone(), + user_defined: data_movement_user_defined(object_info), preserve_etag: object_info.etag.clone(), src_pool_idx, data_movement: true, @@ -104,6 +197,19 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz } } +fn data_movement_user_defined(object_info: &ObjectInfo) -> HashMap { + let mut user_defined = (*object_info.user_defined).clone(); + if !object_info.user_tags.is_empty() { + user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), (*object_info.user_tags).clone()); + } + if let Some(expires) = object_info.expires + && let Ok(expires) = expires.format(&Rfc3339) + { + user_defined.insert("expires".to_string(), expires); + } + user_defined +} + fn data_movement_complete_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions { ObjectOptions { versioned: object_info.version_id.is_some(), @@ -123,12 +229,34 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize) data_movement: true, version_id: object_info.version_id.as_ref().map(|v| v.to_string()), mod_time: object_info.mod_time, - user_defined: (*object_info.user_defined).clone(), + user_defined: data_movement_user_defined(object_info), preserve_etag: object_info.etag.clone(), ..Default::default() } } +fn data_movement_put_object_reader( + bucket: &str, + object_info: &ObjectInfo, + rd: GetObjectReader, + op_label: &str, +) -> Result { + let actual_size = object_info + .get_actual_size() + .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", bucket, object_info.name.as_str(), err))?; + let index = object_info + .parts + .first() + .and_then(|part| decode_part_index(part.index.as_ref())); + let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index); + let hrd = HashReader::from_stream(reader, object_info.size, actual_size, None, None, false) + .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", bucket, object_info.name.as_str(), err))?; + let mut data = PutObjReader::new(hrd); + add_data_movement_calculated_checksum(&mut data, data_movement_object_checksum_type(object_info)) + .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", bucket, object_info.name.as_str(), err))?; + Ok(data) +} + fn resolve_data_movement_abort_result( op_label: &str, bucket: &str, @@ -154,6 +282,46 @@ fn effective_actual_size(info: &ObjectInfo) -> Option { info.get_actual_size().ok() } +fn is_equivalent_data_movement_part(source: &ObjectPartInfo, target: &ObjectPartInfo) -> bool { + source.number == target.number + && source.etag == target.etag + && source.size == target.size + && source.actual_size == target.actual_size + && source.mod_time == target.mod_time + && source.index == target.index + && source.checksums == target.checksums +} + +fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option> { + let mut parts_by_number = BTreeMap::new(); + for part in parts { + if parts_by_number.insert(part.number, part).is_some() { + return None; + } + } + + Some(parts_by_number) +} + +fn are_equivalent_data_movement_parts(source: &[ObjectPartInfo], target: &[ObjectPartInfo]) -> bool { + if source.len() != target.len() { + return false; + } + + let Some(source_parts) = data_movement_parts_by_number(source) else { + return false; + }; + let Some(target_parts) = data_movement_parts_by_number(target) else { + return false; + }; + + source_parts.iter().all(|(number, source_part)| { + target_parts + .get(number) + .is_some_and(|target_part| is_equivalent_data_movement_part(source_part, target_part)) + }) +} + fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool { source.version_id == target.version_id && source.delete_marker == target.delete_marker @@ -162,6 +330,158 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) && source.etag == target.etag && source.checksum == target.checksum && source.mod_time == target.mod_time + && source.storage_class == target.storage_class + && source.user_defined == target.user_defined + && source.user_tags == target.user_tags + && source.expires == target.expires + && source.replication_status_internal == target.replication_status_internal + && source.replication_status == target.replication_status + && source.version_purge_status_internal == target.version_purge_status_internal + && source.version_purge_status == target.version_purge_status + && are_equivalent_data_movement_parts(&source.parts, &target.parts) +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct SourceCleanupPartIdentity { + number: usize, + etag: String, + size: usize, + actual_size: i64, + mod_time: Option, + index: Option>, + checksums: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct SourceCleanupVersionIdentity { + name: String, + version_id: Option, + deleted: bool, + mod_time: Option, + size: i64, + etag: Option, + checksum: Option>, + data_dir: Option, + metadata: BTreeMap, + parts: Vec, +} + +fn source_cleanup_part_identity(part: &ObjectPartInfo) -> SourceCleanupPartIdentity { + SourceCleanupPartIdentity { + number: part.number, + etag: part.etag.clone(), + size: part.size, + actual_size: part.actual_size, + mod_time: part.mod_time, + index: part.index.as_ref().map(|index| index.to_vec()), + checksums: part + .checksums + .as_ref() + .map(|checksums| checksums.iter().map(|(key, value)| (key.clone(), value.clone())).collect()) + .unwrap_or_default(), + } +} + +pub(crate) fn source_cleanup_version_identity(version: &FileInfo) -> SourceCleanupVersionIdentity { + let mut parts: Vec<_> = version.parts.iter().map(source_cleanup_part_identity).collect(); + parts.sort(); + + SourceCleanupVersionIdentity { + name: version.name.clone(), + version_id: version.version_id, + deleted: version.deleted, + mod_time: version.mod_time, + size: version.size, + etag: version.get_etag(), + checksum: version.checksum.as_ref().map(|checksum| checksum.to_vec()), + data_dir: version.data_dir, + metadata: version + .metadata + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + parts, + } +} + +fn source_cleanup_version_identities(fivs: &FileInfoVersions) -> Vec { + let mut identities: Vec<_> = fivs.versions.iter().map(source_cleanup_version_identity).collect(); + identities.sort(); + identities +} + +fn source_cleanup_versions_match(expected: &FileInfoVersions, current: &FileInfoVersions) -> bool { + source_cleanup_versions_match_with_allowed_missing(expected, current, &[]) +} + +fn source_cleanup_versions_match_with_allowed_missing( + expected: &FileInfoVersions, + current: &FileInfoVersions, + allowed_missing: &[SourceCleanupVersionIdentity], +) -> bool { + let mut expected_counts = BTreeMap::new(); + for identity in source_cleanup_version_identities(expected) { + *expected_counts.entry(identity).or_insert(0usize) += 1; + } + + for identity in source_cleanup_version_identities(current) { + let Some(count) = expected_counts.get_mut(&identity) else { + return false; + }; + + *count = count.saturating_sub(1); + if *count == 0 { + expected_counts.remove(&identity); + } + } + + let mut allowed_counts = BTreeMap::new(); + for identity in allowed_missing.iter().cloned() { + *allowed_counts.entry(identity).or_insert(0usize) += 1; + } + + expected_counts + .into_iter() + .all(|(identity, count)| allowed_counts.get(&identity).copied().unwrap_or_default() >= count) +} + +fn source_cleanup_preflight_error(op_label: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error { + Error::other(format!("{op_label}: source cleanup preflight failed for {bucket}/{object}: {err}")) +} + +async fn load_source_cleanup_versions( + set: Arc, + bucket: &str, + object: &str, + op_label: &str, +) -> Result> { + set.load_file_info_versions_exact(bucket, object) + .await + .map_err(|err| source_cleanup_preflight_error(op_label, bucket, object, err)) +} + +pub(crate) async fn ensure_source_cleanup_versions_unchanged( + set: Arc, + bucket: &str, + object: &str, + expected: &FileInfoVersions, + allowed_missing: &[SourceCleanupVersionIdentity], + op_label: &str, +) -> Result<()> { + let Some(current) = load_source_cleanup_versions(set, bucket, object, op_label).await? else { + return Ok(()); + }; + + if source_cleanup_versions_match_with_allowed_missing(expected, ¤t, allowed_missing) { + return Ok(()); + } + + Err(source_cleanup_preflight_error( + op_label, + bucket, + object, + "source versions changed after migration started", + )) } fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool { @@ -236,6 +556,32 @@ async fn should_treat_data_movement_overwrite_as_complete( ) } +async fn should_treat_data_movement_overwrite_as_complete_in_any_target_pool( + store: &ECStore, + src_pool_idx: usize, + bucket: &str, + object_info: &ObjectInfo, + err: &Error, +) -> Result { + if !should_check_data_movement_overwrite_resume(err) { + return Ok(false); + } + + for target_pool_idx in 0..store.pools.len() { + if target_pool_idx == src_pool_idx { + continue; + } + + if should_treat_data_movement_overwrite_as_complete(store, src_pool_idx, target_pool_idx, bucket, object_info, err) + .await? + { + return Ok(true); + } + } + + Ok(false) +} + fn data_movement_part_stage_error( op_label: &str, stage: &str, @@ -247,6 +593,26 @@ fn data_movement_part_stage_error( Error::other(format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}")) } +fn is_data_movement_part_read_error(err: &Error) -> bool { + fn is_unexpected_eof(err: &std::io::Error) -> bool { + err.kind() == std::io::ErrorKind::UnexpectedEof + || err + .get_ref() + .and_then(|inner| inner.downcast_ref::()) + .is_some_and(is_unexpected_eof) + } + + matches!(err, Error::Io(io_err) if is_unexpected_eof(io_err)) +} + +fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str { + if is_data_movement_part_read_error(err) { + "read_part" + } else { + "put_object_part" + } +} + pub(crate) async fn migrate_object( store: Arc, pool_idx: usize, @@ -281,21 +647,10 @@ pub(crate) async fn migrate_object( let abort_multipart_flag = new_multipart_abort_flag(); let multipart_result: Result<()> = async { let mut parts = vec![CompletePart::default(); object_info.parts.len()]; - let mut reader = rd.stream; + let reader = Arc::new(Mutex::new(rd.stream)); + let multipart_checksum_type = data_movement_multipart_checksum_type(&object_info); for (i, part) in object_info.parts.iter().enumerate() { - let mut chunk = vec![0u8; part.size]; - reader.read_exact(&mut chunk).await.map_err(|err| { - data_movement_part_stage_error( - op_label, - "read_part", - bucket.as_str(), - object_info.name.as_str(), - part.number, - Error::other(err.to_string()), - ) - })?; - let part_size = i64::try_from(part.size).map_err(|_| { data_movement_part_stage_error( op_label, @@ -308,7 +663,18 @@ pub(crate) async fn migrate_object( })?; let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size }; let index = decode_part_index(part.index.as_ref()); - let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index).map_err(|err| { + let mut data = + put_obj_reader_from_part_stream(reader.clone(), part_size, part_actual_size, index).map_err(|err| { + data_movement_part_stage_error( + op_label, + "prepare_part", + bucket.as_str(), + object_info.name.as_str(), + part.number, + err, + ) + })?; + add_data_movement_calculated_checksum(&mut data, multipart_checksum_type).map_err(|err| { data_movement_part_stage_error( op_label, "prepare_part", @@ -336,9 +702,10 @@ pub(crate) async fn migrate_object( Ok(pi) => pi, Err(err) => { error!("{op_label}: put_object_part {i} err {:?}", &err); + let stage = data_movement_part_upload_failure_stage(&err); return Err(data_movement_part_stage_error( op_label, - "put_object_part", + stage, bucket.as_str(), object_info.name.as_str(), part.number, @@ -347,11 +714,7 @@ pub(crate) async fn migrate_object( } }; - parts[i] = CompletePart { - part_num: pi.part_num, - etag: pi.etag, - ..Default::default() - }; + parts[i] = data_movement_complete_part(pi.part_num, pi.etag, part); } if let Err(err) = store @@ -375,6 +738,11 @@ pub(crate) async fn migrate_object( ) .await? { + info!( + "{op_label}: complete_multipart_upload overwrite resolved by equivalent target for {}/{}", + bucket.as_str(), + object_info.name.as_str() + ); mark_multipart_upload_completed(&abort_multipart_flag); return Ok(()); } @@ -420,19 +788,7 @@ pub(crate) async fn migrate_object( return Ok(()); } - let actual_size = object_info.get_actual_size().map_err(|err| { - data_movement_stage_error(op_label, "prepare_put_object", bucket.as_str(), object_info.name.as_str(), err) - })?; - let index = object_info - .parts - .first() - .and_then(|part| decode_part_index(part.index.as_ref())); - let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index); - let hrd = - HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false).map_err(|err| { - data_movement_stage_error(op_label, "prepare_put_object", bucket.as_str(), object_info.name.as_str(), err) - })?; - let mut data = PutObjReader::new(hrd); + let mut data = data_movement_put_object_reader(bucket.as_str(), &object_info, rd, op_label)?; if let Err(err) = store .put_object( @@ -443,6 +799,23 @@ pub(crate) async fn migrate_object( ) .await { + if should_treat_data_movement_overwrite_as_complete_in_any_target_pool( + store.as_ref(), + pool_idx, + bucket.as_str(), + &object_info, + &err, + ) + .await? + { + info!( + "{op_label}: put_object overwrite resolved by equivalent target for {}/{}", + bucket.as_str(), + object_info.name.as_str() + ); + return Ok(()); + } + error!("{op_label}: put_object err {:?}", &err); return Err(data_movement_stage_error( op_label, @@ -459,9 +832,181 @@ pub(crate) async fn migrate_object( #[cfg(test)] mod tests { use super::*; + use rustfs_rio::HashReaderMut; + use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE}; + use std::collections::HashMap; + use std::io::Cursor; + use std::sync::atomic::AtomicUsize; use time::OffsetDateTime; + use tokio::io::AsyncReadExt; use uuid::Uuid; + struct MaxReadRequestReader { + remaining: u64, + max_request: usize, + largest_request: Arc, + } + + impl MaxReadRequestReader { + fn new(remaining: u64, max_request: usize, largest_request: Arc) -> Self { + Self { + remaining, + max_request, + largest_request, + } + } + } + + impl AsyncRead for MaxReadRequestReader { + fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let requested = buf.remaining(); + self.largest_request.fetch_max(requested, Ordering::Relaxed); + if requested > self.max_request { + return Poll::Ready(Err(std::io::Error::other(format!("oversized read request: {requested}")))); + } + if self.remaining == 0 || requested == 0 { + return Poll::Ready(Ok(())); + } + + let read = requested.min(usize::try_from(self.remaining).unwrap_or(usize::MAX)); + let target = buf.initialize_unfilled_to(read); + target.fill(b'x'); + buf.advance(read); + self.remaining = self.remaining.saturating_sub(u64::try_from(read).unwrap_or(u64::MAX)); + Poll::Ready(Ok(())) + } + } + + fn assert_data_movement_metadata_equivalent(source: &ObjectInfo, target: &ObjectInfo) { + assert_eq!(source.version_id, target.version_id); + assert_eq!(source.etag, target.etag); + assert_eq!(source.size, target.size); + assert_eq!(effective_actual_size(source), effective_actual_size(target)); + assert_eq!(source.mod_time, target.mod_time); + assert_eq!(source.user_defined, target.user_defined); + assert_eq!(source.storage_class, target.storage_class); + assert_eq!(source.checksum, target.checksum); + assert_eq!(source.replication_status_internal, target.replication_status_internal); + assert_eq!(source.replication_status, target.replication_status); + assert_eq!(source.version_purge_status_internal, target.version_purge_status_internal); + assert_eq!(source.version_purge_status, target.version_purge_status); + assert_eq!( + source.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()), + target.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()) + ); + assert_eq!( + source.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()), + target.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()) + ); + assert_eq!( + source.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()), + target.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) + ); + assert_eq!(source.parts.len(), target.parts.len()); + for (source_part, target_part) in source.parts.iter().zip(target.parts.iter()) { + assert_eq!(source_part.number, target_part.number); + assert_eq!(source_part.etag, target_part.etag); + assert_eq!(source_part.size, target_part.size); + assert_eq!(source_part.actual_size, target_part.actual_size); + assert_eq!(source_part.checksums, target_part.checksums); + } + } + + fn cleanup_test_file_info(name: &str, version_id: Uuid, metadata_value: &str) -> FileInfo { + FileInfo { + name: name.to_string(), + version_id: Some(version_id), + size: 128, + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + data_dir: Some(Uuid::from_u128(100)), + checksum: Some(Bytes::from_static(b"object-checksum")), + metadata: HashMap::from([ + ("etag".to_string(), "etag-value".to_string()), + ("x-amz-meta-key".to_string(), metadata_value.to_string()), + ]), + parts: vec![ObjectPartInfo { + number: 1, + etag: "part-etag".to_string(), + size: 128, + actual_size: 128, + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])), + ..Default::default() + }], + ..Default::default() + } + } + + fn cleanup_test_versions(versions: Vec) -> FileInfoVersions { + FileInfoVersions { + name: "object.txt".to_string(), + versions, + ..Default::default() + } + } + + #[test] + fn test_source_cleanup_version_identities_accept_same_versions_out_of_order() { + let first = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "first"); + let second = cleanup_test_file_info("object.txt", Uuid::from_u128(2), "second"); + let expected = cleanup_test_versions(vec![first.clone(), second.clone()]); + let current = cleanup_test_versions(vec![second, first]); + + assert!(source_cleanup_versions_match(&expected, ¤t)); + } + + #[test] + fn test_rebalance_entry_cleanup_preflight_rejects_changed_source_metadata() { + let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]); + let current = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "changed")]); + + assert!(!source_cleanup_versions_match(&expected, ¤t)); + } + + #[test] + fn test_decommission_entry_cleanup_preflight_rejects_added_source_version() { + let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]); + let current = cleanup_test_versions(vec![ + cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source"), + cleanup_test_file_info("object.txt", Uuid::from_u128(2), "new-version"), + ]); + + assert!(!source_cleanup_versions_match(&expected, ¤t)); + } + + #[test] + fn test_decommission_cleanup_preflight_accepts_allowed_expired_missing_version() { + let migrated = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "migrated"); + let expired = cleanup_test_file_info("object.txt", Uuid::from_u128(2), "expired"); + let expected = cleanup_test_versions(vec![migrated.clone(), expired.clone()]); + let current = cleanup_test_versions(vec![migrated]); + let allowed_missing = vec![source_cleanup_version_identity(&expired)]; + + assert!(source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &allowed_missing)); + } + + #[test] + fn test_decommission_cleanup_preflight_rejects_unexpected_missing_version() { + let migrated = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "migrated"); + let protected = cleanup_test_file_info("object.txt", Uuid::from_u128(2), "protected"); + let expected = cleanup_test_versions(vec![migrated.clone(), protected]); + let current = cleanup_test_versions(vec![migrated]); + + assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &[])); + } + + #[test] + fn test_decommission_cleanup_preflight_rejects_new_version_with_allowed_missing() { + let migrated = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "migrated"); + let expired = cleanup_test_file_info("object.txt", Uuid::from_u128(2), "expired"); + let new_version = cleanup_test_file_info("object.txt", Uuid::from_u128(3), "new-version"); + let expected = cleanup_test_versions(vec![migrated.clone(), expired.clone()]); + let current = cleanup_test_versions(vec![migrated, new_version]); + let allowed_missing = vec![source_cleanup_version_identity(&expired)]; + + assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &allowed_missing)); + } + #[test] fn test_new_multipart_abort_flag_defaults_to_abort_enabled() { let flag = new_multipart_abort_flag(); @@ -509,6 +1054,18 @@ mod tests { assert!(message.contains(Error::SlowDown.to_string().as_str())); } + #[test] + fn test_data_movement_part_upload_failure_stage_reports_short_read() { + let err = Error::Io(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "short part")); + + assert_eq!(data_movement_part_upload_failure_stage(&err), "read_part"); + } + + #[test] + fn test_data_movement_part_upload_failure_stage_keeps_write_errors() { + assert_eq!(data_movement_part_upload_failure_stage(&Error::SlowDown), "put_object_part"); + } + #[test] fn test_should_check_data_movement_overwrite_resume_only_for_overwrite_error() { assert!(should_check_data_movement_overwrite_resume(&Error::DataMovementOverwriteErr( @@ -545,6 +1102,326 @@ mod tests { assert_eq!(decoded.total_compressed, 2_097_152); } + #[tokio::test] + async fn test_data_movement_single_part_checksum_is_recalculated_from_source_type() { + let payload = b"checksum-payload"; + let checksum = + rustfs_rio::Checksum::new_from_data(ChecksumType::CRC32C, payload).expect("source checksum should be created"); + let object_info = ObjectInfo { + checksum: Some(checksum.to_bytes(&[])), + ..Default::default() + }; + let mut data = PutObjReader::from_vec(payload.to_vec()); + + add_data_movement_calculated_checksum(&mut data, data_movement_object_checksum_type(&object_info)) + .expect("source checksum type should be enabled on the migrated reader"); + data.stream + .read_to_end(&mut Vec::new()) + .await + .expect("reader should consume payload and calculate checksum"); + + let migrated = data + .stream + .content_hash() + .as_ref() + .expect("migrated reader should contain calculated checksum"); + assert_eq!(migrated.to_bytes(&[]), checksum.to_bytes(&[])); + } + + #[tokio::test] + async fn test_data_movement_single_part_raw_reader_does_not_validate_source_etag() { + let raw_payload = b"raw-encrypted-or-compressed-bytes".to_vec(); + let object_info = ObjectInfo { + name: "object.txt".to_string(), + size: i64::try_from(raw_payload.len()).expect("test payload size should fit i64"), + actual_size: 128, + etag: Some("logical-source-etag".to_string()), + ..Default::default() + }; + let rd = GetObjectReader { + stream: Box::new(Cursor::new(raw_payload.clone())), + object_info: object_info.clone(), + }; + + let mut data = data_movement_put_object_reader("bucket-a", &object_info, rd, "test_migration") + .expect("raw data movement reader should ignore source ETag during stream validation"); + let mut migrated = Vec::new(); + data.stream + .read_to_end(&mut migrated) + .await + .expect("raw data movement reader should consume payload without ETag mismatch"); + + assert_eq!(migrated, raw_payload); + } + + #[test] + fn test_data_movement_single_part_checksum_uses_raw_source_size() { + let object_info = ObjectInfo { + size: 32, + actual_size: 128, + etag: Some("etag-value".to_string()), + checksum: Some(Bytes::new()), + ..Default::default() + }; + + assert_eq!(object_info.size, 32); + assert_eq!(object_info.get_actual_size().expect("actual size should resolve"), 128); + assert_eq!(data_movement_object_checksum_type(&object_info), None); + } + + #[test] + fn test_data_movement_multipart_checksum_type_uses_source_metadata() { + let object_info = ObjectInfo { + user_defined: Arc::new(HashMap::from([ + (rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), ChecksumType::CRC64_NVME.to_string()), + ( + rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE.to_string(), + ChecksumType::CRC64_NVME.obj_type().to_string(), + ), + ])), + ..Default::default() + }; + + assert_eq!(data_movement_multipart_checksum_type(&object_info), Some(ChecksumType::CRC64_NVME)); + } + + #[test] + fn test_data_movement_complete_part_preserves_source_part_checksums() { + let source_part = ObjectPartInfo { + number: 2, + etag: "etag-2".to_string(), + checksums: Some(HashMap::from([ + (ChecksumType::CRC32.to_string(), "crc32-value".to_string()), + (ChecksumType::CRC32C.to_string(), "crc32c-value".to_string()), + (ChecksumType::SHA1.to_string(), "sha1-value".to_string()), + (ChecksumType::SHA256.to_string(), "sha256-value".to_string()), + (ChecksumType::CRC64_NVME.to_string(), "crc64-value".to_string()), + ])), + ..Default::default() + }; + + let complete = data_movement_complete_part(2, Some("etag-2".to_string()), &source_part); + + assert_eq!(complete.part_num, 2); + assert_eq!(complete.etag.as_deref(), Some("etag-2")); + assert_eq!(complete.checksum_crc32.as_deref(), Some("crc32-value")); + assert_eq!(complete.checksum_crc32c.as_deref(), Some("crc32c-value")); + assert_eq!(complete.checksum_sha1.as_deref(), Some("sha1-value")); + assert_eq!(complete.checksum_sha256.as_deref(), Some("sha256-value")); + assert_eq!(complete.checksum_crc64nvme.as_deref(), Some("crc64-value")); + } + + #[test] + fn test_data_movement_part_reader_uses_stored_part_size_for_raw_stream() { + let source_part = ObjectPartInfo { + number: 1, + size: 32, + actual_size: 128, + etag: "etag-1".to_string(), + ..Default::default() + }; + + let part_size = i64::try_from(source_part.size).expect("part size fits"); + let part_actual_size = if source_part.actual_size > 0 { + source_part.actual_size + } else { + part_size + }; + + assert_eq!(part_size, 32); + assert_eq!(part_actual_size, 128); + } + + #[tokio::test] + async fn test_multipart_part_stream_preserves_boundaries() { + let stream: Box = Box::new(Cursor::new(b"abcdef".to_vec())); + let shared = Arc::new(Mutex::new(stream)); + + let mut first = + put_obj_reader_from_part_stream(shared.clone(), 3, 3, None).expect("first bounded part reader should be created"); + let mut first_data = Vec::new(); + first + .stream + .read_to_end(&mut first_data) + .await + .expect("first part should read only its boundary"); + + let mut second = + put_obj_reader_from_part_stream(shared, 3, 3, None).expect("second bounded part reader should be created"); + let mut second_data = Vec::new(); + second + .stream + .read_to_end(&mut second_data) + .await + .expect("second part should continue at the next boundary"); + + assert_eq!(first_data, b"abc"); + assert_eq!(second_data, b"def"); + } + + #[tokio::test] + async fn test_multipart_part_stream_does_not_request_full_large_part() { + let largest_request = Arc::new(AtomicUsize::new(0)); + let stream: Box = + Box::new(MaxReadRequestReader::new(16 * 1024 * 1024, 8 * 1024, largest_request.clone())); + let shared = Arc::new(Mutex::new(stream)); + let mut data = put_obj_reader_from_part_stream(shared, 16 * 1024 * 1024, 16 * 1024 * 1024, None) + .expect("large bounded part reader should be created without allocating part size"); + + let mut buf = [0u8; 8 * 1024]; + let read = data + .stream + .read(&mut buf) + .await + .expect("bounded reader should satisfy a small read against a large advertised part"); + + assert_eq!(read, buf.len()); + assert!(largest_request.load(Ordering::Relaxed) <= buf.len()); + } + + #[tokio::test] + async fn test_multipart_part_stream_reports_short_part() { + let stream: Box = Box::new(Cursor::new(b"abc".to_vec())); + let shared = Arc::new(Mutex::new(stream)); + let mut data = put_obj_reader_from_part_stream(shared, 5, 5, None).expect("short bounded part reader should be created"); + + let err = data + .stream + .read_to_end(&mut Vec::new()) + .await + .expect_err("short source stream should fail the part reader"); + + assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof); + } + + #[test] + fn test_multipart_part_stream_preserves_index() { + let mut index = Index::new(); + index.add(0, 0).expect("index entry should be accepted"); + + let stream: Box = Box::new(Cursor::new(b"abc".to_vec())); + let shared = Arc::new(Mutex::new(stream)); + let data = put_obj_reader_from_part_stream(shared, 3, 3, Some(index)) + .expect("bounded part reader should retain compression index"); + + assert!(data.stream.try_get_index().is_some()); + } + + #[test] + fn test_data_movement_metadata_equivalence_accepts_required_fields() { + let version_id = Uuid::nil(); + let mod_time = OffsetDateTime::UNIX_EPOCH; + let metadata = Arc::new(HashMap::from([ + ("x-amz-meta-key".to_string(), "value".to_string()), + (rustfs_utils::http::AMZ_STORAGE_CLASS.to_string(), "STANDARD_IA".to_string()), + (X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), "GOVERNANCE".to_string()), + ( + X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), + "2030-01-01T00:00:00Z".to_string(), + ), + (X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string()), + ])); + let part = ObjectPartInfo { + number: 1, + etag: "part-etag".to_string(), + size: 128, + actual_size: 128, + checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])), + ..Default::default() + }; + let info = ObjectInfo { + version_id: Some(version_id), + etag: Some("etag-value".to_string()), + size: 128, + actual_size: 128, + mod_time: Some(mod_time), + user_defined: metadata, + storage_class: Some("STANDARD_IA".to_string()), + checksum: Some(Bytes::from_static(b"object-checksum")), + replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()), + replication_status: rustfs_filemeta::ReplicationStatusType::Completed, + version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()), + version_purge_status: rustfs_filemeta::VersionPurgeStatusType::Pending, + parts: Arc::new(vec![part]), + ..Default::default() + }; + + assert_data_movement_metadata_equivalent(&info, &info.clone()); + } + + #[test] + fn test_data_movement_opts_preserve_replication_and_object_lock_metadata() { + let version_id = Uuid::nil(); + let object_info = ObjectInfo { + version_id: Some(version_id), + user_defined: Arc::new(HashMap::from([ + ( + rustfs_utils::http::SUFFIX_REPLICATION_STATUS.to_string(), + "arn:minio:target=PENDING;".to_string(), + ), + (X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), "COMPLIANCE".to_string()), + ( + X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), + "2031-01-01T00:00:00Z".to_string(), + ), + (X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string()), + ])), + ..Default::default() + }; + + let put_opts = data_movement_put_object_opts(&object_info, 3); + let new_multipart_opts = data_movement_new_multipart_opts(&object_info, 3); + + assert_eq!( + put_opts.user_defined.get(rustfs_utils::http::SUFFIX_REPLICATION_STATUS), + Some(&"arn:minio:target=PENDING;".to_string()) + ); + assert_eq!( + new_multipart_opts.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()), + Some(&"COMPLIANCE".to_string()) + ); + assert_eq!( + put_opts.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()), + Some(&"2031-01-01T00:00:00Z".to_string()) + ); + assert_eq!( + new_multipart_opts.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()), + Some(&"ON".to_string()) + ); + } + + #[test] + fn test_data_movement_opts_preserve_tags_and_expires() { + let expires = OffsetDateTime::from_unix_timestamp(2_000).expect("valid timestamp"); + let object_info = ObjectInfo { + user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())])), + user_tags: Arc::new("tag=value".to_string()), + expires: Some(expires), + ..Default::default() + }; + + let put_opts = data_movement_put_object_opts(&object_info, 1); + let multipart_opts = data_movement_new_multipart_opts(&object_info, 1); + + assert_eq!( + put_opts + .user_defined + .get(rustfs_utils::http::AMZ_OBJECT_TAGGING) + .map(String::as_str), + Some("tag=value") + ); + assert_eq!( + multipart_opts + .user_defined + .get(rustfs_utils::http::AMZ_OBJECT_TAGGING) + .map(String::as_str), + Some("tag=value") + ); + assert!(put_opts.user_defined.contains_key("expires")); + assert!(multipart_opts.user_defined.contains_key("expires")); + assert_eq!(put_opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value")); + } + #[test] fn test_data_movement_new_multipart_opts_preserves_etag_and_version() { let version_id = Uuid::nil(); @@ -644,6 +1521,41 @@ mod tests { assert!(!is_equivalent_data_movement_object(&source, &target)); } + #[test] + fn test_is_equivalent_data_movement_object_rejects_user_metadata_mismatch() { + let source = ObjectInfo { + version_id: Some(Uuid::nil()), + size: 128, + etag: Some("etag-value".to_string()), + user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "source".to_string())])), + storage_class: Some("STANDARD_IA".to_string()), + ..Default::default() + }; + let target = ObjectInfo { + user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "target".to_string())])), + ..source.clone() + }; + + assert!(!is_equivalent_data_movement_object(&source, &target)); + } + + #[test] + fn test_is_equivalent_data_movement_object_rejects_storage_class_mismatch() { + let source = ObjectInfo { + version_id: Some(Uuid::nil()), + size: 128, + etag: Some("etag-value".to_string()), + storage_class: Some("STANDARD_IA".to_string()), + ..Default::default() + }; + let target = ObjectInfo { + storage_class: Some("STANDARD".to_string()), + ..source.clone() + }; + + assert!(!is_equivalent_data_movement_object(&source, &target)); + } + #[test] fn test_is_equivalent_data_movement_object_uses_effective_actual_size() { let source = ObjectInfo { @@ -662,6 +1574,146 @@ mod tests { assert!(is_equivalent_data_movement_object(&source, &target)); } + fn overwrite_equivalence_source() -> ObjectInfo { + let part = ObjectPartInfo { + number: 1, + etag: "part-etag".to_string(), + size: 128, + actual_size: 128, + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + index: Some(Bytes::from_static(&[1, 2, 3])), + checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])), + ..Default::default() + }; + + ObjectInfo { + version_id: Some(Uuid::from_u128(1)), + size: 128, + actual_size: 128, + etag: Some("etag-value".to_string()), + checksum: Some(Bytes::from_static(b"object-checksum")), + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())])), + user_tags: Arc::new("tag=value".to_string()), + expires: Some(OffsetDateTime::from_unix_timestamp(2_000).expect("valid expires timestamp")), + storage_class: Some("STANDARD_IA".to_string()), + replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()), + replication_status: rustfs_filemeta::ReplicationStatusType::Completed, + version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()), + version_purge_status: rustfs_filemeta::VersionPurgeStatusType::Pending, + parts: Arc::new(vec![part]), + ..Default::default() + } + } + + fn overwrite_resume_for_target(source: &ObjectInfo, target: ObjectInfo) -> bool { + let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); + resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), source, 0, 1) + .expect("overwrite target should be evaluated") + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_full_equivalence() { + let source = overwrite_equivalence_source(); + + assert!(overwrite_resume_for_target(&source, source.clone())); + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_missing_part_checksum() { + let source = overwrite_equivalence_source(); + let mut target = source.clone(); + let mut parts = target.parts.as_ref().clone(); + parts[0].checksums = None; + target.parts = Arc::new(parts); + + assert!(!overwrite_resume_for_target(&source, target)); + } + + fn overwrite_equivalence_source_with_two_parts() -> ObjectInfo { + let source = overwrite_equivalence_source(); + let mut parts = source.parts.as_ref().clone(); + let mut second_part = parts[0].clone(); + second_part.number = 2; + second_part.etag = "part-etag-2".to_string(); + second_part.index = Some(Bytes::from_static(&[4, 5, 6])); + second_part.checksums = Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum-2".to_string())])); + parts.push(second_part); + + ObjectInfo { + parts: Arc::new(parts), + ..source + } + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_parts_reordered_by_number() { + let source = overwrite_equivalence_source_with_two_parts(); + let mut target = source.clone(); + let mut parts = target.parts.as_ref().clone(); + parts.reverse(); + target.parts = Arc::new(parts); + + assert!(overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_duplicate_part_number() { + let source = overwrite_equivalence_source_with_two_parts(); + let mut target = source.clone(); + let mut parts = target.parts.as_ref().clone(); + parts[1].number = parts[0].number; + target.parts = Arc::new(parts); + + assert!(!overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_version_purge_mismatch() { + let source = overwrite_equivalence_source(); + let target = ObjectInfo { + version_purge_status_internal: Some("arn:minio:replication:target=COMPLETE;".to_string()), + version_purge_status: rustfs_filemeta::VersionPurgeStatusType::Complete, + ..source.clone() + }; + + assert!(!overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_replication_mismatch() { + let source = overwrite_equivalence_source(); + let target = ObjectInfo { + replication_status_internal: Some("arn:minio:replication:target=FAILED;".to_string()), + replication_status: rustfs_filemeta::ReplicationStatusType::Failed, + ..source.clone() + }; + + assert!(!overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_tag_mismatch() { + let source = overwrite_equivalence_source(); + let target = ObjectInfo { + user_tags: Arc::new(String::new()), + ..source.clone() + }; + + assert!(!overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_expires_mismatch() { + let source = overwrite_equivalence_source(); + let target = ObjectInfo { + expires: None, + ..source.clone() + }; + + assert!(!overwrite_resume_for_target(&source, target)); + } + #[test] fn test_resolve_data_movement_overwrite_resume_result_accepts_equivalent_target() { let source = ObjectInfo { @@ -679,6 +1731,24 @@ mod tests { assert!(should_resume); } + #[test] + fn test_rebalance_overwrite_resume_accepts_equivalent_target_version() { + let source = ObjectInfo { + version_id: Some(Uuid::from_u128(1)), + size: 128, + etag: Some("etag-value".to_string()), + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())])), + ..Default::default() + }; + let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); + + let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(source.clone())), &source, 2, 3) + .expect("rebalance overwrite should converge when the target version is equivalent"); + + assert!(should_resume); + } + #[test] fn test_resolve_data_movement_overwrite_resume_result_rejects_source_pool_target() { let source = ObjectInfo { @@ -696,6 +1766,27 @@ mod tests { assert!(!should_resume); } + #[test] + fn test_rebalance_overwrite_resume_rejects_different_target_version() { + let source = ObjectInfo { + version_id: Some(Uuid::from_u128(1)), + size: 128, + etag: Some("etag-value".to_string()), + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + ..Default::default() + }; + let target = ObjectInfo { + version_id: Some(Uuid::from_u128(2)), + ..source.clone() + }; + let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); + + let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &source, 2, 3) + .expect("rebalance overwrite should evaluate a different target version"); + + assert!(!should_resume); + } + #[test] fn test_resolve_data_movement_overwrite_resume_result_rejects_non_equivalent_target() { let source = ObjectInfo { diff --git a/crates/ecstore/src/notification_sys.rs b/crates/ecstore/src/notification_sys.rs index e01e0afee..7d9e0c735 100644 --- a/crates/ecstore/src/notification_sys.rs +++ b/crates/ecstore/src/notification_sys.rs @@ -32,10 +32,13 @@ use std::hash::{Hash, Hasher}; use std::sync::{Mutex, OnceLock}; use std::time::{Duration, SystemTime}; use tokio::time::timeout; -use tracing::{error, warn}; +use tracing::{debug, error, info, warn}; /// After this many consecutive admin-call failures, mark the peer as offline. const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification"; +const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation"; /// Cached result from the last successful admin call to a peer. struct PeerAdminCache { @@ -472,6 +475,15 @@ impl NotificationSys { let host = client.grid_host.clone(); futures.push(async move { client.reload_pool_meta().await.map_err(|err| (host, err)) }); } else { + warn!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "reload_pool_meta", + result = "peer_unreachable", + peer_index = idx, + "notification peer propagation" + ); failures.push(format!("peer[{idx}] reload_pool_meta failed: peer is not reachable")); } } @@ -479,7 +491,16 @@ impl NotificationSys { for result in join_all(futures).await { if let Err((host, err)) = result { let failure = format!("peer {host} reload_pool_meta failed: {err}"); - error!("notification reload_pool_meta err {}", failure); + error!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "reload_pool_meta", + result = "peer_failed", + peer = %host, + error = %err, + "notification peer propagation" + ); failures.push(failure); } } @@ -489,87 +510,187 @@ impl NotificationSys { #[tracing::instrument(skip(self))] pub async fn load_rebalance_meta(&self, start: bool) -> Result<()> { + let failures = self.load_rebalance_meta_failures(start).await?; + aggregate_notification_failures("load_rebalance_meta", failures) + } + + #[tracing::instrument(skip(self))] + pub async fn load_rebalance_meta_failures(&self, start: bool) -> Result> { let operation = format!("load_rebalance_meta(start={start})"); let mut failures = Vec::new(); let mut futures = Vec::with_capacity(self.peer_clients.len()); for (idx, client) in self.peer_clients.iter().enumerate() { if let Some(client) = client { - warn!( - "notification load_rebalance_meta start: {}, index: {}, client: {:?}", - start, idx, client.host - ); let host = client.grid_host.clone(); - futures.push(async move { client.load_rebalance_meta(start).await.map_err(|err| (host, err)) }); + futures.push(async move { + let result = client.load_rebalance_meta(start).await; + (host, result) + }); } else { + warn!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "load_rebalance_meta", + result = "peer_unreachable", + peer_index = idx, + start_rebalance = start, + "notification peer propagation" + ); failures.push(format!("peer[{idx}] {operation} failed: peer is not reachable")); } } - for result in join_all(futures).await { - if let Err((host, err)) = result { + for (host, result) in join_all(futures).await { + if let Err(err) = result { let failure = format!("peer {host} {operation} failed: {err}"); - error!("notification load_rebalance_meta err {}", failure); + error!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "load_rebalance_meta", + result = "peer_failed", + peer = %host, + start_rebalance = start, + error = %err, + "notification peer propagation" + ); failures.push(failure); } else { - warn!("notification load_rebalance_meta success"); + debug!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "load_rebalance_meta", + result = "peer_success", + peer = %host, + start_rebalance = start, + "notification peer propagation" + ); } } - aggregate_notification_failures("load_rebalance_meta", failures) + Ok(failures) } - pub async fn stop_rebalance(&self) -> Result<()> { - warn!("notification stop_rebalance start"); + pub async fn stop_rebalance(&self, expected_rebalance_id: Option<&str>) -> Result<()> { + let failures = self.stop_rebalance_failures(expected_rebalance_id).await?; + aggregate_notification_failures("stop_rebalance", failures) + } + + pub async fn stop_rebalance_failures(&self, expected_rebalance_id: Option<&str>) -> Result> { + info!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "stop_rebalance", + state = "started", + "notification peer propagation" + ); let Some(store) = resolve_object_store_handle() else { - error!("stop_rebalance: not init"); + error!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "stop_rebalance", + result = "failed", + reason = "object_layer_not_initialized", + "notification peer propagation" + ); return Err(Error::other("stop_rebalance: object layer not initialized")); }; - // warn!("notification stop_rebalance load_rebalance_meta"); - // self.load_rebalance_meta(false).await; - // warn!("notification stop_rebalance load_rebalance_meta done"); - let mut failures = Vec::new(); let mut futures = Vec::with_capacity(self.peer_clients.len()); for (idx, client) in self.peer_clients.iter().enumerate() { if let Some(client) = client { let host = client.grid_host.clone(); - futures.push(async move { client.stop_rebalance().await.map_err(|err| (host, err)) }); + futures.push(async move { + let result = client.stop_rebalance(expected_rebalance_id).await; + (host, result) + }); } else { + warn!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "stop_rebalance", + result = "peer_unreachable", + peer_index = idx, + "notification peer propagation" + ); failures.push(format!("peer[{idx}] stop_rebalance failed: peer is not reachable")); } } - for result in join_all(futures).await { - if let Err((host, err)) = result { + for (host, result) in join_all(futures).await { + if let Err(err) = result { let failure = format!("peer {host} stop_rebalance failed: {err}"); - error!("notification stop_rebalance err {}", failure); + error!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "stop_rebalance", + result = "peer_failed", + peer = %host, + error = %err, + "notification peer propagation" + ); failures.push(failure); + } else { + debug!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "stop_rebalance", + result = "peer_success", + peer = %host, + "notification peer propagation" + ); } } - warn!("notification stop_rebalance stop_rebalance start"); - match store.stop_rebalance().await { + match store.stop_rebalance_for_id(expected_rebalance_id).await { Ok(_) => { if let Err(err) = store.save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt).await { - error!("notification stop_rebalance local save err {:?}", err); + error!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "stop_rebalance", + result = "local_save_failed", + error = %err, + "notification peer propagation" + ); return Err(Error::other(format!( "local stop_rebalance save_rebalance_stats(stopped_at) failed: {err}" ))); } } Err(err) => { - error!("notification stop_rebalance local stop err {:?}", err); + error!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "stop_rebalance", + result = "local_stop_failed", + error = %err, + "notification peer propagation" + ); return Err(Error::other(format!("local stop_rebalance stop failed: {err}"))); } } - if let Err(err) = aggregate_notification_failures("stop_rebalance", failures) { - warn!("{err}"); - } - warn!("notification stop_rebalance stop_rebalance done"); - Ok(()) + info!( + event = EVENT_NOTIFICATION_PEER_PROPAGATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + action = "stop_rebalance", + result = if failures.is_empty() { "success" } else { "partial_failure" }, + "notification peer propagation" + ); + Ok(failures) } pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> { @@ -1094,6 +1215,53 @@ mod tests { assert!(msg.contains("local save failed")); } + #[test] + fn load_rebalance_meta_aggregate_failures_return_error() { + let err = aggregate_notification_failures( + "load_rebalance_meta(start=true)", + vec!["peer[0] load_rebalance_meta failed: peer is not reachable".to_string()], + ) + .expect_err("load_rebalance_meta peer failures must be returned"); + + let msg = err.to_string(); + assert!(msg.contains("load_rebalance_meta(start=true)")); + assert!(msg.contains("1 failure(s)")); + assert!(msg.contains("peer[0]")); + } + + #[test] + fn stop_rebalance_aggregate_failures_return_error() { + let err = aggregate_notification_failures( + "stop_rebalance", + vec!["peer[0] stop_rebalance failed: peer is not reachable".to_string()], + ) + .expect_err("stop_rebalance peer failures must be returned"); + + let msg = err.to_string(); + assert!(msg.contains("stop_rebalance")); + assert!(msg.contains("1 failure(s)")); + assert!(msg.contains("peer[0]")); + } + + #[tokio::test] + async fn reload_pool_meta_reports_unreachable_peers() { + let sys = NotificationSys { + peer_clients: vec![None], + all_peer_clients: Vec::new(), + peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())], + }; + + let err = sys + .reload_pool_meta() + .await + .expect_err("unreachable peers should fail pool metadata reload"); + + let msg = err.to_string(); + assert!(msg.contains("reload_pool_meta")); + assert!(msg.contains("1 failure(s)")); + assert!(msg.contains("peer[0]")); + } + #[tokio::test] async fn load_bucket_metadata_reports_unreachable_peers() { let sys = NotificationSys { diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index f41e255aa..bc92ea707 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -314,6 +314,24 @@ impl ReadPlan { rs = http_range_spec_from_object_info(oi, part_number); } + if opts.raw_data_movement_read { + let (visible_offset, visible_length) = if let Some(rs) = rs { + rs.get_offset_length(oi.size)? + } else { + (0, oi.size) + }; + + return Ok(Self { + storage_offset: visible_offset, + storage_length: visible_length, + object_size: oi.size, + transform: ReadTransform::Plain { + visible_offset, + visible_length, + }, + }); + } + let mut is_encrypted = oi.is_encrypted(); let (algo, compression_backend, mut is_compressed) = oi.compression_read_plan()?; @@ -2162,6 +2180,101 @@ mod tests { )); } + #[tokio::test] + async fn test_raw_data_movement_read_plan_bypasses_compression_transform() { + let object_info = ObjectInfo { + size: 3_000_000, + user_defined: Arc::new(HashMap::from([ + ("x-minio-internal-compression".to_string(), "klauspost/compress/s2".to_string()), + ("x-minio-internal-actual-size".to_string(), "4194304".to_string()), + ])), + ..Default::default() + }; + let opts = ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }; + + let plan = ReadPlan::build(None, &object_info, &opts, &HeaderMap::new()) + .await + .expect("raw data movement read should bypass compression planning"); + + assert_eq!(plan.storage_offset, 0); + assert_eq!(plan.storage_length, object_info.size); + assert_eq!(plan.object_size, object_info.size); + assert!(matches!( + plan.transform, + ReadTransform::Plain { + visible_offset: 0, + visible_length: 3_000_000 + } + )); + } + + #[tokio::test] + async fn test_raw_data_movement_read_plan_bypasses_encryption_transform() { + let object_info = ObjectInfo { + size: 128, + user_defined: Arc::new(HashMap::from([ + ("X-Amz-Server-Side-Encryption".to_string(), "aws:kms".to_string()), + ("X-Amz-Server-Side-Encryption-Iv".to_string(), "AAAAAAAAAAAAAAAA".to_string()), + ("X-Amz-Server-Side-Encryption-Key".to_string(), BASE64_STANDARD.encode([7_u8; 32])), + ("x-rustfs-encryption-original-size".to_string(), "64".to_string()), + ])), + ..Default::default() + }; + let opts = ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }; + + let plan = ReadPlan::build(None, &object_info, &opts, &HeaderMap::new()) + .await + .expect("raw data movement read should not require decryption material"); + + assert_eq!(plan.storage_offset, 0); + assert_eq!(plan.storage_length, object_info.size); + assert_eq!(plan.object_size, object_info.size); + assert!(matches!( + plan.transform, + ReadTransform::Plain { + visible_offset: 0, + visible_length: 128 + } + )); + } + + #[tokio::test] + async fn test_raw_data_movement_read_plan_bypasses_ssec_header_resolution() { + let object_info = ObjectInfo { + size: 256, + user_defined: Arc::new(HashMap::from([ + (SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string()), + (SSEC_KEY_MD5_HEADER.to_string(), "stored-key-md5".to_string()), + ])), + ..Default::default() + }; + let opts = ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }; + + let plan = ReadPlan::build(None, &object_info, &opts, &HeaderMap::new()) + .await + .expect("raw data movement read should not require SSE-C request headers"); + + assert_eq!(plan.storage_offset, 0); + assert_eq!(plan.storage_length, object_info.size); + assert_eq!(plan.object_size, object_info.size); + assert!(matches!( + plan.transform, + ReadTransform::Plain { + visible_offset: 0, + visible_length: 256 + } + )); + } + #[tokio::test] async fn test_get_object_reader_allows_encrypted_full_object_passthrough() { async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 1e3296c2e..ad60dc339 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -25,6 +25,7 @@ pub struct ObjectOptions { pub skip_free_version: bool, pub data_movement: bool, + pub raw_data_movement_read: bool, pub src_pool_idx: usize, pub user_defined: HashMap, pub preserve_etag: Option, diff --git a/crates/ecstore/src/pools.rs b/crates/ecstore/src/pools.rs index 80dee6b08..736a68f1d 100644 --- a/crates/ecstore/src/pools.rs +++ b/crates/ecstore/src/pools.rs @@ -30,15 +30,17 @@ use crate::data_movement; use crate::data_usage::DATA_USAGE_CACHE_NAME; use crate::disk::error::DiskError; use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; +use crate::endpoints::EndpointServerPools; use crate::error::{Error, Result}; use crate::error::{ - StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_data_movement_overwrite, is_err_object_not_found, - is_err_operation_canceled, is_err_version_not_found, + StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_object_not_found, is_err_operation_canceled, + is_err_version_not_found, }; use crate::global::resolve_object_store_handle; use crate::notification_sys::get_global_notification_sys; use crate::object_api::{GetObjectReader, ObjectOptions}; -use crate::set_disk::SetDisks; +use crate::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission}; +use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered}; @@ -48,11 +50,10 @@ use rmp_serde::Deserializer; use rmp_serde::Serializer; use rustfs_common::defer; use rustfs_common::heal_channel::HealOpts; -use rustfs_concurrency::workers::Workers; use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_storage_api::{ - BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _, ObjectOperations as _, - StorageAdminApi, + BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, NamespaceLocking as _, ObjectIO as _, + ObjectOperations as _, StorageAdminApi, }; use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration}; @@ -68,6 +69,7 @@ use std::sync::{ atomic::{AtomicUsize, Ordering}, }; use time::{Duration, OffsetDateTime}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -118,6 +120,28 @@ fn bind_decommission_cancelers( bound } +fn bind_missing_decommission_cancelers( + indices: &[usize], + parent: &CancellationToken, + cancelers: &mut [Option], +) -> Vec<(usize, CancellationToken)> { + let mut bound = Vec::with_capacity(indices.len()); + + for idx in indices { + let Some(slot) = cancelers.get_mut(*idx) else { + continue; + }; + if slot.is_some() { + break; + } + let token = parent.child_token(); + *slot = Some(token.clone()); + bound.push((*idx, token)); + } + + bound +} + fn take_decommission_canceler(cancelers: &mut [Option], idx: usize) -> Option { cancelers.get_mut(idx).and_then(Option::take) } @@ -135,6 +159,11 @@ fn cancel_decommission_canceler(canceler: Option) -> bool { } } +fn take_and_cancel_decommission_canceler(cancelers: &mut [Option], idx: usize) -> bool { + let canceler = take_decommission_canceler(cancelers, idx); + cancel_decommission_canceler(canceler) +} + fn ensure_decommission_routines_scheduled(bound_count: usize, expected_count: usize) -> Result<()> { if bound_count == 0 || bound_count != expected_count { return Err(Error::other(format!( @@ -182,6 +211,178 @@ fn ensure_decommission_not_rebalancing(rebalance_running: bool) -> Result<()> { Ok(()) } +fn ensure_decommission_start_rebalance_meta_allowed(meta: Option<&RebalanceMeta>) -> Result<()> { + ensure_decommission_not_rebalancing(meta.is_some_and(is_rebalance_conflicting_with_decommission)) +} + +fn ensure_local_decommission_pool_leaders(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> { + for idx in indices { + ensure_local_decommission_pool_leader(endpoints, *idx)?; + } + + Ok(()) +} + +fn ensure_local_decommission_pool_leader(endpoints: &EndpointServerPools, idx: usize) -> Result<()> { + let pool = endpoints + .as_ref() + .get(idx) + .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?; + let endpoint = pool + .endpoints + .as_ref() + .first() + .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?; + + if !endpoint.is_local { + return Err(Error::other(format!( + "decommission for pool {idx} must run on the pool first endpoint {endpoint}" + ))); + } + + Ok(()) +} + +fn decommission_pool_first_endpoint_is_local(endpoints: &EndpointServerPools, idx: usize) -> Result { + let pool = endpoints + .as_ref() + .get(idx) + .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?; + let endpoint = pool + .endpoints + .as_ref() + .first() + .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?; + + Ok(endpoint.is_local) +} + +pub(crate) fn local_decommission_queue_prefix(endpoints: &EndpointServerPools, indices: &[usize]) -> Result> { + let mut local = Vec::with_capacity(indices.len()); + + for idx in indices { + if decommission_pool_first_endpoint_is_local(endpoints, *idx)? { + local.push(*idx); + } else { + break; + } + } + + Ok(local) +} + +fn first_resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec { + let mut indices = Vec::new(); + for (idx, pool) in meta.pools.iter().enumerate() { + if let Some(decommission) = &pool.decommission { + if decommission.complete { + continue; + } + if decommission.failed || decommission.canceled { + break; + } + indices.push(idx); + } + } + + indices +} + +fn missing_decommission_worker_prefix(indices: &[usize], cancelers: &[Option]) -> Vec { + let mut missing = Vec::with_capacity(indices.len()); + + for idx in indices { + if cancelers.get(*idx).and_then(Option::as_ref).is_some() { + break; + } + missing.push(*idx); + } + + missing +} + +fn ensure_decommission_start_local_leader(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> { + if let Some(first) = indices.first() { + ensure_local_decommission_pool_leader(endpoints, *first)?; + } + + Ok(()) +} + +fn build_decommission_start_state( + pi: PoolSpaceInfo, + queued: bool, + now: OffsetDateTime, + previous: Option<&PoolDecommissionInfo>, +) -> PoolDecommissionInfo { + let mut info = PoolDecommissionInfo { + start_time: if queued { None } else { Some(now) }, + start_size: pi.free, + total_size: pi.total, + current_size: pi.free, + queued, + ..Default::default() + }; + + if let Some(previous) = previous + && (previous.failed || previous.canceled) + { + info.decommissioned_buckets = previous.decommissioned_buckets.clone(); + info.items_decommissioned = previous.items_decommissioned; + info.bytes_done = previous.bytes_done; + info.mark_progress_saved(); + } + + info +} + +fn spawn_decommission_index_cancelers( + store: Arc, + rx: CancellationToken, + index_cancelers: Vec<(usize, CancellationToken)>, +) { + tokio::spawn(async move { + let mut stop_queue = false; + + for (idx, canceler) in index_cancelers { + if stop_queue || rx.is_cancelled() { + canceler.cancel(); + if let Err(err) = store.decommission_cancel(idx).await { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "queued_cancel_failed", + error = %err, + "Failed to cancel queued decommission" + ); + } + continue; + } + + if let Err(err) = store.do_decommission_in_routine(canceler, idx).await { + error!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "routine_failed", + error = %err, + "Decommission routine failed" + ); + stop_queue = true; + continue; + } + + stop_queue = { + let pool_meta = store.pool_meta.read().await; + !should_continue_decommission_queue(&pool_meta, idx) + }; + } + }); +} + fn decommission_meta_bucket_options() -> MakeBucketOptions { MakeBucketOptions { force_create: true, @@ -193,6 +394,18 @@ fn is_decommission_active(complete: bool, failed: bool, canceled: bool) -> bool !complete && !failed && !canceled } +pub(crate) fn pool_meta_has_active_decommission(meta: &PoolMeta) -> bool { + meta.pools.iter().any(|pool| { + pool.decommission + .as_ref() + .is_some_and(|info| is_decommission_active(info.complete, info.failed, info.canceled)) + }) +} + +fn is_decommission_suspended(info: &PoolDecommissionInfo) -> bool { + !info.queued +} + fn validate_decommission_terminal_state(complete: bool, failed: bool, canceled: bool) -> Result<()> { let terminal_count = [complete, failed, canceled].into_iter().filter(|terminal| *terminal).count(); if terminal_count > 1 { @@ -207,7 +420,7 @@ fn invalid_decommission_pool_index_error(pool_count: usize, idx: usize) -> Error Error::other(format!("invalid decommission pool index {idx} for {pool_count} pools")) } -fn ensure_decommission_start_allowed(pool_present: bool, decommission_active: bool) -> Result<()> { +fn ensure_decommission_start_allowed(pool_present: bool, decommission_active: bool, decommission_complete: bool) -> Result<()> { if !pool_present { return Err(Error::other("failed to start decommission: target pool was not found")); } @@ -216,6 +429,10 @@ fn ensure_decommission_start_allowed(pool_present: bool, decommission_active: bo return Err(StorageError::DecommissionAlreadyRunning); } + if decommission_complete { + return Err(Error::other("failed to start decommission: target pool decommission is already complete")); + } + Ok(()) } @@ -310,6 +527,12 @@ fn resolve_decommission_update_after_result(result: Result) -> Result) -> Option { + result + .err() + .map(|err| Error::other(format!("decommission progress save failed: {err}"))) +} + fn resolve_decommission_preflight_heal_result(bucket: &str, result: Result) -> Result { result.map_err(|err| Error::other(format!("decommission preflight heal failed for bucket {bucket}: {err}"))) } @@ -354,6 +577,12 @@ fn resolve_decommission_terminal_mark_after_error_result(result: Result<()>, idx }) } +fn observe_decommission_terminal_reload_result(result: Result<()>, stage: &str) -> Option { + result + .err() + .map(|err| Error::other(format!("decommission terminal pool meta reload failed during {stage}: {err}"))) +} + fn resolve_decommission_spawn_failure_result(spawn_err: Error, rollback_err: Option) -> Error { if let Some(rollback_err) = rollback_err { Error::other(format!( @@ -389,6 +618,48 @@ fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) result.map_err(|err| Error::other(format!("decommission pool meta reload failed during {stage}: {err}"))) } +fn resolve_start_decommission_pool_meta_reload_result(result: Result<()>) -> Result<()> { + resolve_decommission_pool_meta_reload_result(result, "start_decommission") +} + +fn decommission_rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error { + match err { + rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable { + mode: "write", + bucket: RUSTFS_META_BUCKET.to_string(), + object: REBAL_META_NAME.to_string(), + required, + achieved, + }, + other => Error::other(format!( + "failed to acquire rebalance metadata write lock before decommission start on {RUSTFS_META_BUCKET}/{REBAL_META_NAME}: {other}" + )), + } +} + +fn decommission_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error { + match err { + rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable { + mode: "write", + bucket: RUSTFS_META_BUCKET.to_string(), + object: POOL_META_NAME.to_string(), + required, + achieved, + }, + other => Error::other(format!( + "failed to acquire pool metadata write lock before decommission start on {RUSTFS_META_BUCKET}/{POOL_META_NAME}: {other}" + )), + } +} + +fn rollback_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: PoolMeta) { + *pool_meta = previous_pool_meta; +} + +fn rollback_start_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: PoolMeta) { + rollback_decommission_pool_meta(pool_meta, previous_pool_meta); +} + fn ensure_pool_not_left_in_cmdline_after_decommission(position: usize, cmd_line: &str, completed: bool) -> Result<()> { if completed { return Err(Error::other(format!( @@ -412,19 +683,26 @@ fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: boo cleanup_ignored || (!ignore && !failure) } -fn should_cleanup_decommission_source_entry(decommissioned: usize, total_versions: usize, expired: usize) -> bool { - expired == 0 && decommissioned == total_versions +fn is_decommission_copy_cleanup_safe_error(err: &Error) -> bool { + // DataMovementOverwriteErr only means source and destination pool resolved to + // the same pool. Without a target equivalence check it is not cleanup-safe. + is_err_object_not_found(err) || is_err_version_not_found(err) } -fn decommission_start_guard_state(pool: Option<&PoolStatus>) -> (bool, bool) { +fn should_cleanup_decommission_source_entry(decommissioned: usize, total_versions: usize, expired: usize) -> bool { + decommissioned.saturating_add(expired) == total_versions +} + +fn decommission_start_guard_state(pool: Option<&PoolStatus>) -> (bool, bool, bool) { if let Some(pool) = pool { let active = pool .decommission .as_ref() .is_some_and(|info| is_decommission_active(info.complete, info.failed, info.canceled)); - (true, active) + let complete = pool.decommission.as_ref().is_some_and(|info| info.complete); + (true, active, complete) } else { - (false, false) + (false, false, false) } } @@ -461,6 +739,20 @@ fn decommission_cancel_signal_result(cancel_signal: bool) -> Result<()> { } } +fn is_decommission_cancel_requested(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool { + cancel_signal + || pool + .and_then(|pool| pool.decommission.as_ref()) + .is_some_and(|info| info.canceled) +} + +fn should_skip_canceled_decommission_routine(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool { + cancel_signal + && pool + .and_then(|pool| pool.decommission.as_ref()) + .is_some_and(|info| info.canceled) +} + async fn run_decommission_buckets_bounded( rx: CancellationToken, buckets: Vec, @@ -514,8 +806,22 @@ where Ok(()) } -fn is_decommission_cancel_terminal(complete: bool, failed: bool, canceled: bool) -> bool { - complete || failed || canceled +async fn wait_decommission_worker_drain(workers: &Semaphore, limit: usize) -> Result<()> { + let permits = u32::try_from(limit) + .map_err(|_| Error::other(format!("decommission worker limit {limit} exceeds semaphore drain capacity")))?; + let _drain = workers + .acquire_many(permits) + .await + .map_err(|err| Error::other(format!("decommission worker drain failed: {err}")))?; + Ok(()) +} + +fn should_reject_decommission_cancel_as_terminal(complete: bool, failed: bool) -> bool { + complete || failed +} + +fn should_retry_decommission_cancel_reload(changed: bool, already_canceled: bool) -> bool { + changed || already_canceled } fn ensure_decommission_cancel_allowed(pool_present: bool, decommission_present: bool, terminal: bool) -> Result<()> { @@ -530,6 +836,32 @@ fn ensure_decommission_cancel_allowed(pool_present: bool, decommission_present: Ok(()) } +fn ensure_decommission_clear_allowed( + pool_present: bool, + decommission_present: bool, + complete: bool, + failed: bool, + canceled: bool, +) -> Result<()> { + if !pool_present { + return Err(Error::other("failed to clear decommission: target pool was not found")); + } + + if !decommission_present { + return Err(StorageError::DecommissionNotStarted); + } + + if complete { + return Err(StorageError::DecommissionNotStarted); + } + + if !failed && !canceled { + return Err(StorageError::DecommissionAlreadyRunning); + } + + Ok(()) +} + fn ensure_decommission_terminal_operation_supported(single_pool: bool, operation: &str) -> Result<()> { if single_pool { return Err(Error::other(format!( @@ -545,12 +877,6 @@ fn validate_start_decommission_request(indices: &[usize], single_pool: bool) -> return Err(Error::other("failed to start decommission: no target pools were provided")); } - if indices.len() > 1 { - return Err(Error::other( - "failed to start decommission: decommission supports one target pool at a time", - )); - } - ensure_decommission_terminal_operation_supported(single_pool, "start decommission") } @@ -588,12 +914,14 @@ pub struct PoolMeta { } #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct PersistedPoolMeta { pub version: u16, pub pools: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct PersistedPoolStatus { #[serde(rename = "id")] pub id: usize, @@ -606,6 +934,7 @@ struct PersistedPoolStatus { } #[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct PersistedPoolDecommissionInfo { #[serde(rename = "startTime", with = "time::serde::rfc3339::option")] pub start_time: Option, @@ -621,6 +950,8 @@ struct PersistedPoolDecommissionInfo { pub failed: bool, #[serde(rename = "canceled")] pub canceled: bool, + #[serde(rename = "queued", default)] + pub queued: bool, #[serde(rename = "queuedBuckets", default)] pub queued_buckets: Vec, #[serde(rename = "decommissionedBuckets", default)] @@ -641,6 +972,54 @@ struct PersistedPoolDecommissionInfo { pub bytes_failed: usize, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyPoolMeta { + pub version: u16, + pub pools: Vec, + pub dont_save: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyPoolStatus { + #[serde(rename = "id")] + pub id: usize, + #[serde(rename = "cmdline")] + pub cmd_line: String, + #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")] + pub last_update: OffsetDateTime, + #[serde(rename = "decommissionInfo")] + pub decommission: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyPoolDecommissionInfo { + #[serde(rename = "startTime", with = "time::serde::rfc3339::option")] + pub start_time: Option, + #[serde(rename = "startSize")] + pub start_size: usize, + #[serde(rename = "totalSize")] + pub total_size: usize, + #[serde(rename = "currentSize")] + pub current_size: usize, + #[serde(rename = "complete")] + pub complete: bool, + #[serde(rename = "failed")] + pub failed: bool, + #[serde(rename = "canceled")] + pub canceled: bool, + #[serde(rename = "objectsDecommissioned")] + pub items_decommissioned: usize, + #[serde(rename = "objectsDecommissionedFailed")] + pub items_decommission_failed: usize, + #[serde(rename = "bytesDecommissioned")] + pub bytes_done: usize, + #[serde(rename = "bytesDecommissionedFailed")] + pub bytes_failed: usize, +} + impl TryFrom for PoolMeta { type Error = Error; @@ -653,6 +1032,23 @@ impl TryFrom for PoolMeta { } } +impl TryFrom for PoolMeta { + type Error = Error; + + fn try_from(value: LegacyPoolMeta) -> Result { + let LegacyPoolMeta { + version, + pools, + dont_save: _, + } = value; + Ok(Self { + version, + pools: pools.into_iter().map(TryInto::try_into).collect::>>()?, + dont_save: false, + }) + } +} + impl TryFrom for PoolStatus { type Error = Error; @@ -666,6 +1062,19 @@ impl TryFrom for PoolStatus { } } +impl TryFrom for PoolStatus { + type Error = Error; + + fn try_from(value: LegacyPoolStatus) -> Result { + Ok(Self { + id: value.id, + cmd_line: value.cmd_line, + last_update: value.last_update, + decommission: value.decommission.map(TryInto::try_into).transpose()?, + }) + } +} + impl TryFrom for PoolDecommissionInfo { type Error = Error; @@ -679,6 +1088,7 @@ impl TryFrom for PoolDecommissionInfo { complete: value.complete, failed: value.failed, canceled: value.canceled, + queued: value.queued, queued_buckets: value.queued_buckets, decommissioned_buckets: value.decommissioned_buckets, bucket: value.bucket, @@ -693,6 +1103,34 @@ impl TryFrom for PoolDecommissionInfo { } } +impl TryFrom for PoolDecommissionInfo { + type Error = Error; + + fn try_from(value: LegacyPoolDecommissionInfo) -> Result { + validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?; + Ok(Self { + start_time: value.start_time, + start_size: value.start_size, + total_size: value.total_size, + current_size: value.current_size, + complete: value.complete, + failed: value.failed, + canceled: value.canceled, + queued: false, + queued_buckets: Vec::new(), + decommissioned_buckets: Vec::new(), + bucket: String::new(), + prefix: String::new(), + object: String::new(), + items_decommissioned: value.items_decommissioned, + items_decommission_failed: value.items_decommission_failed, + bytes_done: value.bytes_done, + bytes_failed: value.bytes_failed, + progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed), + }) + } +} + impl From<&PoolMeta> for PersistedPoolMeta { fn from(value: &PoolMeta) -> Self { Self { @@ -723,6 +1161,7 @@ impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo { complete: value.complete, failed: value.failed, canceled: value.canceled, + queued: value.queued, queued_buckets: value.queued_buckets.clone(), decommissioned_buckets: value.decommissioned_buckets.clone(), bucket: value.bucket.clone(), @@ -741,20 +1180,12 @@ impl PoolMeta { match rmp_serde::from_slice::(payload) { Ok(meta) => meta.try_into(), Err(persisted_err) => { - let mut legacy: PoolMeta = rmp_serde::from_slice(payload).map_err(|legacy_err| { + let legacy: LegacyPoolMeta = rmp_serde::from_slice(payload).map_err(|legacy_err| { Error::other(format!( "PoolMeta decode failed for both persisted and legacy formats: persisted={persisted_err}; legacy={legacy_err}" )) })?; - // Runtime-only flag must not be restored from on-disk payload. - legacy.dont_save = false; - for pool in &legacy.pools { - if let Some(decommission) = &pool.decommission { - validate_decommission_terminal_state(decommission.complete, decommission.failed, decommission.canceled)?; - } - } - legacy.mark_decommission_progress_saved(); - Ok(legacy) + legacy.try_into() } } } @@ -793,7 +1224,10 @@ impl PoolMeta { } pub fn is_suspended(&self, idx: usize) -> bool { - self.pools.get(idx).is_some_and(|pool| pool.decommission.is_some()) + self.pools + .get(idx) + .and_then(|pool| pool.decommission.as_ref()) + .is_some_and(is_decommission_suspended) } fn mark_decommission_progress_saved(&mut self) { @@ -869,6 +1303,7 @@ impl PoolMeta { pd.canceled = true; pd.failed = false; pd.complete = false; + pd.start_time = None; stats.decommission = Some(pd); true @@ -892,6 +1327,7 @@ impl PoolMeta { pd.canceled = false; pd.failed = true; pd.complete = false; + pd.start_time = None; stats.decommission = Some(pd); true @@ -905,6 +1341,28 @@ impl PoolMeta { false } } + + pub fn clear_decommission(&mut self, idx: usize) -> Result { + let pool_count = self.pools.len(); + ensure_valid_decommission_pool_index(pool_count, idx)?; + + let Some(pool) = self.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + + let (decommission_present, complete, failed, canceled) = pool + .decommission + .as_ref() + .map(|info| (true, info.complete, info.failed, info.canceled)) + .unwrap_or((false, false, false, false)); + + ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled)?; + + pool.last_update = OffsetDateTime::now_utc(); + pool.decommission = None; + Ok(true) + } + pub fn decommission_complete(&mut self, idx: usize) -> bool { if let Some(stats) = self.pools.get_mut(idx) { if let Some(d) = &stats.decommission { @@ -928,7 +1386,7 @@ impl PoolMeta { false } } - pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> { + fn set_decommission_state(&mut self, idx: usize, pi: PoolSpaceInfo, queued: bool) -> Result<()> { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; @@ -940,20 +1398,40 @@ impl PoolMeta { .decommission .as_ref() .is_some_and(|info| is_decommission_active(info.complete, info.failed, info.canceled)); - ensure_decommission_start_allowed(true, decommission_active)?; + let decommission_complete = pool.decommission.as_ref().is_some_and(|info| info.complete); + ensure_decommission_start_allowed(true, decommission_active, decommission_complete)?; + let previous = pool.decommission.as_ref(); let now = OffsetDateTime::now_utc(); pool.last_update = now; - pool.decommission = Some(PoolDecommissionInfo { - start_time: Some(now), - start_size: pi.free, - total_size: pi.total, - current_size: pi.free, - ..Default::default() - }); + pool.decommission = Some(build_decommission_start_state(pi, queued, now, previous)); Ok(()) } + + pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> { + self.set_decommission_state(idx, pi, false) + } + + pub fn queue_decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> { + self.set_decommission_state(idx, pi, true) + } + + pub fn promote_queued_decommission(&mut self, idx: usize) -> bool { + if let Some(pool) = self.pools.get_mut(idx) + && let Some(info) = pool.decommission.as_mut() + && info.queued + && is_decommission_active(info.complete, info.failed, info.canceled) + { + let now = OffsetDateTime::now_utc(); + pool.last_update = now; + info.queued = false; + info.start_time.get_or_insert(now); + return true; + } + + false + } pub fn queue_buckets(&mut self, idx: usize, bks: Vec) { if let Some(pool) = self.pools.get_mut(idx) && let Some(dec) = pool.decommission.as_mut() @@ -1024,7 +1502,7 @@ impl PoolMeta { } } - pub async fn update_after(&mut self, idx: usize, pools: Vec>, duration: Duration) -> Result { + pub fn update_after(&mut self, idx: usize, duration: Duration) -> Result { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; @@ -1045,14 +1523,6 @@ impl PoolMeta { return Err(invalid_decommission_pool_index_error(pool_count, idx)); }; pool.last_update = now; - self.save(pools).await?; - let Some(pool) = self.pools.get_mut(idx) else { - return Err(invalid_decommission_pool_index_error(pool_count, idx)); - }; - if let Some(info) = pool.decommission.as_mut() { - info.mark_progress_saved(); - } - return Ok(true); } @@ -1125,9 +1595,10 @@ impl PoolMeta { let mut new_pools = Vec::new(); for pool in &self.pools { if let Some(decommission) = &pool.decommission { - if decommission.complete || decommission.canceled { + if decommission.complete || decommission.failed || decommission.canceled { // Recovery is not required when: // - Decommissioning completed + // - Decommissioning failed and must be explicitly restarted or cleared // - Decommissioning was cancelled continue; } @@ -1163,6 +1634,8 @@ pub struct PoolDecommissionInfo { pub failed: bool, #[serde(rename = "canceled")] pub canceled: bool, + #[serde(skip)] + pub queued: bool, #[serde(skip)] pub queued_buckets: Vec, @@ -1201,17 +1674,18 @@ impl PoolDecommissionInfo { } pub fn bucket_push(&mut self, bucket: &DecomBucketInfo) { - for b in self.queued_buckets.iter() { - if self.is_bucket_decommissioned(b) { - return; - } + let bucket_key = bucket.to_string(); + if self.is_bucket_decommissioned(&bucket_key) { + return; + } - if b == &bucket.to_string() { + for b in self.queued_buckets.iter() { + if b == &bucket_key { return; } } - self.queued_buckets.push(bucket.to_string()); + self.queued_buckets.push(bucket_key); self.bucket = bucket.name.clone(); self.prefix = bucket.prefix.clone(); @@ -1295,6 +1769,8 @@ fn should_skip_decommission_delete_marker( remaining_versions: usize, replication_configured: bool, ) -> bool { + // Match MinIO decommission behavior: an empty delete marker is not moved to + // another pool unless replication is configured and its marker state matters. version.deleted && remaining_versions == 1 && !replication_configured } @@ -1316,6 +1792,18 @@ fn decommission_delete_marker_opts( } } +fn decommission_object_migration_read_opts(version_id: Option) -> ObjectOptions { + ObjectOptions { + version_id, + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + skip_decommissioned: true, + skip_rebalancing: true, + ..Default::default() + } +} + fn decommission_remote_tiered_opts( version: &rustfs_filemeta::FileInfo, version_id: Option, @@ -1332,6 +1820,23 @@ fn decommission_remote_tiered_opts( } } +fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool { + matches!( + action, + IlmAction::DeleteVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction + ) +} + +fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result { + if !apply_actions || applied { + return Ok(true); + } + + Err(Error::other(format!( + "failed to apply lifecycle expiry action {action:?} during data movement" + ))) +} + #[allow(clippy::too_many_arguments)] pub(crate) async fn should_skip_lifecycle_for_data_movement( store: Arc, @@ -1342,9 +1847,9 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement( replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, apply_actions: bool, event_source: &LcEventSrc, -) -> bool { +) -> Result { let Some(lifecycle_config) = lifecycle_config else { - return false; + return Ok(false); }; let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await; @@ -1356,22 +1861,105 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement( if apply_actions && object_info.is_remote() { let _ = apply_expiry_on_transitioned_object(store, &object_info, &event, event_source).await; } - false + Ok(false) } - IlmAction::DeleteAction - | IlmAction::DeleteVersionAction - | IlmAction::DeleteAllVersionsAction - | IlmAction::DelMarkerDeleteAllVersionsAction => { - if apply_actions { - let _ = apply_expiry_rule(&event, event_source, &object_info).await; - } - true + action if lifecycle_action_removes_data_movement_version(action) => { + let applied = !apply_actions || apply_expiry_rule(&event, event_source, &object_info).await; + resolve_data_movement_lifecycle_expiry_result(action, apply_actions, applied) } - _ => false, + _ => Ok(false), } } impl ECStore { + async fn save_current_pool_meta(&self) -> Result<()> { + let _save_guard = self.pool_meta_save_gate.lock().await; + let snapshot = { + let pool_meta = self.pool_meta.read().await; + pool_meta.clone() + }; + snapshot.save(self.pools.clone()).await + } + + async fn save_current_pool_meta_for_decommission_start( + &self, + indices: &[usize], + space_infos: Vec<(usize, PoolSpaceInfo)>, + decom_buckets: Vec, + ) -> Result { + let _save_guard = self.pool_meta_save_gate.lock().await; + let rebalance_pool = self + .pools + .first() + .cloned() + .ok_or_else(|| Error::other("decommission start rebalance metadata load failed: no storage pools available"))?; + let pool_meta_lock = rebalance_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?; + let _pool_meta_guard = pool_meta_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(decommission_pool_meta_lock_error)?; + let ns_lock = rebalance_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?; + let _guard = ns_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(decommission_rebalance_meta_lock_error)?; + + let mut rebalance_meta = RebalanceMeta::new(); + match rebalance_meta + .load_with_opts( + rebalance_pool.clone(), + ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(()) => ensure_decommission_start_rebalance_meta_allowed(Some(&rebalance_meta))?, + Err(Error::ConfigNotFound) => {} + Err(err) => { + return Err(Error::other(format!( + "rebalance metadata load before decommission start save failed: {err}" + ))); + } + } + + let current_pool_meta = { + let pool_meta = self.pool_meta.read().await; + pool_meta.clone() + }; + let mut latest_pool_meta = PoolMeta::default(); + latest_pool_meta.load(rebalance_pool, self.pools.clone()).await?; + if latest_pool_meta.pools.is_empty() { + latest_pool_meta = current_pool_meta; + } + + for idx in indices.iter().copied() { + let (pool_present, decommission_active, decommission_complete) = + decommission_start_guard_state(latest_pool_meta.pools.get(idx)); + ensure_decommission_start_allowed(pool_present, decommission_active, decommission_complete)?; + } + + let previous_pool_meta = latest_pool_meta.clone(); + let first_idx = indices.first().copied(); + for (idx, pi) in space_infos { + if Some(idx) == first_idx { + latest_pool_meta.decommission(idx, pi)?; + } else { + latest_pool_meta.queue_decommission(idx, pi)?; + } + latest_pool_meta.queue_buckets(idx, decom_buckets.clone()); + } + + latest_pool_meta.save(self.pools.clone()).await?; + { + let mut pool_meta = self.pool_meta.write().await; + *pool_meta = latest_pool_meta; + } + + Ok(previous_pool_meta) + } + pub async fn status(&self, idx: usize) -> Result { let space_info = self.get_decommission_pool_space_info(idx).await?; @@ -1414,32 +2002,36 @@ impl ECStore { pub async fn decommission_cancel(&self, idx: usize) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; - let mut lock = self.pool_meta.write().await; - let (pool_present, decommission_present, terminal) = if let Some(pool) = lock.pools.get(idx) { - if let Some(info) = pool.decommission.as_ref() { - (true, true, is_decommission_cancel_terminal(info.complete, info.failed, info.canceled)) + let (should_save_pool_meta, should_reload_pool_meta, already_canceled, previous_pool_meta) = { + let mut lock = self.pool_meta.write().await; + let mut already_canceled = false; + let (pool_present, decommission_present, terminal) = if let Some(pool) = lock.pools.get(idx) { + if let Some(info) = pool.decommission.as_ref() { + already_canceled = info.canceled; + (true, true, should_reject_decommission_cancel_as_terminal(info.complete, info.failed)) + } else { + (true, false, false) + } } else { - (true, false, false) - } - } else { - (false, false, false) + (false, false, false) + }; + + ensure_decommission_cancel_allowed(pool_present, decommission_present, terminal)?; + let previous_pool_meta = lock.clone(); + let changed = lock.decommission_cancel(idx); + ( + changed, + should_retry_decommission_cancel_reload(changed, already_canceled), + already_canceled, + changed.then_some(previous_pool_meta), + ) }; - ensure_decommission_cancel_allowed(pool_present, decommission_present, terminal)?; - - let should_reload_pool_meta = if lock.decommission_cancel(idx) { - lock.save(self.pools.clone()).await?; - true - } else { - false - }; - drop(lock); - - let canceler = { + let canceled_worker = { let mut cancelers = self.decommission_cancelers.write().await; - take_decommission_canceler(cancelers.as_mut_slice(), idx) + take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx) }; - if !cancel_decommission_canceler(canceler) { + if !canceled_worker && !already_canceled { warn!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, @@ -1451,17 +2043,71 @@ impl ECStore { ); } + if should_save_pool_meta && let Err(err) = self.save_current_pool_meta().await { + if let Some(previous_pool_meta) = previous_pool_meta { + let mut pool_meta = self.pool_meta.write().await; + rollback_decommission_pool_meta(&mut pool_meta, previous_pool_meta); + } + return Err(err); + } + if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() { let stage = format!("decommission_cancel for pool {idx}"); - if let Err(err) = - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) - { - warn!("{err}"); + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; + } + + Ok(()) + } + + #[tracing::instrument(skip(self))] + pub async fn clear_decommission(&self, idx: usize) -> Result<()> { + ensure_decommission_terminal_operation_supported(self.single_pool(), "clear decommission")?; + + let (should_reload_pool_meta, previous_pool_meta) = { + let mut pool_meta = self.pool_meta.write().await; + let previous_pool_meta = pool_meta.clone(); + let changed = pool_meta.clear_decommission(idx)?; + (changed, changed.then_some(previous_pool_meta)) + }; + + { + let mut cancelers = self.decommission_cancelers.write().await; + take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx); + } + + if should_reload_pool_meta && let Err(err) = self.save_current_pool_meta().await { + if let Some(previous_pool_meta) = previous_pool_meta { + let mut pool_meta = self.pool_meta.write().await; + rollback_decommission_pool_meta(&mut pool_meta, previous_pool_meta); + } + return Err(err); + } + + if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() { + let stage = format!("clear_decommission for pool {idx}"); + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; + } + + Ok(()) + } + + async fn promote_queued_decommission(&self, idx: usize) -> Result<()> { + let promoted = { + let mut pool_meta = self.pool_meta.write().await; + pool_meta.promote_queued_decommission(idx) + }; + + if promoted { + self.save_current_pool_meta().await?; + if let Some(notification_sys) = get_global_notification_sys() { + let stage = format!("promote_queued_decommission for pool {idx}"); + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; } } Ok(()) } + pub async fn is_decommission_running(&self) -> bool { { let cancelers = self.decommission_cancelers.read().await; @@ -1484,6 +2130,11 @@ impl ECStore { false } + async fn decommission_cancel_requested(&self, idx: usize, rx: &CancellationToken) -> bool { + let pool_meta = self.pool_meta.read().await; + is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx)) + } + pub(crate) async fn spawn_decommission_routines( &self, store: Arc, @@ -1502,47 +2153,33 @@ impl ECStore { ensure_decommission_routines_scheduled(index_cancelers.len(), indices.len())?; - tokio::spawn(async move { - let mut stop_queue = false; + spawn_decommission_index_cancelers(store, rx, index_cancelers); - for (idx, canceler) in index_cancelers { - if stop_queue || rx.is_cancelled() { - canceler.cancel(); - if let Err(err) = store.decommission_cancel(idx).await { - warn!( - event = EVENT_DECOMMISSION_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_POOLS, - pool_index = idx, - state = "queued_cancel_failed", - error = %err, - "Failed to cancel queued decommission" - ); - } - continue; - } + Ok(()) + } - if let Err(err) = store.do_decommission_in_routine(canceler, idx).await { - error!( - event = EVENT_DECOMMISSION_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_POOLS, - pool_index = idx, - state = "routine_failed", - error = %err, - "Decommission routine failed" - ); - stop_queue = true; - continue; - } + pub async fn spawn_missing_local_decommission_routines(self: &Arc) -> Result<()> { + let indices = { + let pool_meta = self.pool_meta.read().await; + first_resumable_decommission_queue_indices(&pool_meta) + }; + let indices = local_decommission_queue_prefix(&self.endpoints(), &indices)?; + if indices.is_empty() { + return Ok(()); + } - stop_queue = { - let pool_meta = store.pool_meta.read().await; - !should_continue_decommission_queue(&pool_meta, idx) - }; - } - }); + let rx = CancellationToken::new(); + let index_cancelers = { + let mut cancelers = self.decommission_cancelers.write().await; + let missing = missing_decommission_worker_prefix(indices.as_slice(), cancelers.as_slice()); + bind_missing_decommission_cancelers(missing.as_slice(), &rx, cancelers.as_mut_slice()) + }; + if index_cancelers.is_empty() { + return Ok(()); + } + + spawn_decommission_index_cancelers(self.clone(), rx, index_cancelers); Ok(()) } @@ -1563,9 +2200,10 @@ impl ECStore { ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?; let store = require_decommission_store(resolve_object_store_handle(), "start decommission")?; + let local_indices = local_decommission_queue_prefix(&self.endpoints(), &indices)?; self.start_decommission(indices.clone()).await?; - if let Err(err) = self.spawn_decommission_routines(store, rx, indices.clone()).await { + if let Err(err) = self.spawn_decommission_routines(store, rx, local_indices).await { let mut rollback_err: Option = None; for idx in indices { if let Err(cancel_err) = self.decommission_cancel(idx).await { @@ -1590,14 +2228,15 @@ impl ECStore { } #[allow(unused_assignments, clippy::too_many_arguments)] - #[tracing::instrument(skip(self, set, wk, lifecycle_config, lock_retention, replication_config))] + #[tracing::instrument(skip(self, set, _worker_permit, lifecycle_config, lock_retention, replication_config))] async fn decommission_entry( self: &Arc, + rx: CancellationToken, idx: usize, entry: MetaCacheEntry, bucket: String, set: Arc, - wk: Arc, + _worker_permit: OwnedSemaphorePermit, lifecycle_config: Option, lock_retention: Option, replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, @@ -1612,7 +2251,6 @@ impl ECStore { state = "started", "Decommission entry started" ); - wk.give().await; if entry.is_dir() { debug!( event = EVENT_DECOMMISSION_ENTRY, @@ -1626,6 +2264,10 @@ impl ECStore { ); return Ok(()); } + if self.decommission_cancel_requested(idx, &rx).await { + rx.cancel(); + } + decommission_cancel_signal_result(rx.is_cancelled())?; let mut fivs = load_decommission_entry_versions(&entry, &bucket, "file_info_versions")?; @@ -1634,8 +2276,14 @@ impl ECStore { let mut decommissioned: usize = 0; let mut expired: usize = 0; + let mut cleanup_preflight_allowed_missing = Vec::new(); for version in fivs.versions.iter() { + if self.decommission_cancel_requested(idx, &rx).await { + rx.cancel(); + } + decommission_cancel_signal_result(rx.is_cancelled())?; + if should_skip_lifecycle_for_data_movement( self.clone(), &bucket, @@ -1647,8 +2295,10 @@ impl ECStore { &LcEventSrc::Decom, ) .await + .map_err(|err| with_decommission_entry_context("lifecycle_expiry", bucket.as_str(), version.name.as_str(), err))? { expired += 1; + cleanup_preflight_allowed_missing.push(data_movement::source_cleanup_version_identity(version)); continue; } @@ -1684,7 +2334,7 @@ impl ECStore { ) .await { - if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { + if is_decommission_copy_cleanup_safe_error(&err) { warn!( event = EVENT_DECOMMISSION_ENTRY, component = LOG_COMPONENT_ECSTORE, @@ -1765,8 +2415,7 @@ impl ECStore { ) .await { - if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) - { + if is_decommission_copy_cleanup_safe_error(&err) { ignore = true; cleanup_ignored = true; break; @@ -1787,11 +2436,7 @@ impl ECStore { &encode_dir_object(&version.name), None, HeaderMap::new(), - &ObjectOptions { - version_id: version_id.clone(), - no_lock: true, - ..Default::default() - }, + &decommission_object_migration_read_opts(version_id.clone()), ) .await { @@ -1822,7 +2467,7 @@ impl ECStore { let object_name = rd.object_info.name.clone(); if let Err(err) = self.clone().decommission_object(idx, bucket, rd).await { - if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { + if is_decommission_copy_cleanup_safe_error(&err) { ignore = true; cleanup_ignored = true; break; @@ -1889,6 +2534,19 @@ impl ECStore { } if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) { + decommission_cancel_signal_result(rx.is_cancelled())?; + + data_movement::ensure_source_cleanup_versions_unchanged( + set.clone(), + bucket.as_str(), + entry.name.as_str(), + &fivs, + &cleanup_preflight_allowed_missing, + "decommission", + ) + .await + .map_err(|err| with_decommission_entry_context("cleanup_preflight", bucket.as_str(), entry.name.as_str(), err))?; + let cleanup_result = set .delete_object( bucket.as_str(), @@ -1918,7 +2576,7 @@ impl ECStore { ); } - { + let should_save_progress = { let mut pool_meta = self.pool_meta.write().await; if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) { @@ -1930,27 +2588,40 @@ impl ECStore { )); } - let ok = match resolve_decommission_update_after_result( - pool_meta - .update_after(idx, self.pools.clone(), DECOMMISSION_PROGRESS_SAVE_INTERVAL) - .await, - ) { + match resolve_decommission_update_after_result(pool_meta.update_after(idx, DECOMMISSION_PROGRESS_SAVE_INTERVAL)) { Ok(ok) => ok, Err(err) => { return Err(with_decommission_entry_context("update_after", bucket.as_str(), entry.name.as_str(), err)); } - }; + } + }; - drop(pool_meta); - if ok - && let Some(notification_sys) = get_global_notification_sys() - && let Err(err) = resolve_decommission_entry_reload_result( - notification_sys.reload_pool_meta().await, - bucket.as_str(), - entry.name.as_str(), - ) - { - warn!("{err}"); + if should_save_progress { + let save_result = self.save_current_pool_meta().await; + if let Some(err) = resolve_decommission_progress_save_result(save_result) { + warn!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %entry.name, + state = "progress_save_failed", + error = %err, + "Decommission progress save failed; continuing and will retry at the next checkpoint" + ); + } else { + let mut pool_meta = self.pool_meta.write().await; + pool_meta.mark_decommission_progress_saved(); + if let Some(notification_sys) = get_global_notification_sys() + && let Err(err) = resolve_decommission_entry_reload_result( + notification_sys.reload_pool_meta().await, + bucket.as_str(), + entry.name.as_str(), + ) + { + warn!("{err}"); + } } } @@ -1975,7 +2646,11 @@ impl ECStore { pool: Arc, bi: DecomBucketInfo, ) -> Result<()> { - let wk = Workers::new(pool.disk_set.len() * 2).map_err(Error::other)?; + let worker_limit = pool.disk_set.len() * 2; + if worker_limit == 0 { + return Err(Error::other("decommission worker limit must be greater than zero")); + } + let workers = Arc::new(Semaphore::new(worker_limit)); let entry_error = Arc::new(tokio::sync::Mutex::new(None::)); let mut listing_workers = Vec::with_capacity(pool.disk_set.len()); @@ -1999,7 +2674,11 @@ impl ECStore { } for (set_idx, set) in pool.disk_set.iter().enumerate() { - wk.clone().take().await; + let listing_permit = workers + .clone() + .acquire_owned() + .await + .map_err(|err| Error::other(format!("decommission listing worker permit acquire failed: {err}")))?; debug!( event = EVENT_DECOMMISSION_BUCKET, @@ -2015,7 +2694,7 @@ impl ECStore { let decommission_entry: ListCallback = Arc::new({ let this = Arc::clone(self); let bucket = bi.name.clone(); - let wk = wk.clone(); + let workers = workers.clone(); let set = set.clone(); let lifecycle_config = lifecycle_config.clone(); let lock_retention = lock_retention.clone(); @@ -2025,7 +2704,7 @@ impl ECStore { move |entry: MetaCacheEntry| { let this = this.clone(); let bucket = bucket.clone(); - let wk = wk.clone(); + let workers = workers.clone(); let set = set.clone(); let lifecycle_config = lifecycle_config.clone(); let lock_retention = lock_retention.clone(); @@ -2034,9 +2713,32 @@ impl ECStore { let callback_rx = callback_rx.clone(); Box::pin(async move { - wk.take().await; + let worker_permit = match workers.clone().acquire_owned().await { + Ok(permit) => permit, + Err(err) => { + let err = Error::other(format!("decommission entry worker permit acquire failed: {err}")); + error!("decommission_pool: decommission_entry failed: {err}"); + let mut first_err = entry_error.lock().await; + if first_err.is_none() { + *first_err = Some(err); + callback_rx.cancel(); + } + return; + } + }; + let entry_rx = callback_rx.clone(); if let Err(err) = this - .decommission_entry(idx, entry, bucket, set, wk, lifecycle_config, lock_retention, replication_config) + .decommission_entry( + entry_rx, + idx, + entry, + bucket, + set, + worker_permit, + lifecycle_config, + lock_retention, + replication_config, + ) .await { error!("decommission_pool: decommission_entry failed: {err}"); @@ -2054,8 +2756,8 @@ impl ECStore { let rx_clone = rx.clone(); let bi = bi.clone(); let set_id = set_idx; - let wk_clone = wk.clone(); let worker = tokio::spawn(async move { + let _listing_permit = listing_permit; loop { if rx_clone.is_cancelled() { debug!( @@ -2128,8 +2830,6 @@ impl ECStore { } } } - - wk_clone.give().await; }); listing_workers.push((set_id, worker)); } @@ -2148,14 +2848,13 @@ impl ECStore { for (set_id, worker) in listing_workers { if let Err(err) = resolve_decommission_listing_worker_result(set_id, worker.await) { rx.cancel(); - wk.give().await; if listing_worker_error.is_none() { listing_worker_error = Some(err); } } } - wk.wait().await; + wait_decommission_worker_drain(&workers, worker_limit).await?; if let Some(err) = listing_worker_error { return Err(err); @@ -2208,6 +2907,32 @@ impl ECStore { } }); + if let Err(err) = self.promote_queued_decommission(idx).await { + resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?; + return Err(err); + } + if rx.is_cancelled() { + let already_canceled = { + let pool_meta = self.pool_meta.read().await; + should_skip_canceled_decommission_routine(true, pool_meta.pools.get(idx)) + }; + if already_canceled { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "canceled_preserved", + "Decommission routine skipped because pool is already canceled" + ); + return Ok(()); + } + if let Err(err) = self.decommission_cancel(idx).await { + resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?; + return Err(err); + } + return Ok(()); + } let result = self.decommission_in_background(rx.clone(), idx).await; let (final_state, canceled, cmd_line) = { @@ -2356,29 +3081,47 @@ impl ECStore { pub async fn decommission_failed(&self, idx: usize) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?; - let mut pool_meta = self.pool_meta.write().await; - if pool_meta.decommission_failed(idx) { - pool_meta.save(self.pools.clone()).await?; - pool_meta.mark_decommission_progress_saved(); + let (should_reload_pool_meta, previous_pool_meta) = { + let mut pool_meta = self.pool_meta.write().await; + let previous_pool_meta = pool_meta.clone(); + let changed = pool_meta.decommission_failed(idx); + (changed, changed.then_some(previous_pool_meta)) + }; - drop(pool_meta); - - if let Some(notification_sys) = get_global_notification_sys() { - let stage = format!("decommission_failed for pool {idx}"); - if let Err(err) = - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) - { - warn!("{err}"); - } - } + { + let mut cancelers = self.decommission_cancelers.write().await; + take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx); } - let canceler = { - let mut cancelers = self.decommission_cancelers.write().await; - cancelers.get_mut(idx).and_then(Option::take) - }; - if let Some(canceler) = canceler { - canceler.cancel(); + if should_reload_pool_meta { + if let Err(err) = self.save_current_pool_meta().await { + if let Some(previous_pool_meta) = previous_pool_meta { + let mut pool_meta = self.pool_meta.write().await; + rollback_decommission_pool_meta(&mut pool_meta, previous_pool_meta); + } + return Err(err); + } + { + let mut pool_meta = self.pool_meta.write().await; + pool_meta.mark_decommission_progress_saved(); + } + if let Some(notification_sys) = get_global_notification_sys() { + let stage = format!("decommission_failed for pool {idx}"); + if let Some(err) = observe_decommission_terminal_reload_result( + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()), + stage.as_str(), + ) { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_reload_failed", + error = %err, + "Decommission terminal state saved but pool meta reload failed" + ); + } + } } Ok(()) @@ -2388,27 +3131,47 @@ impl ECStore { pub async fn complete_decommission(&self, idx: usize) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?; - let mut pool_meta = self.pool_meta.write().await; - if pool_meta.decommission_complete(idx) { - pool_meta.save(self.pools.clone()).await?; - pool_meta.mark_decommission_progress_saved(); - drop(pool_meta); - if let Some(notification_sys) = get_global_notification_sys() { - let stage = format!("complete_decommission for pool {idx}"); - if let Err(err) = - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) - { - warn!("{err}"); - } - } + let (should_reload_pool_meta, previous_pool_meta) = { + let mut pool_meta = self.pool_meta.write().await; + let previous_pool_meta = pool_meta.clone(); + let changed = pool_meta.decommission_complete(idx); + (changed, changed.then_some(previous_pool_meta)) + }; + + { + let mut cancelers = self.decommission_cancelers.write().await; + take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx); } - let canceler = { - let mut cancelers = self.decommission_cancelers.write().await; - cancelers.get_mut(idx).and_then(Option::take) - }; - if let Some(canceler) = canceler { - canceler.cancel(); + if should_reload_pool_meta { + if let Err(err) = self.save_current_pool_meta().await { + if let Some(previous_pool_meta) = previous_pool_meta { + let mut pool_meta = self.pool_meta.write().await; + rollback_decommission_pool_meta(&mut pool_meta, previous_pool_meta); + } + return Err(err); + } + { + let mut pool_meta = self.pool_meta.write().await; + pool_meta.mark_decommission_progress_saved(); + } + if let Some(notification_sys) = get_global_notification_sys() { + let stage = format!("complete_decommission for pool {idx}"); + if let Some(err) = observe_decommission_terminal_reload_result( + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()), + stage.as_str(), + ) { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_reload_failed", + error = %err, + "Decommission terminal state saved but pool meta reload failed" + ); + } + } } Ok(()) @@ -2429,14 +3192,14 @@ impl ECStore { if is_decommissioned { warn!("decommission: already done, moving on {}", bucket.to_string()); - { + let bucket_done = { let mut pool_meta = self.pool_meta.write().await; - if mark_decommission_bucket_done(&mut pool_meta, idx, &bucket)? { - resolve_decommission_bucket_done_save_result( - pool_meta.save(self.pools.clone()).await, - idx, - bucket.name.as_str(), - )?; + mark_decommission_bucket_done(&mut pool_meta, idx, &bucket)? + }; + if bucket_done { + resolve_decommission_bucket_done_save_result(self.save_current_pool_meta().await, idx, bucket.name.as_str())?; + { + let mut pool_meta = self.pool_meta.write().await; pool_meta.mark_decommission_progress_saved(); } } @@ -2457,20 +3220,18 @@ impl ECStore { return Err(err); } - { + let bucket_done = { let mut pool_meta = self.pool_meta.write().await; - if mark_decommission_bucket_done(&mut pool_meta, idx, &bucket)? { - resolve_decommission_bucket_done_save_result( - pool_meta.save(self.pools.clone()).await, - idx, - bucket.name.as_str(), - )?; - pool_meta.mark_decommission_progress_saved(); - } - - warn!("decommission: decommission_pool bucket_done {}", &bucket.name); + mark_decommission_bucket_done(&mut pool_meta, idx, &bucket)? + }; + if bucket_done { + resolve_decommission_bucket_done_save_result(self.save_current_pool_meta().await, idx, bucket.name.as_str())?; + let mut pool_meta = self.pool_meta.write().await; + pool_meta.mark_decommission_progress_saved(); } + warn!("decommission: decommission_pool bucket_done {}", &bucket.name); + Ok(()) } @@ -2527,6 +3288,7 @@ impl ECStore { validate_start_decommission_request(&indices, self.single_pool())?; ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?; + ensure_decommission_start_local_leader(&self.endpoints(), &indices)?; for idx in indices.iter().copied() { ensure_valid_decommission_pool_index(self.pools.len(), idx)?; @@ -2535,8 +3297,9 @@ impl ECStore { { let pool_meta = self.pool_meta.read().await; for idx in indices.iter().copied() { - let (pool_present, decommission_active) = decommission_start_guard_state(pool_meta.pools.get(idx)); - ensure_decommission_start_allowed(pool_present, decommission_active)?; + let (pool_present, decommission_active, decommission_complete) = + decommission_start_guard_state(pool_meta.pools.get(idx)); + ensure_decommission_start_allowed(pool_present, decommission_active, decommission_complete)?; } } @@ -2569,26 +3332,74 @@ impl ECStore { space_infos.push((idx, pi)); } + let _start_guard = self.start_gate.lock().await; ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?; - let mut pool_meta = self.pool_meta.write().await; - for idx in indices.iter().copied() { - let (pool_present, decommission_active) = decommission_start_guard_state(pool_meta.pools.get(idx)); - ensure_decommission_start_allowed(pool_present, decommission_active)?; - } - - for (idx, pi) in space_infos { - pool_meta.decommission(idx, pi)?; - pool_meta.queue_buckets(idx, decom_buckets.clone()); - } - - pool_meta.save(self.pools.clone()).await?; + let previous_pool_meta = self + .save_current_pool_meta_for_decommission_start(&indices, space_infos, decom_buckets) + .await?; if let Some(notification_sys) = get_global_notification_sys() - && let Err(err) = - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, "start_decommission") + && let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await) { - warn!("{err}"); + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + state = "start_failed", + stage = "reload_pool_meta", + error = %err, + "Decommission start failed after pool metadata save" + ); + + { + let mut pool_meta = self.pool_meta.write().await; + rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone()); + } + if let Err(rollback_save_err) = self.save_current_pool_meta().await { + error!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + state = "rollback_failed", + stage = "save_pool_meta", + error = %rollback_save_err, + original_error = %err, + "Decommission rollback failed after pool metadata reload failure" + ); + return Err(Error::other(format!( + "{err}; decommission start rollback save failed: {rollback_save_err}" + ))); + } + + if let Err(rollback_reload_err) = resolve_decommission_pool_meta_reload_result( + notification_sys.reload_pool_meta().await, + "start_decommission_rollback", + ) { + error!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + state = "rollback_partial", + stage = "reload_pool_meta", + error = %rollback_reload_err, + original_error = %err, + "Decommission rollback metadata reload failed after local rollback save" + ); + return Err(Error::other(format!( + "{err}; decommission start rollback saved locally but peer reload failed: {rollback_reload_err}" + ))); + } + + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + state = "rollback_success", + original_error = %err, + "Decommission start rolled back after pool metadata reload failure" + ); + return Err(Error::other(format!("{err}; decommission start rollback succeeded"))); } Ok(()) @@ -2691,7 +3502,7 @@ impl ECStore { if version.deleted { continue; } - if should_skip_lifecycle_for_data_movement( + let skip_lifecycle = match should_skip_lifecycle_for_data_movement( Arc::clone(&store), &bucket_name, version, @@ -2703,6 +3514,17 @@ impl ECStore { ) .await { + Ok(skip_lifecycle) => skip_lifecycle, + Err(err) => { + let mut first_err = entry_error.lock().await; + if first_err.is_none() { + *first_err = Some(err); + callback_rx.cancel(); + } + return; + } + }; + if skip_lifecycle { continue; } remaining += 1; @@ -2747,6 +3569,7 @@ impl ECStore { #[allow(clippy::items_after_test_module)] mod tests { use super::*; + use serde::Serialize; #[test] fn ensure_pool_not_left_in_cmdline_after_decommission_allows_active_pool() { @@ -2779,7 +3602,64 @@ mod tests { } #[test] - fn should_skip_decommission_delete_marker_when_last_remaining_without_replication() { + fn lifecycle_action_removes_data_movement_version_rejects_delete_marker_action() { + assert!(!lifecycle_action_removes_data_movement_version(IlmAction::DeleteAction)); + } + + #[test] + fn lifecycle_action_removes_data_movement_version_accepts_version_delete_actions() { + assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteVersionAction)); + assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteAllVersionsAction)); + assert!(lifecycle_action_removes_data_movement_version( + IlmAction::DelMarkerDeleteAllVersionsAction + )); + } + + #[test] + fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() { + let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false) + .expect("dry-run lifecycle evaluation should not require expiry enqueue"); + + assert!(skip); + } + + #[test] + fn resolve_data_movement_lifecycle_expiry_result_rejects_apply_failure() { + let err = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, true, false) + .expect_err("failed lifecycle expiry enqueue should not be treated as skipped"); + + assert!(err.to_string().contains("failed to apply lifecycle expiry action")); + } + + #[test] + fn decommission_copy_cleanup_safe_error_accepts_missing_source_errors() { + assert!(is_decommission_copy_cleanup_safe_error(&Error::ObjectNotFound( + "bucket".to_string(), + "object".to_string() + ))); + assert!(is_decommission_copy_cleanup_safe_error(&Error::VersionNotFound( + "bucket".to_string(), + "object".to_string(), + "version".to_string() + ))); + } + + #[test] + fn decommission_delete_marker_copy_error_rejects_data_movement_overwrite() { + let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); + + assert!(!is_decommission_copy_cleanup_safe_error(&err)); + } + + #[test] + fn decommission_remote_tiered_copy_error_rejects_data_movement_overwrite() { + let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); + + assert!(!is_decommission_copy_cleanup_safe_error(&err)); + } + + #[test] + fn should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication() { let version = rustfs_filemeta::FileInfo { deleted: true, ..Default::default() @@ -2789,7 +3669,7 @@ mod tests { } #[test] - fn should_skip_decommission_delete_marker_rejects_configured_replication() { + fn should_skip_decommission_delete_marker_characterizes_replication_configured() { let version = rustfs_filemeta::FileInfo { deleted: true, ..Default::default() @@ -2844,6 +3724,18 @@ mod tests { assert_eq!(replication.replicate_decision_str, "existing"); } + #[test] + fn test_decommission_object_migration_read_opts_are_raw_data_movement() { + let opts = decommission_object_migration_read_opts(Some("vid-1".to_string())); + + assert_eq!(opts.version_id.as_deref(), Some("vid-1")); + assert!(opts.no_lock); + assert!(opts.data_movement); + assert!(opts.raw_data_movement_read); + assert!(opts.skip_rebalancing); + assert!(opts.skip_decommissioned); + } + #[test] fn decommission_remote_tiered_opts_preserves_versioning_context() { let mod_time = OffsetDateTime::now_utc(); @@ -2864,9 +3756,9 @@ mod tests { } #[test] - fn decommission_state_transitions_preserve_start_time() { + fn decommission_terminal_state_transitions_update_start_time() { let start_time = OffsetDateTime::now_utc(); - let mut pool_meta = PoolMeta { + let build_pool_meta = || PoolMeta { version: POOL_META_VERSION, pools: vec![PoolStatus { id: 0, @@ -2880,23 +3772,20 @@ mod tests { dont_save: true, }; + let mut pool_meta = build_pool_meta(); assert!(pool_meta.decommission_failed(0)); - assert_eq!( - pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), - Some(start_time) - ); + assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None); + let mut pool_meta = build_pool_meta(); assert!(pool_meta.decommission_complete(0)); assert_eq!( pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), Some(start_time) ); + let mut pool_meta = build_pool_meta(); assert!(pool_meta.decommission_cancel(0)); - assert_eq!( - pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), - Some(start_time) - ); + assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None); } #[test] @@ -2910,6 +3799,7 @@ mod tests { last_update: start_time, decommission: Some(PoolDecommissionInfo { start_time: Some(start_time), + queued: true, queued_buckets: vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()], decommissioned_buckets: vec!["bucket-done".to_string()], bucket: "bucket-b".to_string(), @@ -2952,6 +3842,7 @@ mod tests { assert_eq!(restored_decommission.items_decommission_failed, 1); assert_eq!(restored_decommission.bytes_done, 1024); assert_eq!(restored_decommission.bytes_failed, 128); + assert!(restored_decommission.queued); assert_eq!(restored_decommission.items_since_last_progress_save(), 0); } @@ -3013,6 +3904,170 @@ mod tests { assert!(decommission.object.is_empty()); } + #[test] + fn pool_meta_decode_rejects_unknown_legacy_fields() { + #[derive(Serialize)] + struct LegacyPoolMetaWithUnknownField { + version: u16, + pools: Vec, + dont_save: bool, + unexpected: bool, + } + + let payload = rmp_serde::to_vec_named(&LegacyPoolMetaWithUnknownField { + version: POOL_META_VERSION, + pools: Vec::new(), + dont_save: true, + unexpected: true, + }) + .expect("legacy pool metadata with unknown field should serialize"); + + let err = PoolMeta::decode_pool_meta_payload(payload.as_slice()) + .expect_err("unknown legacy pool metadata field should fail decode"); + let rendered = err.to_string(); + assert!(rendered.contains("PoolMeta decode failed for both persisted and legacy formats")); + assert!(rendered.contains("unknown field") || rendered.contains("missing field")); + } + + #[test] + fn pool_meta_decode_rejects_unknown_persisted_fields() { + #[derive(Serialize)] + struct PersistedPoolMetaWithUnknownField { + version: u16, + pools: Vec, + unexpected: bool, + } + + let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownField { + version: POOL_META_VERSION, + pools: Vec::new(), + unexpected: true, + }) + .expect("pool metadata with unknown field should serialize"); + + let err = PoolMeta::decode_pool_meta_payload(payload.as_slice()) + .expect_err("unknown persisted pool metadata field should fail decode"); + let rendered = err.to_string(); + assert!(rendered.contains("PoolMeta decode failed for both persisted and legacy formats")); + assert!(rendered.contains("unknown field") || rendered.contains("missing field")); + } + + #[test] + fn pool_meta_decode_rejects_missing_critical_persisted_fields() { + #[derive(Serialize)] + struct PersistedPoolMetaWithoutPools { + version: u16, + } + + let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithoutPools { + version: POOL_META_VERSION, + }) + .expect("pool metadata without pools should serialize"); + + let err = PoolMeta::decode_pool_meta_payload(payload.as_slice()) + .expect_err("missing persisted pool metadata pools should fail decode"); + assert!( + err.to_string() + .contains("PoolMeta decode failed for both persisted and legacy formats") + ); + } + + #[test] + fn pool_meta_decode_rejects_unknown_decommission_fields() { + #[derive(Serialize)] + struct PersistedPoolStatusWithUnknownDecommission { + #[serde(rename = "id")] + id: usize, + #[serde(rename = "cmdline")] + cmd_line: String, + #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")] + last_update: OffsetDateTime, + #[serde(rename = "decommissionInfo")] + decommission: Option, + } + + #[derive(Serialize)] + struct PersistedPoolDecommissionInfoWithUnknownField { + #[serde(rename = "startTime", with = "time::serde::rfc3339::option")] + start_time: Option, + #[serde(rename = "startSize")] + start_size: usize, + #[serde(rename = "totalSize")] + total_size: usize, + #[serde(rename = "currentSize")] + current_size: usize, + #[serde(rename = "complete")] + complete: bool, + #[serde(rename = "failed")] + failed: bool, + #[serde(rename = "canceled")] + canceled: bool, + #[serde(rename = "queuedBuckets")] + queued_buckets: Vec, + #[serde(rename = "decommissionedBuckets")] + decommissioned_buckets: Vec, + #[serde(rename = "bucket")] + bucket: String, + #[serde(rename = "prefix")] + prefix: String, + #[serde(rename = "object")] + object: String, + #[serde(rename = "objectsDecommissioned")] + items_decommissioned: usize, + #[serde(rename = "objectsDecommissionedFailed")] + items_decommission_failed: usize, + #[serde(rename = "bytesDecommissioned")] + bytes_done: usize, + #[serde(rename = "bytesDecommissionedFailed")] + bytes_failed: usize, + #[serde(rename = "unexpected")] + unexpected: bool, + } + + #[derive(Serialize)] + struct PersistedPoolMetaWithUnknownDecommission { + version: u16, + pools: Vec, + } + + let start_time = OffsetDateTime::now_utc(); + let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownDecommission { + version: POOL_META_VERSION, + pools: vec![PersistedPoolStatusWithUnknownDecommission { + id: 0, + cmd_line: "/data/pool".to_string(), + last_update: start_time, + decommission: Some(PersistedPoolDecommissionInfoWithUnknownField { + start_time: Some(start_time), + start_size: 0, + total_size: 0, + current_size: 0, + complete: false, + failed: false, + canceled: false, + queued_buckets: Vec::new(), + decommissioned_buckets: Vec::new(), + bucket: String::new(), + prefix: String::new(), + object: String::new(), + items_decommissioned: 0, + items_decommission_failed: 0, + bytes_done: 0, + bytes_failed: 0, + unexpected: true, + }), + }], + }) + .expect("pool metadata with unknown decommission field should serialize"); + + let err = PoolMeta::decode_pool_meta_payload(payload.as_slice()) + .expect_err("unknown persisted decommission metadata field should fail decode"); + assert!( + err.to_string() + .contains("PoolMeta decode failed for both persisted and legacy formats") + ); + } + #[test] fn pool_meta_decode_rejects_invalid_decommission_terminal_state() { let start_time = OffsetDateTime::now_utc(); @@ -3312,27 +4367,37 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi mod pools_tests { use super::{ DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo, - DecommissionTerminalState, PoolDecommissionInfo, PoolMeta, PoolStatus, bind_decommission_cancelers, - cancel_decommission_canceler, classify_decommission_terminal_state, count_decommission_item, - decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options, + DecommissionTerminalState, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, bind_decommission_cancelers, + bind_missing_decommission_cancelers, cancel_decommission_canceler, classify_decommission_terminal_state, + count_decommission_item, decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options, decommission_start_guard_state, dedup_indices, default_decommission_bucket_concurrency, - ensure_decommission_cancel_allowed, ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, - ensure_decommission_start_allowed, ensure_decommission_terminal_operation_supported, - ensure_valid_decommission_pool_index, get_by_index, has_active_decommission_canceler, is_decommission_active, - is_decommission_cancel_terminal, load_decommission_entry_versions, mark_decommission_bucket_done, + ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available, + ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_local_leader, + ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_terminal_operation_supported, + ensure_local_decommission_pool_leaders, ensure_valid_decommission_pool_index, first_resumable_decommission_queue_indices, + get_by_index, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, + load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done, + missing_decommission_worker_prefix, observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, require_decommission_store, resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result, resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result, resolve_decommission_pool_meta_reload_result, - resolve_decommission_preflight_heal_result, resolve_decommission_spawn_failure_result, - resolve_decommission_terminal_mark_after_error_result, resolve_decommission_terminal_mark_result, - resolve_decommission_update_after_result, run_decommission_buckets_bounded, should_cleanup_decommission_source_entry, - should_continue_decommission_queue, should_count_decommission_version_complete, - should_preserve_decommission_canceled_state, split_decommission_buckets, take_decommission_canceler, - track_decommission_current_object, validate_start_decommission_request, with_decommission_entry_context, + resolve_decommission_preflight_heal_result, resolve_decommission_progress_save_result, + resolve_decommission_spawn_failure_result, resolve_decommission_terminal_mark_after_error_result, + resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result, + resolve_start_decommission_pool_meta_reload_result, rollback_start_decommission_pool_meta, + run_decommission_buckets_bounded, should_cleanup_decommission_source_entry, should_continue_decommission_queue, + should_count_decommission_version_complete, should_preserve_decommission_canceled_state, + should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload, + should_skip_canceled_decommission_routine, split_decommission_buckets, take_and_cancel_decommission_canceler, + take_decommission_canceler, track_decommission_current_object, validate_start_decommission_request, + wait_decommission_worker_drain, with_decommission_entry_context, }; use crate::data_movement; + use crate::disk::endpoint::Endpoint; + use crate::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::error::Error; + use crate::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats}; use rustfs_filemeta::MetaCacheEntry; use rustfs_rio::Index; use std::sync::{ @@ -3341,8 +4406,35 @@ mod pools_tests { }; use std::time::Duration as StdDuration; use time::{Duration, OffsetDateTime}; + use tokio::sync::Semaphore; use tokio_util::sync::CancellationToken; + fn decommission_test_pool_endpoint(idx: usize, is_local: bool) -> PoolEndpoints { + let port = 9000usize + idx; + let mut endpoint = + Endpoint::try_from(format!("http://127.0.0.1:{port}/disk").as_str()).expect("test endpoint should parse"); + endpoint.is_local = is_local; + endpoint.pool_idx = i32::try_from(idx).expect("test pool index should fit i32"); + + PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 1, + endpoints: Endpoints::from(vec![endpoint]), + cmd_line: format!("pool-{idx}"), + platform: String::new(), + } + } + + fn decommission_test_pool_status(idx: usize, decommission: Option) -> PoolStatus { + PoolStatus { + id: idx, + cmd_line: format!("pool-{idx}"), + last_update: OffsetDateTime::now_utc(), + decommission, + } + } + #[test] fn test_dedup_indices_removes_duplicates_preserving_order() { assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]); @@ -3508,6 +4600,31 @@ mod pools_tests { assert_eq!(started.load(Ordering::SeqCst), 1); } + #[tokio::test] + async fn test_wait_decommission_worker_drain_waits_for_entry_permit() { + let workers = Arc::new(Semaphore::new(1)); + let permit = workers + .clone() + .acquire_owned() + .await + .expect("test worker permit should acquire"); + + let drain = tokio::spawn({ + let workers = workers.clone(); + async move { wait_decommission_worker_drain(&workers, 1).await } + }); + + tokio::task::yield_now().await; + assert!(!drain.is_finished(), "drain should wait while a worker permit is held"); + + drop(permit); + let result = tokio::time::timeout(StdDuration::from_secs(1), drain) + .await + .expect("drain should finish after permit release") + .expect("drain task should not panic"); + assert!(result.is_ok()); + } + #[test] fn test_get_by_index_returns_value_when_in_range() { let values = vec!["a", "b", "c"]; @@ -3532,6 +4649,31 @@ mod pools_tests { assert!(!meta.is_suspended(1)); } + #[test] + fn test_rollback_start_decommission_pool_meta_clears_active_state() { + let previous = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }], + ..Default::default() + }; + let mut active = previous.clone(); + active.pools[0].decommission = Some(PoolDecommissionInfo::default()); + + assert!(active.is_suspended(0)); + let (_, decommission_active, _) = decommission_start_guard_state(active.pools.first()); + assert!(decommission_active); + + rollback_start_decommission_pool_meta(&mut active, previous); + + assert!(!active.is_suspended(0)); + let (_, decommission_active, _) = decommission_start_guard_state(active.pools.first()); + assert!(!decommission_active); + } + #[test] fn test_pool_meta_queue_buckets_ignores_out_of_range_index() { let mut meta = PoolMeta { @@ -3798,6 +4940,20 @@ mod pools_tests { assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools")); } + #[test] + fn test_resolve_decommission_progress_save_result_returns_none_on_success() { + assert!(resolve_decommission_progress_save_result(Ok(())).is_none()); + } + + #[test] + fn test_resolve_decommission_progress_save_result_returns_error_for_best_effort_failure() { + let err = resolve_decommission_progress_save_result(Err(Error::SlowDown)) + .expect("progress save failure should be returned for logging"); + + assert!(err.to_string().contains("decommission progress save failed")); + assert!(err.to_string().contains(Error::SlowDown.to_string().as_str())); + } + #[test] fn test_resolve_decommission_preflight_heal_result_passthrough_ok() { assert!(resolve_decommission_preflight_heal_result::<()>("bucket-a", Ok(())).is_ok()); @@ -3915,6 +5071,20 @@ mod pools_tests { assert!(message.contains("mark error")); } + #[test] + fn test_observe_decommission_terminal_reload_result_returns_none_on_success() { + assert!(observe_decommission_terminal_reload_result(Ok(()), "complete_decommission for pool 3").is_none()); + } + + #[test] + fn test_observe_decommission_terminal_reload_result_keeps_failure_for_logging() { + let err = observe_decommission_terminal_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3") + .expect("reload failure should be observable"); + let message = err.to_string(); + assert!(message.contains("decommission terminal pool meta reload failed during decommission_failed for pool 3")); + assert!(message.contains(Error::SlowDown.to_string().as_str())); + } + #[test] fn test_resolve_decommission_spawn_failure_result_keeps_primary_without_rollback_error() { let err = resolve_decommission_spawn_failure_result(Error::SlowDown, None); @@ -4021,6 +5191,19 @@ mod pools_tests { assert!(message.contains(Error::SlowDown.to_string().as_str())); } + #[test] + fn test_resolve_start_decommission_pool_meta_reload_result_returns_failure() { + let err = resolve_start_decommission_pool_meta_reload_result(Err(Error::other( + "reload_pool_meta encountered 1 failure(s): peer[0] reload_pool_meta failed", + ))) + .expect_err("start_decommission must fail when peer pool meta reload fails"); + let message = err.to_string(); + + assert!(message.contains("decommission pool meta reload failed during start_decommission")); + assert!(message.contains("reload_pool_meta encountered 1 failure(s)")); + assert!(message.contains("peer[0]")); + } + #[test] fn test_resolve_decommission_listing_worker_result_passthrough_ok() { assert!(resolve_decommission_listing_worker_result(2, Ok(())).is_ok()); @@ -4067,22 +5250,41 @@ mod pools_tests { } #[test] - fn test_should_cleanup_decommission_source_entry_rejects_versions_only_expired_by_lifecycle() { - assert!(!should_cleanup_decommission_source_entry(2, 3, 1)); + fn test_should_cleanup_decommission_source_entry_accepts_migrated_and_safely_expired_versions() { + assert!(should_cleanup_decommission_source_entry(1, 2, 1)); } - #[tokio::test] - async fn test_pool_meta_update_after_rejects_out_of_range_index() { + #[test] + fn test_should_cleanup_decommission_source_entry_accepts_versions_only_safely_expired_by_lifecycle() { + assert!(should_cleanup_decommission_source_entry(0, 2, 2)); + } + + #[test] + fn test_should_cleanup_decommission_source_entry_rejects_object_lock_retained_version() { + assert!(!should_cleanup_decommission_source_entry(1, 2, 0)); + } + + #[test] + fn test_should_cleanup_decommission_source_entry_rejects_replication_pending_version() { + assert!(!should_cleanup_decommission_source_entry(2, 3, 0)); + } + + #[test] + fn test_should_cleanup_decommission_source_entry_rejects_counter_overrun() { + assert!(!should_cleanup_decommission_source_entry(2, 2, 1)); + } + + #[test] + fn test_pool_meta_update_after_rejects_out_of_range_index() { let mut meta = PoolMeta::default(); let err = meta - .update_after(1, Vec::new(), Duration::seconds(1)) - .await + .update_after(1, Duration::seconds(1)) .expect_err("out-of-range index should fail"); assert!(err.to_string().contains("invalid decommission pool index 1 for 0 pools")); } - #[tokio::test] - async fn test_pool_meta_update_after_rejects_when_decommission_missing() { + #[test] + fn test_pool_meta_update_after_rejects_when_decommission_missing() { let mut meta = PoolMeta { pools: vec![PoolStatus { id: 0, @@ -4094,8 +5296,7 @@ mod pools_tests { }; let err = meta - .update_after(0, Vec::new(), Duration::seconds(1)) - .await + .update_after(0, Duration::seconds(1)) .expect_err("pool without decommission should fail"); assert!( err.to_string() @@ -4103,8 +5304,8 @@ mod pools_tests { ); } - #[tokio::test] - async fn test_pool_meta_update_after_skips_before_time_and_item_thresholds() { + #[test] + fn test_pool_meta_update_after_skips_before_time_and_item_thresholds() { let mut meta = PoolMeta { pools: vec![PoolStatus { id: 0, @@ -4119,8 +5320,7 @@ mod pools_tests { }; let saved = meta - .update_after(0, Vec::new(), DECOMMISSION_PROGRESS_SAVE_INTERVAL) - .await + .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL) .expect("valid decommission state should update"); assert!(!saved); @@ -4128,8 +5328,8 @@ mod pools_tests { assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD - 1); } - #[tokio::test] - async fn test_pool_meta_update_after_saves_when_item_threshold_reached() { + #[test] + fn test_pool_meta_update_after_requests_save_when_item_threshold_reached() { let mut meta = PoolMeta { pools: vec![PoolStatus { id: 0, @@ -4144,17 +5344,16 @@ mod pools_tests { }; let saved = meta - .update_after(0, Vec::new(), DECOMMISSION_PROGRESS_SAVE_INTERVAL) - .await + .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL) .expect("item threshold should save progress"); assert!(saved); let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist"); - assert_eq!(info.items_since_last_progress_save(), 0); + assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD); } - #[tokio::test] - async fn test_pool_meta_update_after_saves_when_time_threshold_reached() { + #[test] + fn test_pool_meta_update_after_requests_save_when_time_threshold_reached() { let mut meta = PoolMeta { pools: vec![PoolStatus { id: 0, @@ -4169,13 +5368,12 @@ mod pools_tests { }; let saved = meta - .update_after(0, Vec::new(), DECOMMISSION_PROGRESS_SAVE_INTERVAL) - .await + .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL) .expect("time threshold should save progress"); assert!(saved); let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist"); - assert_eq!(info.items_since_last_progress_save(), 0); + assert_eq!(info.items_since_last_progress_save(), 1); } #[test] @@ -4189,6 +5387,101 @@ mod pools_tests { assert!(ensure_decommission_not_rebalancing(false).is_ok()); } + #[test] + fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_active_rebalance() { + let meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta)) + .expect_err("persisted active rebalance should block decommission start"); + + assert!(matches!(err, Error::RebalanceAlreadyRunning)); + } + + #[test] + fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_stopping_rebalance() { + let meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + info: RebalanceInfo { + status: RebalStatus::Started, + stopping: true, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta)) + .expect_err("persisted stopping rebalance should block decommission start"); + + assert!(matches!(err, Error::RebalanceAlreadyRunning)); + } + + #[test] + fn test_ensure_decommission_start_rebalance_meta_allowed_allows_terminal_or_missing_rebalance() { + let terminal_meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Completed, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + assert!(ensure_decommission_start_rebalance_meta_allowed(Some(&terminal_meta)).is_ok()); + assert!(ensure_decommission_start_rebalance_meta_allowed(None).is_ok()); + } + + #[test] + fn test_ensure_local_decommission_pool_leaders_allows_local_first_endpoint() { + let endpoints = EndpointServerPools::from(vec![ + decommission_test_pool_endpoint(0, false), + decommission_test_pool_endpoint(1, true), + ]); + + assert!(ensure_local_decommission_pool_leaders(&endpoints, &[1]).is_ok()); + } + + #[test] + fn test_ensure_local_decommission_pool_leaders_rejects_remote_first_endpoint() { + let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]); + + let err = ensure_local_decommission_pool_leaders(&endpoints, &[0]) + .expect_err("remote first endpoint should reject local decommission start"); + + assert!(err.to_string().contains("must run on the pool first endpoint")); + } + + #[test] + fn test_ensure_local_decommission_pool_leaders_rejects_empty_endpoints() { + let endpoints = EndpointServerPools::from(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 1, + endpoints: Endpoints::from(Vec::::new()), + cmd_line: "pool-0".to_string(), + platform: String::new(), + }]); + + let err = ensure_local_decommission_pool_leaders(&endpoints, &[0]) + .expect_err("pool without endpoints should reject local decommission start"); + + assert!(err.to_string().contains("has no configured endpoints")); + } + #[test] fn test_decommission_meta_bucket_options_are_idempotent() { let opts = decommission_meta_bucket_options(); @@ -4204,9 +5497,75 @@ mod pools_tests { assert!(!is_decommission_active(false, false, true)); } + #[test] + fn test_pool_meta_has_active_decommission_counts_active_and_queued_states() { + let active_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo::default()), + }], + ..Default::default() + }; + let queued_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + queued: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + assert!(pool_meta_has_active_decommission(&active_meta)); + assert!(pool_meta_has_active_decommission(&queued_meta)); + } + + #[test] + fn test_pool_meta_has_active_decommission_ignores_terminal_states() { + let terminal_meta = PoolMeta { + pools: vec![ + PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }, + PoolStatus { + id: 1, + cmd_line: "pool-1".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }, + PoolStatus { + id: 2, + cmd_line: "pool-2".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + canceled: true, + ..Default::default() + }), + }, + ], + ..Default::default() + }; + + assert!(!pool_meta_has_active_decommission(&terminal_meta)); + } + #[test] fn test_ensure_decommission_start_allowed_rejects_missing_pool() { - let err = ensure_decommission_start_allowed(false, false).expect_err("missing pool should be invalid"); + let err = ensure_decommission_start_allowed(false, false, false).expect_err("missing pool should be invalid"); assert!( err.to_string() .contains("failed to start decommission: target pool was not found") @@ -4215,18 +5574,24 @@ mod pools_tests { #[test] fn test_ensure_decommission_start_allowed_rejects_running_state() { - let err = ensure_decommission_start_allowed(true, true).expect_err("active decommission should be rejected"); + let err = ensure_decommission_start_allowed(true, true, false).expect_err("active decommission should be rejected"); assert!(matches!(err, Error::DecommissionAlreadyRunning)); } #[test] - fn test_ensure_decommission_start_allowed_allows_terminal_state() { - assert!(ensure_decommission_start_allowed(true, false).is_ok()); + fn test_ensure_decommission_start_allowed_rejects_completed_state() { + let err = ensure_decommission_start_allowed(true, false, true).expect_err("completed decommission should be rejected"); + assert!(err.to_string().contains("target pool decommission is already complete")); + } + + #[test] + fn test_ensure_decommission_start_allowed_allows_failed_or_canceled_state() { + assert!(ensure_decommission_start_allowed(true, false, false).is_ok()); } #[test] fn test_decommission_start_guard_state_reports_missing_pool() { - assert_eq!(decommission_start_guard_state(None), (false, false)); + assert_eq!(decommission_start_guard_state(None), (false, false, false)); } #[test] @@ -4238,7 +5603,7 @@ mod pools_tests { decommission: None, }; - assert_eq!(decommission_start_guard_state(Some(&pool)), (true, false)); + assert_eq!(decommission_start_guard_state(Some(&pool)), (true, false, false)); } #[test] @@ -4255,11 +5620,11 @@ mod pools_tests { }), }; - assert_eq!(decommission_start_guard_state(Some(&pool)), (true, true)); + assert_eq!(decommission_start_guard_state(Some(&pool)), (true, true, false)); } #[test] - fn test_decommission_start_guard_state_reports_terminal_pool_as_not_active() { + fn test_decommission_start_guard_state_reports_canceled_pool_as_restartable() { let pool = PoolStatus { id: 0, cmd_line: "pool-0".to_string(), @@ -4272,7 +5637,37 @@ mod pools_tests { }), }; - assert_eq!(decommission_start_guard_state(Some(&pool)), (true, false)); + assert_eq!(decommission_start_guard_state(Some(&pool)), (true, false, false)); + } + + #[test] + fn test_decommission_start_guard_state_reports_failed_pool_as_restartable() { + let pool = PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }; + + assert_eq!(decommission_start_guard_state(Some(&pool)), (true, false, false)); + } + + #[test] + fn test_decommission_start_guard_state_reports_completed_pool() { + let pool = PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }; + + assert_eq!(decommission_start_guard_state(Some(&pool)), (true, false, true)); } #[test] @@ -4383,6 +5778,59 @@ mod pools_tests { assert!(decommission_cancel_signal_result(false).is_ok()); } + #[test] + fn test_is_decommission_cancel_requested_accepts_signal_or_metadata() { + let pool = PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + canceled: true, + ..Default::default() + }), + }; + + assert!(is_decommission_cancel_requested(false, Some(&pool))); + assert!(is_decommission_cancel_requested(true, None)); + } + + #[test] + fn test_is_decommission_cancel_requested_rejects_active_without_signal() { + let pool = PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo::default()), + }; + + assert!(!is_decommission_cancel_requested(false, Some(&pool))); + assert!(!is_decommission_cancel_requested(false, None)); + } + + #[test] + fn test_skip_canceled_decommission_routine_only_for_terminal_canceled_state() { + let canceled = PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + canceled: true, + ..Default::default() + }), + }; + let active = PoolStatus { + id: 1, + cmd_line: "pool-1".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo::default()), + }; + + assert!(should_skip_canceled_decommission_routine(true, Some(&canceled))); + assert!(!should_skip_canceled_decommission_routine(false, Some(&canceled))); + assert!(!should_skip_canceled_decommission_routine(true, Some(&active))); + assert!(!should_skip_canceled_decommission_routine(true, None)); + } + #[test] fn test_ensure_decommission_cancel_allowed_rejects_missing_pool() { let err = ensure_decommission_cancel_allowed(false, false, false).expect_err("missing pool should be invalid"); @@ -4393,23 +5841,25 @@ mod pools_tests { } #[test] - fn test_is_decommission_cancel_terminal_true_when_completed() { - assert!(is_decommission_cancel_terminal(true, false, false)); + fn test_should_reject_decommission_cancel_as_terminal_true_when_completed() { + assert!(should_reject_decommission_cancel_as_terminal(true, false)); } #[test] - fn test_is_decommission_cancel_terminal_true_when_failed() { - assert!(is_decommission_cancel_terminal(false, true, false)); + fn test_should_reject_decommission_cancel_as_terminal_true_when_failed() { + assert!(should_reject_decommission_cancel_as_terminal(false, true)); } #[test] - fn test_is_decommission_cancel_terminal_true_when_canceled() { - assert!(is_decommission_cancel_terminal(false, false, true)); + fn test_should_reject_decommission_cancel_as_terminal_false_when_active_or_canceled() { + assert!(!should_reject_decommission_cancel_as_terminal(false, false)); } #[test] - fn test_is_decommission_cancel_terminal_false_when_active() { - assert!(!is_decommission_cancel_terminal(false, false, false)); + fn test_should_retry_decommission_cancel_reload_when_changed_or_already_canceled() { + assert!(should_retry_decommission_cancel_reload(true, false)); + assert!(should_retry_decommission_cancel_reload(false, true)); + assert!(!should_retry_decommission_cancel_reload(false, false)); } #[test] @@ -4430,6 +5880,76 @@ mod pools_tests { assert!(ensure_decommission_cancel_allowed(true, true, false).is_ok()); } + #[test] + fn test_ensure_decommission_clear_allowed_allows_failed_or_canceled() { + assert!(ensure_decommission_clear_allowed(true, true, false, true, false).is_ok()); + assert!(ensure_decommission_clear_allowed(true, true, false, false, true).is_ok()); + } + + #[test] + fn test_ensure_decommission_clear_allowed_rejects_active_or_completed() { + let active = ensure_decommission_clear_allowed(true, true, false, false, false) + .expect_err("active decommission should not be clearable"); + assert!(matches!(active, Error::DecommissionAlreadyRunning)); + + let complete = ensure_decommission_clear_allowed(true, true, true, false, false) + .expect_err("completed decommission should not be clearable"); + assert!(matches!(complete, Error::DecommissionNotStarted)); + } + + #[test] + fn test_pool_meta_clear_decommission_restores_failed_or_canceled_pool() { + for decommission in [ + PoolDecommissionInfo { + failed: true, + ..Default::default() + }, + PoolDecommissionInfo { + canceled: true, + ..Default::default() + }, + ] { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(decommission), + }], + ..Default::default() + }; + + assert!(meta.is_suspended(0)); + assert!(meta.clear_decommission(0).expect("terminal decommission should clear")); + assert!(meta.pools[0].decommission.is_none()); + assert!(!meta.is_suspended(0)); + } + } + + #[test] + fn test_pool_meta_clear_decommission_rejects_active_or_completed_pool() { + for decommission in [ + PoolDecommissionInfo::default(), + PoolDecommissionInfo { + complete: true, + ..Default::default() + }, + ] { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(decommission), + }], + ..Default::default() + }; + + assert!(meta.clear_decommission(0).is_err()); + assert!(meta.pools[0].decommission.is_some()); + } + } + #[test] fn test_contextualized_decommission_terminal_operation_supported_rejects_single_pool() { let err = ensure_decommission_terminal_operation_supported(true, "complete decommission") @@ -4465,13 +5985,8 @@ mod pools_tests { } #[test] - fn test_contextualized_decommission_start_request_rejects_multiple_target_pools() { - let err = validate_start_decommission_request(&[0, 1], false) - .expect_err("multiple target pools should be rejected for single active draining"); - assert!( - err.to_string() - .contains("failed to start decommission: decommission supports one target pool at a time") - ); + fn test_contextualized_decommission_start_request_allows_multiple_target_pools() { + assert!(validate_start_decommission_request(&[0, 1], false).is_ok()); } #[test] @@ -4479,6 +5994,180 @@ mod pools_tests { assert!(validate_start_decommission_request(&[0], false).is_ok()); } + #[test] + fn test_pool_meta_queued_decommission_is_not_suspended_until_promoted() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }], + ..Default::default() + }; + + meta.queue_decommission( + 0, + PoolSpaceInfo { + total: 100, + free: 10, + used: 90, + }, + ) + .expect("queued decommission should be stored"); + + assert!(!meta.is_suspended(0)); + assert!(meta.promote_queued_decommission(0)); + assert!(meta.is_suspended(0)); + } + + #[test] + fn test_pool_meta_promoted_queued_decommission_can_be_canceled() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: None, + }], + ..Default::default() + }; + + meta.queue_decommission( + 0, + PoolSpaceInfo { + total: 100, + free: 10, + used: 90, + }, + ) + .expect("queued decommission should be stored"); + + assert!(pool_meta_has_active_decommission(&meta)); + assert!(meta.promote_queued_decommission(0)); + assert!(meta.decommission_cancel(0)); + + let info = meta.pools[0] + .decommission + .as_ref() + .expect("canceled decommission state should be kept for clear"); + assert!(info.canceled); + assert!(!info.queued); + assert!(!info.failed); + assert!(!info.complete); + assert!(!pool_meta_has_active_decommission(&meta)); + } + + #[test] + fn test_pool_meta_decommission_retry_preserves_completed_bucket_progress() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + failed: true, + decommissioned_buckets: vec!["bucket-done".to_string()], + queued_buckets: vec!["bucket-pending".to_string()], + bucket: "bucket-pending".to_string(), + prefix: "prefix".to_string(), + object: "object.txt".to_string(), + items_decommissioned: 7, + items_decommission_failed: 3, + bytes_done: 1024, + bytes_failed: 256, + progress_save_item_baseline: 10, + ..Default::default() + }), + }], + ..Default::default() + }; + + meta.decommission( + 0, + PoolSpaceInfo { + total: 200, + free: 50, + used: 150, + }, + ) + .expect("failed decommission should be restartable"); + meta.queue_buckets( + 0, + vec![ + DecomBucketInfo { + name: "bucket-done".to_string(), + prefix: String::new(), + }, + DecomBucketInfo { + name: "bucket-pending".to_string(), + prefix: String::new(), + }, + ], + ); + + let info = meta.pools[0] + .decommission + .as_ref() + .expect("decommission info should be rebuilt"); + assert!(!info.failed); + assert!(!info.canceled); + assert!(!info.complete); + assert_eq!(info.decommissioned_buckets, vec!["bucket-done".to_string()]); + assert_eq!(info.queued_buckets, vec!["bucket-pending".to_string()]); + assert_eq!(info.items_decommissioned, 7); + assert_eq!(info.items_decommission_failed, 0); + assert_eq!(info.bytes_done, 1024); + assert_eq!(info.bytes_failed, 0); + assert_eq!(info.items_since_last_progress_save(), 0); + assert_eq!(info.start_size, 50); + assert_eq!(info.total_size, 200); + assert_eq!(info.current_size, 50); + assert_eq!(info.bucket, "bucket-pending"); + assert!(info.prefix.is_empty()); + assert!(info.object.is_empty()); + assert!(info.start_time.is_some()); + } + + #[test] + fn test_pool_meta_queued_decommission_retry_preserves_canceled_completed_buckets() { + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + canceled: true, + decommissioned_buckets: vec!["bucket-done".to_string()], + items_decommissioned: 5, + bytes_done: 512, + ..Default::default() + }), + }], + ..Default::default() + }; + + meta.queue_decommission( + 0, + PoolSpaceInfo { + total: 100, + free: 25, + used: 75, + }, + ) + .expect("canceled queued decommission should be restartable"); + + let info = meta.pools[0] + .decommission + .as_ref() + .expect("decommission info should be rebuilt"); + assert!(info.queued); + assert!(info.start_time.is_none()); + assert_eq!(info.decommissioned_buckets, vec!["bucket-done".to_string()]); + assert_eq!(info.items_decommissioned, 5); + assert_eq!(info.bytes_done, 512); + } + #[test] fn test_contextualized_decommission_listing_disks_available_rejects_empty_set() { let err = ensure_decommission_listing_disks_available(false, "bucket-a") @@ -4553,6 +6242,179 @@ mod pools_tests { assert!(replacement.is_cancelled()); } + #[test] + fn test_bind_missing_decommission_cancelers_stops_at_existing_slot() { + let parent = CancellationToken::new(); + let existing = CancellationToken::new(); + let mut cancelers = vec![None, Some(existing.clone()), None]; + + let bound = bind_missing_decommission_cancelers(&[0, 1, 2], &parent, cancelers.as_mut_slice()); + + assert_eq!(bound.len(), 1); + assert_eq!(bound[0].0, 0); + assert!(cancelers[0].is_some()); + assert!(cancelers[1].is_some()); + assert!(cancelers[2].is_none()); + assert!(!existing.is_cancelled()); + } + + #[test] + fn test_local_decommission_queue_prefix_stops_at_remote_leader() { + let endpoints = EndpointServerPools::from(vec![ + decommission_test_pool_endpoint(0, true), + decommission_test_pool_endpoint(1, true), + decommission_test_pool_endpoint(2, false), + decommission_test_pool_endpoint(3, true), + ]); + + let local = local_decommission_queue_prefix(&endpoints, &[0, 1, 2, 3]).expect("prefix should resolve"); + + assert_eq!(local, vec![0, 1]); + } + + #[test] + fn test_local_decommission_queue_prefix_empty_when_first_leader_remote() { + let endpoints = EndpointServerPools::from(vec![ + decommission_test_pool_endpoint(0, false), + decommission_test_pool_endpoint(1, true), + ]); + + let local = local_decommission_queue_prefix(&endpoints, &[0, 1]).expect("prefix should resolve"); + + assert!(local.is_empty()); + } + + #[test] + fn test_decommission_start_local_leader_allows_remote_queued_pool() { + let endpoints = EndpointServerPools::from(vec![ + decommission_test_pool_endpoint(0, true), + decommission_test_pool_endpoint(1, false), + ]); + + assert!(ensure_decommission_start_local_leader(&endpoints, &[0, 1]).is_ok()); + } + + #[test] + fn test_decommission_start_local_leader_rejects_remote_active_pool() { + let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]); + + let err = ensure_decommission_start_local_leader(&endpoints, &[0]).expect_err("remote active pool should be rejected"); + + assert!( + err.to_string() + .contains("decommission for pool 0 must run on the pool first endpoint") + ); + } + + #[test] + fn test_missing_decommission_worker_prefix_stops_at_active_worker() { + let cancelers = vec![None, Some(CancellationToken::new()), None]; + + let missing = missing_decommission_worker_prefix(&[0, 1, 2], cancelers.as_slice()); + + assert_eq!(missing, vec![0]); + } + + #[test] + fn test_first_resumable_decommission_queue_indices_stops_at_failed_or_canceled_state() { + let meta = PoolMeta { + pools: vec![ + decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + ), + decommission_test_pool_status( + 1, + Some(PoolDecommissionInfo { + canceled: true, + ..Default::default() + }), + ), + decommission_test_pool_status( + 2, + Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + ), + decommission_test_pool_status( + 3, + Some(PoolDecommissionInfo { + queued: true, + ..Default::default() + }), + ), + decommission_test_pool_status(4, None), + ], + ..Default::default() + }; + + assert!(first_resumable_decommission_queue_indices(&meta).is_empty()); + } + + #[test] + fn test_first_resumable_decommission_queue_indices_allows_after_completed_prefix() { + let meta = PoolMeta { + pools: vec![ + decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + ), + decommission_test_pool_status( + 1, + Some(PoolDecommissionInfo { + queued: true, + ..Default::default() + }), + ), + decommission_test_pool_status( + 2, + Some(PoolDecommissionInfo { + queued: true, + ..Default::default() + }), + ), + ], + ..Default::default() + }; + + assert_eq!(first_resumable_decommission_queue_indices(&meta), vec![1, 2]); + } + + #[test] + fn test_return_resumable_pools_skips_failed_decommission() { + let meta = PoolMeta { + pools: vec![ + decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + ), + decommission_test_pool_status( + 1, + Some(PoolDecommissionInfo { + queued: true, + ..Default::default() + }), + ), + ], + ..Default::default() + }; + + let resumable = meta.return_resumable_pools(); + + assert_eq!(resumable.len(), 1); + assert_eq!(resumable[0].id, 1); + } + #[test] fn test_take_decommission_canceler_takes_and_clears_slot() { let token = CancellationToken::new(); @@ -4595,6 +6457,24 @@ mod pools_tests { assert!(!cancel_decommission_canceler(None)); } + #[test] + fn test_take_and_cancel_decommission_canceler_clears_slot() { + let token = CancellationToken::new(); + let mut cancelers = vec![Some(token.clone())]; + + assert!(take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0)); + assert!(cancelers[0].is_none()); + assert!(token.is_cancelled()); + } + + #[test] + fn test_take_and_cancel_decommission_canceler_missing_slot_is_false() { + let mut cancelers = vec![None]; + + assert!(!take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0)); + assert!(cancelers[0].is_none()); + } + #[test] fn test_ensure_decommission_routines_scheduled_accepts_positive_bound_count() { assert!(super::ensure_decommission_routines_scheduled(2, 2).is_ok()); diff --git a/crates/ecstore/src/rebalance.rs b/crates/ecstore/src/rebalance.rs index 95b468c50..07295b76d 100644 --- a/crates/ecstore/src/rebalance.rs +++ b/crates/ecstore/src/rebalance.rs @@ -25,13 +25,15 @@ const EVENT_REBALANCE_LISTING: &str = "rebalance_listing"; const REBAL_META_FMT: u16 = 1; // Replace with actual format value const REBAL_META_VER: u16 = 1; // Replace with actual version value -const REBAL_META_NAME: &str = "rebalance.bin"; +pub(crate) const REBAL_META_NAME: &str = "rebalance.bin"; const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3; const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS"; +const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: "; const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250); const REBALANCE_MIGRATION_RETRY_BASE_DELAY: Duration = Duration::from_millis(250); const REBALANCE_MIGRATION_LOCK_RETRY_CAP: Duration = Duration::from_secs(10); const REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX: &str = "deferred transient rebalance entry failure:"; +const REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT: usize = 10; mod control; mod entry; @@ -41,7 +43,12 @@ mod runtime; mod types; mod worker; -pub use types::{DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, RebalanceStats}; +pub(crate) use meta::is_rebalance_conflicting_with_decommission; +pub use meta::{decode_rebalance_stop_propagation_record, encode_rebalance_stop_propagation_record}; +pub use types::{ + DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, + RebalanceStats, RebalanceStopPropagationRecord, +}; use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome}; #[cfg(test)] diff --git a/crates/ecstore/src/rebalance/control.rs b/crates/ecstore/src/rebalance/control.rs index d0e60f8cd..9539674ce 100644 --- a/crates/ecstore/src/rebalance/control.rs +++ b/crates/ecstore/src/rebalance/control.rs @@ -2,7 +2,9 @@ use super::meta::{ clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue, ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done, merge_rebalance_meta, percent_free_ratio, rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta, - resolve_next_rebalance_bucket, should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot, + record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id, + should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id, + validate_init_rebalance_state, }; use super::worker::{ rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result, resolve_rebalance_meta_load_result, @@ -10,7 +12,8 @@ use super::worker::{ }; use super::{ DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME, - RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, + RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, + encode_rebalance_stop_propagation_record, }; use crate::error::{Error, Result}; use crate::object_api::ObjectOptions; @@ -247,6 +250,35 @@ impl ECStore { Ok(id) } + #[tracing::instrument(skip(self, bucktes))] + pub async fn init_and_start_rebalance(self: &Arc, bucktes: Vec) -> Result { + let _start_guard = self.start_gate.lock().await; + + let decommission_running = self.is_decommission_running().await; + { + let rebalance_meta = self.rebalance_meta.read().await; + validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?; + } + + let id = self.init_rebalance_meta(bucktes).await?; + if let Err(start_err) = self.start_rebalance().await { + if let Err(rollback_err) = self + .rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string()) + .await + { + return Err(Error::other(format!( + "failed to start rebalance after metadata initialized for {id}: {start_err}; rollback failed: {rollback_err}" + ))); + } + + return Err(Error::other(format!( + "failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}" + ))); + } + + Ok(id) + } + #[tracing::instrument(skip(self, fi))] pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> { self.update_pool_stats_batch(pool_index, bucket, &[fi]).await @@ -389,12 +421,24 @@ impl ECStore { false } + pub async fn current_rebalance_id(&self) -> Option { + let rebalance_meta = self.rebalance_meta.read().await; + rebalance_meta + .as_ref() + .and_then(|meta| (!meta.id.is_empty()).then(|| meta.id.clone())) + } + #[tracing::instrument(skip(self))] pub async fn stop_rebalance(self: &Arc) -> Result<()> { + self.stop_rebalance_for_id(None).await + } + + #[tracing::instrument(skip(self))] + pub async fn stop_rebalance_for_id(self: &Arc, expected_id: Option<&str>) -> Result<()> { let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; - stop_rebalance_meta_snapshot(rebalance_meta.as_mut(), OffsetDateTime::now_utc()) - }; + stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id) + }?; if let Some(meta_to_save) = meta_to_save { let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?; @@ -407,4 +451,54 @@ impl ECStore { Ok(()) } + + async fn rollback_rebalance_start_without_worker_for_id( + self: &Arc, + expected_id: Option<&str>, + start_error: String, + ) -> Result<()> { + let meta_to_save = { + let mut rebalance_meta = self.rebalance_meta.write().await; + rollback_rebalance_start_meta_snapshot_for_id( + rebalance_meta.as_mut(), + OffsetDateTime::now_utc(), + expected_id, + start_error, + ) + }; + + if let Some(meta_to_save) = meta_to_save { + let pool = clone_first_arc(self.pools.as_slice(), "rollback_rebalance_start: no pools available")?; + resolve_rebalance_meta_save_result( + self.save_rebalance_meta_with_merge(pool, &meta_to_save, "rollback_rebalance_start") + .await, + "rollback_rebalance_start", + )?; + } + + Ok(()) + } + + pub async fn record_rebalance_stop_propagation(self: &Arc, record: RebalanceStopPropagationRecord) -> Result<()> { + if !record.has_failures() { + return Ok(()); + } + + let encoded_error = encode_rebalance_stop_propagation_record(&record); + let meta_to_save = { + let mut rebalance_meta = self.rebalance_meta.write().await; + record_rebalance_stop_propagation_snapshot(rebalance_meta.as_mut(), encoded_error, OffsetDateTime::now_utc()) + }; + + if let Some(meta_to_save) = meta_to_save { + let pool = clone_first_arc(self.pools.as_slice(), "record_rebalance_stop_propagation: no pools available")?; + resolve_rebalance_meta_save_result( + self.save_rebalance_meta_with_merge(pool, &meta_to_save, "record_rebalance_stop_propagation") + .await, + "record_rebalance_stop_propagation", + )?; + } + + Ok(()) + } } diff --git a/crates/ecstore/src/rebalance/entry.rs b/crates/ecstore/src/rebalance/entry.rs index c903fcd60..a954dda63 100644 --- a/crates/ecstore/src/rebalance/entry.rs +++ b/crates/ecstore/src/rebalance/entry.rs @@ -122,7 +122,7 @@ impl ECStore { true, &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Rebal, ) - .await + .await? { expired += 1; debug!( diff --git a/crates/ecstore/src/rebalance/meta.rs b/crates/ecstore/src/rebalance/meta.rs index a707c840e..c78ee68cc 100644 --- a/crates/ecstore/src/rebalance/meta.rs +++ b/crates/ecstore/src/rebalance/meta.rs @@ -1,8 +1,9 @@ use super::{ EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, Error, GetObjectReader, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, ObjectInfo, ObjectOptions, PutObjReader, REBAL_META_FMT, REBAL_META_NAME, REBAL_META_VER, - REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalSaveOpt, RebalStatus, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats, - Result, + REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT, REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_STOP_PROPAGATION_ERROR_PREFIX, + RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats, + RebalanceStopPropagationRecord, Result, }; use crate::config::com::{read_config_with_metadata, save_config_with_opts}; use crate::error::is_err_operation_canceled; @@ -197,15 +198,15 @@ pub(super) fn is_rebalance_pool_started(pool_stat: &RebalanceStats) -> bool { pool_stat.participating && pool_stat.info.status == RebalStatus::Started } -pub(super) fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool { - if meta.stopped_at.is_some() { - return false; - } - - meta.pool_stats.iter().any(is_rebalance_pool_started) +pub(super) fn is_rebalance_pool_active(pool_stat: &RebalanceStats) -> bool { + is_rebalance_pool_started(pool_stat) || pool_stat.info.stopping } -pub(super) fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool { +pub(super) fn is_rebalance_in_progress(meta: &RebalanceMeta) -> bool { + meta.pool_stats.iter().any(is_rebalance_pool_active) +} + +pub(crate) fn is_rebalance_conflicting_with_decommission(meta: &RebalanceMeta) -> bool { is_rebalance_in_progress(meta) } @@ -224,6 +225,30 @@ pub(super) fn rebalance_meta_load_unknown_format_error(fmt: u16) -> Error { pub(super) fn rebalance_meta_load_unknown_version_error(ver: u16) -> Error { Error::other(format!("rebalance metadata load failed: unknown version {ver}")) } + +pub fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String { + match serde_json::to_string(record) { + Ok(payload) => format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}"), + Err(err) => { + let payload = serde_json::json!({ + "encodeError": err.to_string(), + "stopFailures": [], + "terminalReloadFailures": [], + }); + format!("{REBALANCE_STOP_PROPAGATION_ERROR_PREFIX}{payload}") + } + } +} + +pub fn decode_rebalance_stop_propagation_record(message: &str) -> Option { + let payload = message.strip_prefix(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX)?; + serde_json::from_str(payload).ok() +} + +fn is_rebalance_stop_propagation_error(message: Option<&str>) -> bool { + message.is_some_and(|message| message.starts_with(REBALANCE_STOP_PROPAGATION_ERROR_PREFIX)) +} + pub(super) fn rebalance_goal_reached(init_free_space: u64, init_capacity: u64, bytes: u64, percent_free_goal: f64) -> bool { if init_capacity == 0 { return false; @@ -384,10 +409,17 @@ pub(super) fn record_rebalance_cleanup_warning_in_meta( }; pool_stat.cleanup_warnings.count = pool_stat.cleanup_warnings.count.saturating_add(1); - pool_stat.cleanup_warnings.last_message = Some(message); + pool_stat.cleanup_warnings.last_message = Some(message.clone()); pool_stat.cleanup_warnings.last_bucket = Some(bucket.to_string()); pool_stat.cleanup_warnings.last_object = Some(object.to_string()); pool_stat.cleanup_warnings.last_at = Some(now); + pool_stat.cleanup_warnings.entries.push(RebalanceCleanupWarningEntry { + bucket: bucket.to_string(), + object: object.to_string(), + message, + timestamp: Some(now), + }); + truncate_rebalance_cleanup_warning_entries(&mut pool_stat.cleanup_warnings.entries); meta.last_refreshed_at = Some(now); Ok(()) } @@ -572,6 +604,17 @@ pub(super) fn validate_start_rebalance_state(decommission_running: bool, meta_lo Ok(()) } +pub(super) fn validate_init_rebalance_state(decommission_running: bool, current_meta: Option<&RebalanceMeta>) -> Result<()> { + if !ensure_rebalance_not_decommissioning(decommission_running) { + return Err(Error::DecommissionAlreadyRunning); + } + if current_meta.is_some_and(is_rebalance_in_progress) { + return Err(Error::RebalanceAlreadyRunning); + } + + Ok(()) +} + pub(super) fn should_skip_start_rebalance(cancel_attached: bool, in_progress: bool) -> bool { cancel_attached && in_progress } @@ -654,6 +697,31 @@ pub(super) fn merge_rebalance_cleanup_warnings(remote: &mut RebalanceCleanupWarn remote.last_object = local.last_object.clone(); remote.last_at = local.last_at; } + + merge_rebalance_cleanup_warning_entries(&mut remote.entries, &local.entries); + let retained_entries = u64::try_from(remote.entries.len()).unwrap_or(u64::MAX); + remote.count = remote.count.max(retained_entries); +} + +pub(super) fn merge_rebalance_cleanup_warning_entries( + remote: &mut Vec, + local: &[RebalanceCleanupWarningEntry], +) { + for entry in local { + if !remote.iter().any(|existing| existing == entry) { + remote.push(entry.clone()); + } + } + + remote.sort_by_key(|entry| entry.timestamp); + truncate_rebalance_cleanup_warning_entries(remote); +} + +fn truncate_rebalance_cleanup_warning_entries(entries: &mut Vec) { + if entries.len() > REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT { + let remove_count = entries.len() - REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT; + entries.drain(0..remove_count); + } } pub(super) fn should_replace_rebalance_cleanup_warning( @@ -667,6 +735,16 @@ pub(super) fn should_replace_rebalance_cleanup_warning( } } +pub(super) fn merge_rebalance_stop_propagation_error(remote: Option, local: Option) -> Option { + if is_rebalance_stop_propagation_error(local.as_deref()) { + local + } else if is_rebalance_stop_propagation_error(remote.as_deref()) { + remote + } else { + None + } +} + pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &RebalanceStats) { remote.init_free_space = remote.init_free_space.max(local.init_free_space); remote.init_capacity = remote.init_capacity.max(local.init_capacity); @@ -702,19 +780,23 @@ pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &Re match local.info.status { RebalStatus::Failed => { remote.info.status = RebalStatus::Failed; + remote.info.stopping = false; remote.info.end_time = local.info.end_time.or(remote.info.end_time); remote.info.last_error = local.info.last_error.clone().or_else(|| remote.info.last_error.clone()); } RebalStatus::Stopped => { if remote.info.status != RebalStatus::Failed { remote.info.status = RebalStatus::Stopped; + remote.info.stopping = false; remote.info.end_time = local.info.end_time.or(remote.info.end_time); - remote.info.last_error = None; + remote.info.last_error = + merge_rebalance_stop_propagation_error(remote.info.last_error.clone(), local.info.last_error.clone()); } } RebalStatus::Completed => { if !matches!(remote.info.status, RebalStatus::Failed | RebalStatus::Stopped) { remote.info.status = RebalStatus::Completed; + remote.info.stopping = false; remote.info.end_time = local.info.end_time.or(remote.info.end_time); remote.info.last_error = None; } @@ -722,6 +804,7 @@ pub(super) fn merge_rebalance_pool_stats(remote: &mut RebalanceStats, local: &Re RebalStatus::Started => { if !is_rebalance_terminal_status(remote.info.status) { remote.info.status = RebalStatus::Started; + remote.info.stopping |= local.info.stopping; remote.info.last_error = local.info.last_error.clone(); } } @@ -736,7 +819,6 @@ pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &Rebalance } if !local.id.is_empty() && remote.id != local.id { - *remote = local.clone(); return; } @@ -761,35 +843,120 @@ pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, sto for pool_stat in meta.pool_stats.iter_mut() { if pool_stat.info.status == RebalStatus::Started { pool_stat.info.status = RebalStatus::Stopped; + pool_stat.info.stopping = false; pool_stat.info.end_time.get_or_insert(stop_time); + if !is_rebalance_stop_propagation_error(pool_stat.info.last_error.as_deref()) { + pool_stat.info.last_error = None; + } + } + } +} + +pub(super) fn mark_started_rebalance_pools_stopping(meta: &mut RebalanceMeta) { + for pool_stat in meta.pool_stats.iter_mut() { + if pool_stat.info.status == RebalStatus::Started { + pool_stat.info.stopping = true; pool_stat.info.last_error = None; } } } pub(super) fn apply_stopped_at(meta: &mut RebalanceMeta, now: OffsetDateTime) { - meta.stopped_at = Some(now); - mark_started_rebalance_pools_stopped(meta, now); + meta.stopped_at.get_or_insert(now); + mark_started_rebalance_pools_stopping(meta); +} + +pub(super) fn clear_rebalance_cancel_token(meta: Option<&mut RebalanceMeta>) -> bool { + let Some(meta) = meta else { + return false; + }; + if let Some(cancel_tx) = meta.cancel.take() { + cancel_tx.cancel(); + return true; + } + false } pub(super) fn stop_rebalance_state(meta: &mut RebalanceMeta, now: OffsetDateTime) { - if let Some(cancel_tx) = meta.cancel.take() { - cancel_tx.cancel(); - } - - let stop_time = meta.stopped_at.unwrap_or(now); + clear_rebalance_cancel_token(Some(meta)); if meta.stopped_at.is_none() && is_rebalance_in_progress(meta) { - meta.stopped_at = Some(stop_time); - } - - if meta.stopped_at.is_some() { - mark_started_rebalance_pools_stopped(meta, stop_time); + apply_stopped_at(meta, now); + } else if meta.stopped_at.is_some() { + mark_started_rebalance_pools_stopping(meta); } } +pub(super) fn stop_rebalance_meta_snapshot_for_id( + meta: Option<&mut RebalanceMeta>, + now: OffsetDateTime, + expected_id: Option<&str>, +) -> Result> { + let Some(meta) = meta else { + return Ok(None); + }; + + if let Some(expected_id) = expected_id + && !expected_id.is_empty() + && meta.id != expected_id + { + return Err(Error::other(format!( + "rebalance stop id mismatch: expected {expected_id}, found {}", + meta.id + ))); + } + + stop_rebalance_state(meta, now); + meta.last_refreshed_at = Some(now); + Ok(Some(meta.clone())) +} + +pub(super) fn rollback_rebalance_start_meta_snapshot_for_id( + meta: Option<&mut RebalanceMeta>, + now: OffsetDateTime, + expected_id: Option<&str>, + start_error: String, +) -> Option { + meta.and_then(|meta| { + if let Some(expected_id) = expected_id + && !expected_id.is_empty() + && meta.id != expected_id + { + return None; + } + + clear_rebalance_cancel_token(Some(meta)); + meta.stopped_at.get_or_insert(now); + meta.last_refreshed_at = Some(now); + for pool_stat in meta.pool_stats.iter_mut() { + if pool_stat.info.status == RebalStatus::Started { + pool_stat.info.status = RebalStatus::Failed; + pool_stat.info.stopping = false; + pool_stat.info.end_time.get_or_insert(now); + pool_stat.info.last_error = Some(start_error.clone()); + } + } + Some(meta.clone()) + }) +} + pub(super) fn stop_rebalance_meta_snapshot(meta: Option<&mut RebalanceMeta>, now: OffsetDateTime) -> Option { + let meta = meta?; + stop_rebalance_state(meta, now); + meta.last_refreshed_at = Some(now); + Some(meta.clone()) +} + +pub(super) fn record_rebalance_stop_propagation_snapshot( + meta: Option<&mut RebalanceMeta>, + encoded_error: String, + now: OffsetDateTime, +) -> Option { meta.map(|meta| { - stop_rebalance_state(meta, now); + for pool_stat in meta.pool_stats.iter_mut() { + if pool_stat.participating || pool_stat.info.stopping || pool_stat.info.status == RebalStatus::Stopped { + pool_stat.info.last_error = Some(encoded_error.clone()); + } + } meta.last_refreshed_at = Some(now); meta.clone() }) diff --git a/crates/ecstore/src/rebalance/rebalance_unit_tests.rs b/crates/ecstore/src/rebalance/rebalance_unit_tests.rs index 3dfbf44d6..0beb835c1 100644 --- a/crates/ecstore/src/rebalance/rebalance_unit_tests.rs +++ b/crates/ecstore/src/rebalance/rebalance_unit_tests.rs @@ -25,7 +25,7 @@ use super::meta::{ record_rebalance_cleanup_warning_in_meta, remove_rebalanced_buckets_from_queue, resolve_next_rebalance_bucket, resolve_rebalance_participants, should_accept_rebalance_stats_update, should_ignore_rebalance_data_usage_cache, should_pool_participate, should_preserve_rebalance_stopped_state, should_skip_start_rebalance, stop_rebalance_meta_snapshot, - stop_rebalance_state, take_bucket_from_rebalance_queue, validate_start_rebalance_state, + stop_rebalance_state, take_bucket_from_rebalance_queue, validate_init_rebalance_state, validate_start_rebalance_state, }; use super::migration::{ MigrationBackend, MigrationVersionResult, migrate_entry_version, migrate_entry_version_with_retry_wait, @@ -1199,6 +1199,7 @@ fn test_merge_rebalance_meta_preserves_updates_from_multiple_pools() { last_bucket: Some("bucket-a".to_string()), last_object: Some("local-object".to_string()), last_at: Some(warning_at), + entries: Vec::new(), }, ..Default::default() }, @@ -1242,7 +1243,7 @@ fn test_merge_rebalance_meta_does_not_overwrite_failed_with_started_stats() { pool_stats: vec![RebalanceStats { participating: true, info: RebalanceInfo { - status: RebalStatus::Started, + status: RebalStatus::Stopped, ..Default::default() }, num_versions: 8, @@ -1307,7 +1308,7 @@ fn test_merge_rebalance_meta_preserves_failed_status_over_stopped() { stopped_at: Some(stopped_at), pool_stats: vec![RebalanceStats { info: RebalanceInfo { - status: RebalStatus::Stopped, + status: RebalStatus::Started, end_time: Some(stopped_at), ..Default::default() }, @@ -1812,7 +1813,7 @@ fn test_should_accept_rebalance_stats_update_rejects_stopped_meta() { stopped_at: Some(OffsetDateTime::UNIX_EPOCH), pool_stats: vec![RebalanceStats { info: RebalanceInfo { - status: RebalStatus::Started, + status: RebalStatus::Stopped, ..Default::default() }, ..Default::default() @@ -2173,6 +2174,93 @@ fn test_validate_start_rebalance_state_allows_loaded_meta() { validate_start_rebalance_state(false, true).expect("loaded rebalance meta should allow start"); } +#[test] +fn test_validate_init_rebalance_state_rejects_running_decommission() { + let err = validate_init_rebalance_state(true, None).expect_err("running decommission should block rebalance init"); + assert!(matches!(err, Error::DecommissionAlreadyRunning)); +} + +#[test] +fn test_validate_init_rebalance_state_rejects_active_rebalance() { + let now = OffsetDateTime::now_utc(); + let meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + start_time: Some(now), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + let err = validate_init_rebalance_state(false, Some(&meta)).expect_err("active rebalance should block rebalance init"); + assert!(matches!(err, Error::RebalanceAlreadyRunning)); +} + +#[test] +fn test_validate_init_rebalance_state_allows_terminal_or_missing_rebalance() { + let completed = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Completed, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + validate_init_rebalance_state(false, None).expect("missing rebalance meta should allow init"); + validate_init_rebalance_state(false, Some(&completed)).expect("terminal rebalance meta should allow init"); +} + +#[tokio::test] +async fn test_init_and_start_rebalance_rejects_second_start_after_gate() { + let now = OffsetDateTime::now_utc(); + let active_meta = RebalanceMeta { + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + start_time: Some(now), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + let endpoint_pools: crate::endpoints::EndpointServerPools = Vec::new().into(); + let store = Arc::new(crate::store::ECStore { + id: uuid::Uuid::new_v4(), + disk_map: std::collections::HashMap::new(), + pools: Vec::new(), + peer_sys: crate::rpc::S3PeerSys::new(&endpoint_pools), + pool_meta: tokio::sync::RwLock::new(crate::pools::PoolMeta::default()), + rebalance_meta: tokio::sync::RwLock::new(Some(active_meta)), + decommission_cancelers: tokio::sync::RwLock::new(Vec::new()), + start_gate: tokio::sync::Mutex::new(()), + pool_meta_save_gate: tokio::sync::Mutex::new(()), + local_disk_map: crate::global::GLOBAL_LOCAL_DISK_MAP.clone(), + local_disk_id_map: crate::global::GLOBAL_LOCAL_DISK_ID_MAP.clone(), + local_disk_set_drives: crate::global::GLOBAL_LOCAL_DISK_SET_DRIVES.clone(), + tier_config_mgr: crate::tier::tier::TierConfigMgr::new(), + event_notifier: crate::event_notification::EventNotifier::new(), + bucket_monitor: std::sync::OnceLock::new(), + }); + + let err = store + .init_and_start_rebalance(vec!["bucket".to_string()]) + .await + .expect_err("second rebalance start should be rejected before metadata init"); + + assert!(matches!(err, Error::RebalanceAlreadyRunning)); +} + #[test] fn test_percent_free_ratio_zero_capacity_is_zero() { assert_eq!(percent_free_ratio(100, 0), 0.0); @@ -2412,7 +2500,7 @@ fn test_resolve_rebalance_participants_respects_runtime_pool_count() { RebalanceStats { participating: false, info: RebalanceInfo { - status: RebalStatus::Started, + status: RebalStatus::Stopped, start_time: Some(now), ..Default::default() }, @@ -2421,7 +2509,7 @@ fn test_resolve_rebalance_participants_respects_runtime_pool_count() { RebalanceStats { participating: true, info: RebalanceInfo { - status: RebalStatus::Started, + status: RebalStatus::Stopped, start_time: Some(now), ..Default::default() }, @@ -2542,7 +2630,7 @@ fn test_is_rebalance_conflicting_with_decommission_false_when_stopped() { participating: true, info: RebalanceInfo { start_time: Some(now), - status: RebalStatus::Started, + status: RebalStatus::Stopped, ..Default::default() }, ..Default::default() @@ -2562,7 +2650,7 @@ fn test_is_rebalance_in_progress_stopped_takes_precedence() { participating: true, info: RebalanceInfo { start_time: Some(now), - status: RebalStatus::Started, + status: RebalStatus::Stopped, ..Default::default() }, ..Default::default() @@ -2871,6 +2959,7 @@ fn test_complete_rebalance_pools_with_empty_queue_preserves_cleanup_warnings() { last_bucket: Some("bucket-a".to_string()), last_object: Some("obj.txt".to_string()), last_at: Some(warning_at), + entries: Vec::new(), }, ..Default::default() }], @@ -2916,8 +3005,9 @@ fn test_apply_stopped_at_transitions_started_pools_only() { apply_stopped_at(&mut meta, now); assert_eq!(meta.stopped_at, Some(now)); - assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); - assert_eq!(meta.pool_stats[0].info.end_time, Some(now)); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started); + assert!(meta.pool_stats[0].info.stopping); + assert_eq!(meta.pool_stats[0].info.end_time, None); assert_eq!(meta.pool_stats[0].info.last_error, None); assert_eq!(meta.pool_stats[1].info.status, RebalStatus::Failed); @@ -2948,8 +3038,9 @@ fn test_stop_rebalance_state_cancels_token_and_marks_stopped_when_in_progress() assert!(cancel_clone.is_cancelled()); assert!(meta.cancel.is_none()); assert_eq!(meta.stopped_at, Some(now)); - assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); - assert_eq!(meta.pool_stats[0].info.end_time, Some(now)); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started); + assert!(meta.pool_stats[0].info.stopping); + assert_eq!(meta.pool_stats[0].info.end_time, None); } #[test] @@ -3005,8 +3096,9 @@ fn test_stop_rebalance_state_normalizes_started_pool_when_stopped_at_already_set assert!(cancel_clone.is_cancelled()); assert!(meta.cancel.is_none()); assert_eq!(meta.stopped_at, Some(stopped_at)); - assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); - assert_eq!(meta.pool_stats[0].info.end_time, Some(stopped_at)); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started); + assert!(meta.pool_stats[0].info.stopping); + assert_eq!(meta.pool_stats[0].info.end_time, None); assert_eq!(meta.pool_stats[0].info.last_error, None); } @@ -3040,13 +3132,15 @@ fn test_stop_rebalance_meta_snapshot_stops_meta_and_returns_snapshot() { assert!(meta.cancel.is_none()); assert_eq!(meta.stopped_at, Some(now)); assert_eq!(meta.last_refreshed_at, Some(now)); - assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started); + assert!(meta.pool_stats[0].info.stopping); assert!(snapshot.cancel.is_none()); assert_eq!(snapshot.stopped_at, Some(now)); assert_eq!(snapshot.last_refreshed_at, Some(now)); - assert_eq!(snapshot.pool_stats[0].info.status, RebalStatus::Stopped); - assert_eq!(snapshot.pool_stats[0].info.end_time, Some(now)); + assert_eq!(snapshot.pool_stats[0].info.status, RebalStatus::Started); + assert!(snapshot.pool_stats[0].info.stopping); + assert_eq!(snapshot.pool_stats[0].info.end_time, None); } #[test] @@ -3108,8 +3202,9 @@ fn test_apply_rebalance_save_option_stopped_at_updates_refresh_and_statuses() { assert_eq!(meta.stopped_at, Some(now)); assert_eq!(meta.last_refreshed_at, Some(now)); - assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped); - assert_eq!(meta.pool_stats[0].info.end_time, Some(now)); + assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Started); + assert!(meta.pool_stats[0].info.stopping); + assert_eq!(meta.pool_stats[0].info.end_time, None); assert!(meta.pool_stats[0].info.last_error.is_none()); assert_eq!(meta.pool_stats[1].info.status, RebalStatus::Failed); assert_eq!(meta.pool_stats[1].info.last_error.as_deref(), Some("previous failure")); diff --git a/crates/ecstore/src/rebalance/runtime.rs b/crates/ecstore/src/rebalance/runtime.rs index 391087712..bbb5707de 100644 --- a/crates/ecstore/src/rebalance/runtime.rs +++ b/crates/ecstore/src/rebalance/runtime.rs @@ -225,6 +225,7 @@ impl ECStore { "Preserved stopped rebalance status" ); } else { + pool_stat.info.stopping = false; apply_rebalance_terminal_event( &mut pool_stat.info.status, &mut pool_stat.info.end_time, diff --git a/crates/ecstore/src/rebalance/types.rs b/crates/ecstore/src/rebalance/types.rs index e9efbe4df..4c42cbf45 100644 --- a/crates/ecstore/src/rebalance/types.rs +++ b/crates/ecstore/src/rebalance/types.rs @@ -4,6 +4,7 @@ use time::OffsetDateTime; use tokio_util::sync::CancellationToken; #[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct RebalanceStats { #[serde(rename = "ifs")] pub init_free_space: u64, // Pool free space at the start of rebalance @@ -70,6 +71,7 @@ pub enum RebalSaveOpt { } #[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct RebalanceInfo { #[serde(rename = "startTs")] pub start_time: Option, // Time at which rebalance-start was issued @@ -79,9 +81,25 @@ pub struct RebalanceInfo { pub last_error: Option, // Last rebalance error message #[serde(rename = "status")] pub status: RebalStatus, // Current state of rebalance operation + #[serde(rename = "stopping", default)] + pub stopping: bool, // True after stop is requested and before worker terminal acknowledgement } #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RebalanceCleanupWarningEntry { + #[serde(rename = "bucket", default)] + pub bucket: String, + #[serde(rename = "object", default)] + pub object: String, + #[serde(rename = "message", default)] + pub message: String, + #[serde(rename = "timestamp", default)] + pub timestamp: Option, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] pub struct RebalanceCleanupWarnings { #[serde(rename = "count", default)] pub count: u64, @@ -93,6 +111,27 @@ pub struct RebalanceCleanupWarnings { pub last_object: Option, #[serde(rename = "lastAt", default)] pub last_at: Option, + #[serde(rename = "entries", default)] + pub entries: Vec, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RebalanceStopPropagationRecord { + #[serde(rename = "stopAttemptAt", default)] + pub stop_attempt_at: Option, + #[serde(rename = "stopFailures", default)] + pub stop_failures: Vec, + #[serde(rename = "terminalReloadAttemptAt", default)] + pub terminal_reload_attempt_at: Option, + #[serde(rename = "terminalReloadFailures", default)] + pub terminal_reload_failures: Vec, +} + +impl RebalanceStopPropagationRecord { + pub fn has_failures(&self) -> bool { + !self.stop_failures.is_empty() || !self.terminal_reload_failures.is_empty() + } } #[allow(dead_code)] @@ -103,6 +142,7 @@ pub struct DiskStat { } #[derive(Debug, Default, Serialize, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct RebalanceMeta { #[serde(skip)] pub cancel: Option, // To be invoked on rebalance-stop diff --git a/crates/ecstore/src/rpc/peer_rest_client.rs b/crates/ecstore/src/rpc/peer_rest_client.rs index 78b13eece..c7cbb30d9 100644 --- a/crates/ecstore/src/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/rpc/peer_rest_client.rs @@ -52,7 +52,7 @@ use tokio::{net::TcpStream, time::Duration}; use tonic::Request; use tonic::service::interceptor::InterceptedService; use tonic::transport::Channel; -use tracing::{Instrument, warn}; +use tracing::{Instrument, debug, warn}; pub const PEER_RESTSIGNAL: &str = "signal"; pub const PEER_RESTSUB_SYS: &str = "sub-sys"; @@ -916,11 +916,13 @@ impl PeerRestClient { .await } - pub async fn stop_rebalance(&self) -> Result<()> { + pub async fn stop_rebalance(&self, expected_rebalance_id: Option<&str>) -> Result<()> { self.finalize_result( async { let mut client = self.get_client().await?; - let request = Request::new(StopRebalanceRequest {}); + let request = Request::new(StopRebalanceRequest { + expected_rebalance_id: expected_rebalance_id.unwrap_or_default().to_string(), + }); let response = client.stop_rebalance(request).await?.into_inner(); if !response.success { @@ -945,7 +947,17 @@ impl PeerRestClient { let response = client.load_rebalance_meta(request).await?.into_inner(); - warn!("load_rebalance_meta response {:?}, grid_host: {:?}", response, &self.grid_host); + debug!( + event = "peer_rebalance_meta", + component = "ecstore", + subsystem = "peer_rest_client", + action = "load_rebalance_meta", + result = "response_received", + peer = %self.grid_host, + success = response.success, + start_rebalance = start_rebalance, + "peer rebalance metadata response" + ); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); diff --git a/crates/ecstore/src/rpc/peer_s3_client.rs b/crates/ecstore/src/rpc/peer_s3_client.rs index f50347907..8ad31466b 100644 --- a/crates/ecstore/src/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/rpc/peer_s3_client.rs @@ -346,7 +346,8 @@ impl S3PeerSys { #[derive(Debug)] pub struct LocalPeerS3Client { - // pub local_disks: Vec, + #[cfg(test)] + local_disks: Option>, // pub node: Node, pub pools: Option>, } @@ -354,13 +355,29 @@ pub struct LocalPeerS3Client { impl LocalPeerS3Client { pub fn new(_node: Option, pools: Option>) -> Self { Self { - // local_disks, + #[cfg(test)] + local_disks: None, // node, pools, } } + #[cfg(test)] + fn new_with_local_disks(_node: Option, pools: Option>, local_disks: Vec) -> Self { + Self { + local_disks: Some(local_disks), + pools, + } + } + async fn local_disks_for_pools(&self) -> Vec { + #[cfg(test)] + let local_disks = if let Some(local_disks) = self.local_disks.as_ref() { + local_disks.clone() + } else { + all_local_disk().await + }; + #[cfg(not(test))] let local_disks = all_local_disk().await; let Some(pools) = self.pools.as_ref() else { return local_disks; @@ -1323,7 +1340,7 @@ mod tests { assert_eq!(walk_err, DiskError::Timeout); - let info = LocalPeerS3Client::new(None, Some(vec![0])) + let info = LocalPeerS3Client::new_with_local_disks(None, Some(vec![0]), disks.clone()) .get_bucket_info(bucket, &BucketOptions::default()) .await .expect("bucket info should still succeed after prior walk timeout"); @@ -1347,7 +1364,7 @@ mod tests { .await .expect("bucket should be created on one disk"); - let err = LocalPeerS3Client::new(None, Some(vec![0])) + let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![0]), disks.clone()) .get_bucket_info("partial-bucket", &BucketOptions::default()) .await .expect_err("partial bucket should not satisfy local write quorum"); @@ -1375,19 +1392,19 @@ mod tests { .await .expect("bucket should be created on pool 0 disk 1"); - let pool0_info = LocalPeerS3Client::new(None, Some(vec![0])) + let pool0_info = LocalPeerS3Client::new_with_local_disks(None, Some(vec![0]), disks.clone()) .get_bucket_info(bucket, &BucketOptions::default()) .await .expect("pool 0 peer should see bucket on pool 0 disks"); assert_eq!(pool0_info.name, bucket); - let pool1_err = LocalPeerS3Client::new(None, Some(vec![1])) + let pool1_err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone()) .get_bucket_info(bucket, &BucketOptions::default()) .await .expect_err("pool 1 peer should not count pool 0 disks"); assert_eq!(pool1_err, Error::VolumeNotFound); - let pool1_buckets = LocalPeerS3Client::new(None, Some(vec![1])) + let pool1_buckets = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone()) .list_bucket(&BucketOptions::default()) .await .expect("pool 1 local listing should succeed against its own disks"); diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 94f507a45..cd02df998 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -1234,7 +1234,10 @@ impl rustfs_storage_api::ObjectIO for SetDisks { //TODO: userDefined - let etag = data.stream.try_resolve_etag().unwrap_or_default(); + let mut etag = data.stream.try_resolve_etag().unwrap_or_default(); + if let Some(ref tag) = opts.preserve_etag { + etag = tag.clone(); + } user_defined.insert("etag".to_owned(), etag.clone()); @@ -2275,7 +2278,7 @@ impl rustfs_storage_api::ObjectOperations for SetDisks { if let Some(err) = &gerr && goi.name.is_empty() { - if opts.delete_marker { + if should_force_delete_marker_for_missing_version(&opts) { version_found = false; } else { return Err(err.clone()); @@ -2336,7 +2339,7 @@ impl rustfs_storage_api::ObjectOperations for SetDisks { fi.set_skip_tier_free_version(); } - fi.version_id = if let Some(vid) = opts.version_id { + fi.version_id = if let Some(vid) = opts.version_id.as_ref() { Some(Uuid::parse_str(vid.as_str())?) } else if opts.versioned { Some(Uuid::new_v4()) @@ -2344,7 +2347,7 @@ impl rustfs_storage_api::ObjectOperations for SetDisks { None }; - self.delete_object_version(bucket, object, &fi, opts.delete_marker) + self.delete_object_version(bucket, object, &fi, should_force_delete_marker_for_missing_version(&opts)) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -2905,8 +2908,12 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool { }) || opts.version_purge_status() == VersionPurgeStatusType::Complete } +fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool { + opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement) +} + fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) { - let mut mark_delete = goi.version_id.is_some(); + let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none()); let mut delete_marker = opts.versioned; if opts.version_id.is_some() { @@ -3956,15 +3963,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks { object_size += ext_part.size; object_actual_size += ext_part.actual_size; - fi.parts.push(ObjectPartInfo { - etag: ext_part.etag.clone(), - number: p.part_num, - size: ext_part.size, - mod_time: ext_part.mod_time, - actual_size: ext_part.actual_size, - index: ext_part.index.clone(), - ..Default::default() - }); + fi.parts.push(completed_multipart_object_part(p.part_num, ext_part)); } if let Some(wtcs) = opts.want_checksum.as_ref() { @@ -4913,6 +4912,19 @@ fn get_complete_multipart_md5(parts: &[CompletePart]) -> String { format!("{}-{}", etag_hex, parts.len()) } +fn completed_multipart_object_part(part_num: usize, ext_part: &ObjectPartInfo) -> ObjectPartInfo { + ObjectPartInfo { + etag: ext_part.etag.clone(), + number: part_num, + size: ext_part.size, + mod_time: ext_part.mod_time, + actual_size: ext_part.actual_size, + index: ext_part.index.clone(), + checksums: ext_part.checksums.clone(), + ..Default::default() + } +} + fn complete_part_checksum(part: &CompletePart, checksum_type: rustfs_rio::ChecksumType) -> Option> { match checksum_type.base() { rustfs_rio::ChecksumType::SHA256 => Some(part.checksum_sha256.clone()), @@ -5300,6 +5312,42 @@ mod tests { assert!(delete_marker); } + #[test] + fn resolve_delete_version_state_creates_marker_for_missing_latest_versioned_delete() { + let opts = ObjectOptions { + versioned: true, + ..Default::default() + }; + + let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false); + + assert!(mark_delete); + assert!(delete_marker); + } + + #[test] + fn should_force_delete_marker_for_missing_version_rejects_data_movement_latest_delete() { + let opts = ObjectOptions { + versioned: true, + data_movement: true, + ..Default::default() + }; + + assert!(!should_force_delete_marker_for_missing_version(&opts)); + } + + #[test] + fn should_force_delete_marker_for_missing_version_allows_explicit_marker_creation() { + let opts = ObjectOptions { + versioned: true, + data_movement: true, + delete_marker: true, + ..Default::default() + }; + + assert!(should_force_delete_marker_for_missing_version(&opts)); + } + #[test] fn resolve_delete_version_state_skips_marker_creation_for_replica_purge_when_version_missing() { let opts = ObjectOptions { @@ -5433,6 +5481,33 @@ mod tests { assert!(single_result.ends_with("-1")); } + #[test] + fn test_completed_multipart_object_part_preserves_checksums() { + let checksums = HashMap::from([ + (rustfs_rio::ChecksumType::CRC32.to_string(), "crc32-value".to_string()), + (rustfs_rio::ChecksumType::CRC32C.to_string(), "crc32c-value".to_string()), + ]); + let ext_part = ObjectPartInfo { + number: 7, + etag: "etag-7".to_string(), + size: 123, + actual_size: 456, + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + index: Some(Bytes::from_static(&[1, 2, 3])), + checksums: Some(checksums.clone()), + ..Default::default() + }; + + let completed = completed_multipart_object_part(7, &ext_part); + + assert_eq!(completed.number, 7); + assert_eq!(completed.etag, ext_part.etag); + assert_eq!(completed.size, ext_part.size); + assert_eq!(completed.actual_size, ext_part.actual_size); + assert_eq!(completed.index, ext_part.index); + assert_eq!(completed.checksums, Some(checksums)); + } + #[test] fn test_get_upload_id_dir() { // Test upload ID directory path generation @@ -6305,6 +6380,71 @@ mod tests { drop(temp_dirs); } + #[tokio::test] + async fn load_file_info_versions_exact_returns_none_for_explicit_not_found() { + let format = FormatV3::new(1, 1); + let (temp_dir, endpoint, disk) = make_formatted_local_disk_for_info_test(0, &format).await; + let bucket = "bucket"; + + disk.make_volume(bucket).await.expect("bucket should be created"); + + let set_disks = SetDisks::new( + "test-owner".to_string(), + Arc::new(RwLock::new(vec![Some(disk)])), + 1, + 0, + 0, + 0, + vec![endpoint], + format, + Vec::new(), + ) + .await; + + let versions = set_disks + .load_file_info_versions_exact(bucket, "missing-object") + .await + .expect("explicit object not found should be accepted"); + + assert!(versions.is_none()); + drop(temp_dir); + } + + #[tokio::test] + async fn load_file_info_versions_exact_rejects_corrupt_metadata() { + let format = FormatV3::new(1, 1); + let (temp_dir, endpoint, disk) = make_formatted_local_disk_for_info_test(0, &format).await; + let bucket = "bucket"; + let object = "object.txt"; + + disk.make_volume(bucket).await.expect("bucket should be created"); + let metadata_path = format!("{object}/{STORAGE_FORMAT_FILE}"); + disk.write_all(bucket, &metadata_path, bytes::Bytes::from_static(b"not-xl-meta")) + .await + .expect("corrupt metadata file should be written"); + + let set_disks = SetDisks::new( + "test-owner".to_string(), + Arc::new(RwLock::new(vec![Some(disk)])), + 1, + 0, + 0, + 0, + vec![endpoint], + format, + Vec::new(), + ) + .await; + + let err = set_disks + .load_file_info_versions_exact(bucket, object) + .await + .expect_err("corrupt exact metadata must fail closed"); + + assert!(!is_err_object_not_found(&err), "corrupt metadata must not be treated as not found: {err}"); + drop(temp_dir); + } + #[tokio::test] async fn list_path_still_uses_disk_after_prior_walk_timeout() { use std::pin::Pin; diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index f0236c7ff..bb0585520 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -342,6 +342,54 @@ impl SetDisks { Self::pick_latest_quorum_files_info(fileinfos, errs, bucket, object, read_data, incl_free_vers).await } + pub(crate) async fn load_file_info_versions_exact( + &self, + bucket: &str, + object: &str, + ) -> Result> { + let disks = self.get_disks_internal().await; + if disks.is_empty() { + return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object])); + } + + let read_quorum = disks.len().div_ceil(2).max(1); + let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, object, false).await; + + if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { + let object_err = to_object_err(err.into(), vec![bucket, object]); + if is_err_object_not_found(&object_err) || is_err_version_not_found(&object_err) { + return Ok(None); + } + return Err(object_err); + } + + let mut shallow_versions = Vec::with_capacity(raw_fileinfos.len()); + for raw_fileinfo in raw_fileinfos.into_iter().flatten() { + let meta = FileMeta::load(&raw_fileinfo.buf) + .map_err(|err| Error::other(format!("exact object metadata decode failed for {bucket}/{object}: {err}")))?; + shallow_versions.push(meta.versions); + } + + if shallow_versions.len() < read_quorum { + return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object])); + } + + let versions = merge_file_meta_versions(read_quorum, true, 0, &shallow_versions); + if versions.is_empty() { + return Err(Error::other(format!( + "exact object metadata read returned no quorum versions for {bucket}/{object}" + ))); + } + + FileMeta { + versions, + ..Default::default() + } + .get_all_file_info_versions(bucket, object, true) + .map(Some) + .map_err(|err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}"))) + } + pub(super) async fn read_all_raw_file_info( disks: &[Option], bucket: &str, diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index a1c159201..c52edebf9 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -93,7 +93,7 @@ use std::{ }; use time::OffsetDateTime; use tokio::select; -use tokio::sync::RwLock; +use tokio::sync::{Mutex, RwLock}; use tokio::time::sleep; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, warn}; @@ -188,6 +188,18 @@ pub struct ECStore { pub pool_meta: RwLock, pub rebalance_meta: RwLock>, pub decommission_cancelers: RwLock>>, + /// Serializes rebalance/decommission start transitions. + /// + /// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`, + /// or `decommission_cancelers`. The guarded sections may perform bounded + /// async metadata work so check/init/start cannot race across operations. + pub(crate) start_gate: Mutex<()>, + /// Serializes full-document pool metadata saves. + /// + /// Lock order: acquire `pool_meta_save_gate` without holding `pool_meta`. + /// The saver then clones the latest `pool_meta` under a short read lock and + /// releases it before awaiting disk writes. + pub(crate) pool_meta_save_gate: Mutex<()>, // Phase 2 migration pending - do not use directly. /// Local disk maps (migrated from GLOBAL_LOCAL_DISK_MAP/ID_MAP/SET_DRIVES) diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 859b6c6d8..b4f02eb92 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -18,6 +18,7 @@ use crate::global::{ GLOBAL_EventNotifier, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr, get_global_bucket_monitor, is_dist_erasure, is_first_cluster_node_local, }; +use crate::pools::local_decommission_queue_prefix; use tracing::{debug, error, info, warn}; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; @@ -54,6 +55,22 @@ fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool { matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES } +fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_meta_loaded: bool) -> bool { + rebalance_meta_loaded && !decommission_running +} + +fn pool_meta_has_active_decommission(meta: &PoolMeta) -> bool { + meta.pools.iter().any(|pool| { + pool.decommission + .as_ref() + .is_some_and(|info| !info.complete && !info.failed && !info.canceled) + }) +} + +fn should_auto_start_rebalance_after_recovered_meta(pool_meta: &PoolMeta, rebalance_meta_loaded: bool) -> bool { + should_auto_start_rebalance_after_init(pool_meta_has_active_decommission(pool_meta), rebalance_meta_loaded) +} + async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool { tokio::select! { _ = rx.cancelled() => false, @@ -305,6 +322,8 @@ impl ECStore { pool_meta: RwLock::new(pool_meta), rebalance_meta: RwLock::new(None), decommission_cancelers, + start_gate: tokio::sync::Mutex::new(()), + pool_meta_save_gate: tokio::sync::Mutex::new(()), local_disk_map: GLOBAL_LOCAL_DISK_MAP.clone(), local_disk_id_map: GLOBAL_LOCAL_DISK_ID_MAP.clone(), @@ -354,11 +373,6 @@ impl ECStore { pub async fn init(self: &Arc, rx: CancellationToken) -> Result<()> { GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await; - resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?; - if self.rebalance_meta.read().await.is_some() { - resolve_store_init_stage_result(self.start_rebalance().await, "start_rebalance")?; - } - let mut meta = PoolMeta::default(); resolve_store_init_stage_result( meta.load( @@ -375,11 +389,8 @@ impl ECStore { let endpoints = get_global_endpoints(); let should_persist_pool_meta = is_first_cluster_node_local().await; - if !update { - { - let mut pool_meta = self.pool_meta.write().await; - *pool_meta = meta.clone(); - } + let installed_pool_meta = if !update { + meta.clone() } else { let new_meta = PoolMeta::new(&self.pools, &meta); // Only one local node should persist validated pool metadata here; otherwise @@ -387,13 +398,32 @@ impl ECStore { if should_persist_pool_meta { resolve_store_init_stage_result(new_meta.save(self.pools.clone()).await, "save_validated_pool_meta")?; } - { - let mut pool_meta = self.pool_meta.write().await; - *pool_meta = new_meta; - } + new_meta + }; + + { + let mut pool_meta = self.pool_meta.write().await; + *pool_meta = installed_pool_meta.clone(); } - let pools = meta.return_resumable_pools(); + resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?; + let rebalance_meta_loaded = self.rebalance_meta.read().await.is_some(); + let decommission_running = + pool_meta_has_active_decommission(&installed_pool_meta) || self.is_decommission_running().await; + if should_auto_start_rebalance_after_init(decommission_running, rebalance_meta_loaded) { + resolve_store_init_stage_result(self.start_rebalance().await, "start_rebalance")?; + } else if decommission_running && rebalance_meta_loaded { + warn!( + event = EVENT_ECSTORE_INIT_STATUS, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_STORE_INIT, + stage = "start_rebalance", + reason = "active_decommission", + "Deferred rebalance auto-start during store init because decommission is active" + ); + } + + let pools = installed_pool_meta.return_resumable_pools(); let mut pool_indices = Vec::with_capacity(pools.len()); for p in pools.iter() { @@ -407,18 +437,16 @@ impl ECStore { } } - if !pool_indices.is_empty() { - let idx = pool_indices[0]; - if should_resume_local_decommission(&endpoints, idx)? { - let store = self.clone(); + let local_pool_indices = local_decommission_queue_prefix(&endpoints, &pool_indices)?; + if !local_pool_indices.is_empty() { + let store = self.clone(); - tokio::spawn(async move { - if !wait_for_local_decommission_resume_delay(&rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await { - return; - } - resume_local_decommission_after_init(store, rx, pool_indices).await; - }); - } + tokio::spawn(async move { + if !wait_for_local_decommission_resume_delay(&rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await { + return; + } + resume_local_decommission_after_init(store, rx, local_pool_indices).await; + }); } let num_nodes = get_global_endpoints().get_nodes().len() as u64; @@ -448,16 +476,33 @@ impl ECStore { mod tests { use super::{ LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, pool_first_endpoint_is_local, resolve_store_init_stage_result, + should_auto_start_rebalance_after_init, should_auto_start_rebalance_after_recovered_meta, should_resume_local_decommission, should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay, }; use crate::{ disk::endpoint::Endpoint, endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, error::StorageError, + pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus}, + rebalance::RebalanceMeta, }; use std::time::Duration; + use time::OffsetDateTime; use tokio_util::sync::CancellationToken; + fn init_test_pool_meta(decommission: Option) -> PoolMeta { + PoolMeta { + version: POOL_META_VERSION, + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission, + }], + dont_save: false, + } + } + #[test] fn test_should_resume_local_decommission_respects_local_flag() { let mut local_endpoint = Endpoint::try_from("http://127.0.0.1:9000/data").expect("endpoint should parse"); @@ -519,6 +564,43 @@ mod tests { assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0)); } + #[test] + fn test_should_auto_start_rebalance_after_init_allows_loaded_rebalance_without_decommission() { + assert!(should_auto_start_rebalance_after_init(false, true)); + } + + #[test] + fn test_should_auto_start_rebalance_after_init_rejects_active_decommission() { + assert!(!should_auto_start_rebalance_after_init(true, true)); + } + + #[test] + fn test_should_auto_start_rebalance_after_init_rejects_missing_rebalance_meta() { + assert!(!should_auto_start_rebalance_after_init(false, false)); + } + + #[test] + fn test_store_init_recovery_skips_rebalance_when_decommission_metadata_is_active() { + let pool_meta = init_test_pool_meta(Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::UNIX_EPOCH), + complete: false, + failed: false, + canceled: false, + ..Default::default() + })); + let rebalance_meta = Some(RebalanceMeta::default()); + + assert!(!should_auto_start_rebalance_after_recovered_meta(&pool_meta, rebalance_meta.is_some())); + } + + #[test] + fn test_store_init_recovery_allows_rebalance_when_only_rebalance_metadata_exists() { + let pool_meta = init_test_pool_meta(None); + let rebalance_meta = Some(RebalanceMeta::default()); + + assert!(should_auto_start_rebalance_after_recovered_meta(&pool_meta, rebalance_meta.is_some())); + } + #[test] fn test_resolve_store_init_stage_result_passthrough_ok() { resolve_store_init_stage_result(Ok(()), "load_rebalance_meta").expect("successful stage should pass through"); diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 726833a0e..5efa719f0 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -216,6 +216,10 @@ fn resolve_latest_object_access( Ok((info, idx)) } +fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool { + opts.versioned && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement +} + fn version_aware_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions { let mut lookup_opts = opts.clone(); lookup_opts.no_lock = no_lock; @@ -234,6 +238,12 @@ fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> Object lookup_opts } +fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions { + let mut lookup_opts = opts.clone(); + lookup_opts.skip_decommissioned = true; + lookup_opts +} + fn effective_object_actual_size(info: &ObjectInfo) -> Option { info.get_actual_size().ok() } @@ -243,27 +253,44 @@ fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &Objec && is_data_movement_delete_marker(target) && source.version_id == target.version_id && source.mod_time == target.mod_time + && source.user_defined == target.user_defined + && source.user_tags == target.user_tags + && source.replication_status_internal == target.replication_status_internal + && source.replication_status == target.replication_status + && source.version_purge_status_internal == target.version_purge_status_internal + && source.version_purge_status == target.version_purge_status } fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool { info.delete_marker } +fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo { + ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some()) +} + fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool { + let expected = expected_data_movement_tiered_object(source); + source.version_id == target.version_id && !target.delete_marker && source.size == target.size && source.get_etag() == target.etag && source.checksum == target.checksum && source.mod_time == target.mod_time - && source.transition_status == target.transitioned_object.status - && source.transitioned_objname == target.transitioned_object.name - && source.transition_tier == target.transitioned_object.tier - && source - .transition_version_id - .map(|version_id| version_id.to_string()) - .unwrap_or_default() - == target.transitioned_object.version_id + && expected.user_defined == target.user_defined + && expected.user_tags == target.user_tags + && expected.expires == target.expires + && expected.storage_class == target.storage_class + && expected.replication_status_internal == target.replication_status_internal + && expected.replication_status == target.replication_status + && expected.version_purge_status_internal == target.version_purge_status_internal + && expected.version_purge_status == target.version_purge_status + && expected.transitioned_object.status == target.transitioned_object.status + && expected.transitioned_object.name == target.transitioned_object.name + && expected.transitioned_object.tier == target.transitioned_object.tier + && expected.transitioned_object.version_id == target.transitioned_object.version_id + && expected.transitioned_object.free_version == target.transitioned_object.free_version && effective_object_actual_size(target) == Some(source.size) } @@ -836,7 +863,7 @@ impl ECStore { let target_pool_idx = resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx); - if opts.src_pool_idx == selected_target_pool_idx { + if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) { if let Ok((source_pool_info, _)) = existing_pool_info && opts.delete_marker && is_data_movement_delete_marker(&source_pool_info.object_info) @@ -862,24 +889,23 @@ impl ECStore { )); } - let mut obj = self.pools[selected_target_pool_idx] - .delete_object(bucket, object, opts) - .await?; + let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?; obj.name = decode_dir_object(obj.name.as_str()); return Ok(obj); } // Determine which pool contains it - let (mut pinfo, errs) = self - .get_pool_info_existing_with_opts(bucket, object, &gopts) - .await - .map_err(|e| { - if is_err_read_quorum(&e) { - StorageError::ErasureWriteQuorum - } else { - e - } - })?; + let (mut pinfo, errs) = match self.get_pool_info_existing_with_opts(bucket, object, &gopts).await { + Ok(res) => res, + Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum), + Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => { + let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?; + let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?; + obj.name = decode_dir_object(object); + return Ok(obj); + } + Err(err) => return Err(err), + }; if pinfo.object_info.delete_marker && opts.version_id.is_none() { pinfo.object_info.name = decode_dir_object(object); @@ -1133,11 +1159,12 @@ impl ECStore { return self.pools[0].transition_object(bucket, &object, opts).await; } - //opts.skip_decommissioned = true; - //opts.no_lock = true; - let (_, idx) = self.get_latest_accessible_object_info_with_idx(bucket, &object, opts).await?; + let opts = transition_restore_pool_opts(opts); + let (_, idx) = self + .get_latest_accessible_object_info_with_idx(bucket, &object, &opts) + .await?; - self.pools[idx].transition_object(bucket, &object, opts).await + self.pools[idx].transition_object(bucket, &object, &opts).await } #[instrument(skip(self))] @@ -1152,15 +1179,14 @@ impl ECStore { return self.pools[0].clone().restore_transitioned_object(bucket, &object, opts).await; } - //opts.skip_decommissioned = true; - //opts.nolock = true; + let opts = transition_restore_pool_opts(opts); let (_, idx) = self - .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts) + .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts) .await?; self.pools[idx] .clone() - .restore_transitioned_object(bucket, &object, opts) + .restore_transitioned_object(bucket, &object, &opts) .await } @@ -1269,8 +1295,8 @@ mod tests { use super::*; use crate::bucket::lifecycle::core::TRANSITION_COMPLETE; use bytes::Bytes; - use rustfs_storage_api::TransitionedObject; use std::io::Cursor; + use std::sync::Arc; use tokio::io::AsyncReadExt; #[test] @@ -1313,6 +1339,33 @@ mod tests { assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched)); } + #[test] + fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() { + let version_id = Uuid::nil(); + let mod_time = OffsetDateTime::UNIX_EPOCH; + let source = ObjectInfo { + version_id: Some(version_id), + delete_marker: true, + mod_time: Some(mod_time), + user_defined: Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "true".to_string())])), + replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()), + version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()), + ..Default::default() + }; + + let mut target = source.clone(); + target.user_defined = Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "false".to_string())])); + assert!(!is_equivalent_data_movement_delete_marker(&source, &target)); + + let mut target = source.clone(); + target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string()); + assert!(!is_equivalent_data_movement_delete_marker(&source, &target)); + + let mut target = source.clone(); + target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string()); + assert!(!is_equivalent_data_movement_delete_marker(&source, &target)); + } + #[test] fn equivalent_data_movement_delete_marker_rejects_live_object() { let source = ObjectInfo { @@ -1367,6 +1420,7 @@ mod tests { fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() { let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1); assert_eq!(target_pool_idx, 3); + assert!(should_check_data_movement_resume_target(1, target_pool_idx)); } #[test] @@ -1386,12 +1440,12 @@ mod tests { assert!(matches!(result, Err(Error::SlowDown))); } - #[test] - fn equivalent_data_movement_tiered_object_accepts_matching_transition_metadata() { + fn tiered_equivalence_source() -> FileInfo { let version_id = Uuid::nil(); let transition_version_id = Uuid::new_v4(); let mod_time = OffsetDateTime::UNIX_EPOCH; - let source = FileInfo { + + FileInfo { version_id: Some(version_id), size: 1024, mod_time: Some(mod_time), @@ -1400,80 +1454,86 @@ mod tests { transitioned_objname: "remote/object".to_string(), transition_tier: "WARM".to_string(), transition_version_id: Some(transition_version_id), - metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]), - ..Default::default() - }; - let target = ObjectInfo { - version_id: Some(version_id), - size: 1024, - mod_time: Some(mod_time), - checksum: Some(Bytes::from_static(b"checksum")), - etag: Some("etag-value".to_string()), - transitioned_object: TransitionedObject { - name: "remote/object".to_string(), - version_id: transition_version_id.to_string(), - tier: "WARM".to_string(), - status: TRANSITION_COMPLETE.to_string(), + replication_state_internal: Some(rustfs_filemeta::ReplicationState { + replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()), + targets: rustfs_filemeta::replication_statuses_map("arn:minio:replication:target=COMPLETED;"), + version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()), + purge_targets: rustfs_filemeta::version_purge_statuses_map("arn:minio:replication:target=PENDING;"), ..Default::default() - }, + }), + metadata: HashMap::from([ + ("etag".to_string(), "etag-value".to_string()), + ("x-amz-meta-key".to_string(), "metadata-value".to_string()), + (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), "tag=value".to_string()), + ("expires".to_string(), "1970-01-01T00:33:20Z".to_string()), + ]), ..Default::default() - }; + } + } + + fn tiered_equivalence_target(source: &FileInfo) -> ObjectInfo { + ObjectInfo::from_file_info(source, "bucket", "object", source.version_id.is_some()) + } + + #[test] + fn equivalent_data_movement_tiered_object_accepts_matching_persisted_metadata() { + let source = tiered_equivalence_source(); + let target = tiered_equivalence_target(&source); assert!(is_equivalent_data_movement_tiered_object(&source, &target)); } #[test] fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() { - let source = FileInfo { - version_id: Some(Uuid::nil()), - size: 1024, - transition_status: TRANSITION_COMPLETE.to_string(), - transitioned_objname: "remote/source".to_string(), - transition_tier: "WARM".to_string(), - ..Default::default() - }; - let target = ObjectInfo { - version_id: source.version_id, - size: 1024, - transitioned_object: TransitionedObject { - name: "remote/target".to_string(), - tier: "WARM".to_string(), - status: TRANSITION_COMPLETE.to_string(), - ..Default::default() - }, - ..Default::default() - }; + let source = tiered_equivalence_source(); + let mut target = tiered_equivalence_target(&source); + target.transitioned_object.name = "remote/target".to_string(); + + assert!(!is_equivalent_data_movement_tiered_object(&source, &target)); + } + + #[test] + fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() { + let source = tiered_equivalence_source(); + let mut target = tiered_equivalence_target(&source); + target.user_defined = Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "target-value".to_string())])); + + assert!(!is_equivalent_data_movement_tiered_object(&source, &target)); + } + + #[test] + fn equivalent_data_movement_tiered_object_rejects_tag_mismatch() { + let source = tiered_equivalence_source(); + let mut target = tiered_equivalence_target(&source); + target.user_tags = Arc::new("tag=target".to_string()); + + assert!(!is_equivalent_data_movement_tiered_object(&source, &target)); + } + + #[test] + fn equivalent_data_movement_tiered_object_rejects_replication_mismatch() { + let source = tiered_equivalence_source(); + let mut target = tiered_equivalence_target(&source); + target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string()); + target.replication_status = rustfs_filemeta::ReplicationStatusType::Failed; + + assert!(!is_equivalent_data_movement_tiered_object(&source, &target)); + } + + #[test] + fn equivalent_data_movement_tiered_object_rejects_version_purge_mismatch() { + let source = tiered_equivalence_source(); + let mut target = tiered_equivalence_target(&source); + target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string()); + target.version_purge_status = rustfs_filemeta::VersionPurgeStatusType::Complete; assert!(!is_equivalent_data_movement_tiered_object(&source, &target)); } #[test] fn data_movement_tiered_resume_accepts_equivalent_target() { - let version_id = Uuid::nil(); - let transition_version_id = Uuid::new_v4(); - let source = FileInfo { - version_id: Some(version_id), - size: 1024, - transition_status: TRANSITION_COMPLETE.to_string(), - transitioned_objname: "remote/object".to_string(), - transition_tier: "WARM".to_string(), - transition_version_id: Some(transition_version_id), - metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]), - ..Default::default() - }; - let target = ObjectInfo { - version_id: Some(version_id), - size: 1024, - etag: Some("etag-value".to_string()), - transitioned_object: TransitionedObject { - name: "remote/object".to_string(), - version_id: transition_version_id.to_string(), - tier: "WARM".to_string(), - status: TRANSITION_COMPLETE.to_string(), - ..Default::default() - }, - ..Default::default() - }; + let source = tiered_equivalence_source(); + let target = tiered_equivalence_target(&source); let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1) .expect("equivalent tiered target should be evaluated"); @@ -1483,28 +1543,8 @@ mod tests { #[test] fn data_movement_tiered_resume_rejects_source_pool_target() { - let version_id = Uuid::nil(); - let source = FileInfo { - version_id: Some(version_id), - size: 1024, - transition_status: TRANSITION_COMPLETE.to_string(), - transitioned_objname: "remote/object".to_string(), - transition_tier: "WARM".to_string(), - metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]), - ..Default::default() - }; - let target = ObjectInfo { - version_id: Some(version_id), - size: 1024, - etag: Some("etag-value".to_string()), - transitioned_object: TransitionedObject { - name: "remote/object".to_string(), - tier: "WARM".to_string(), - status: TRANSITION_COMPLETE.to_string(), - ..Default::default() - }, - ..Default::default() - }; + let source = tiered_equivalence_source(); + let target = tiered_equivalence_target(&source); let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0) .expect("source-pool target should be rejected before target lookup"); @@ -1623,6 +1663,39 @@ mod tests { assert!(matches!(err, Error::MethodNotAllowed)); } + #[test] + fn should_create_delete_marker_for_missing_object_allows_latest_versioned_delete() { + let opts = ObjectOptions { + versioned: true, + ..Default::default() + }; + + assert!(should_create_delete_marker_for_missing_object(&opts)); + } + + #[test] + fn should_create_delete_marker_for_missing_object_rejects_specialized_deletes() { + let version_delete = ObjectOptions { + versioned: true, + version_id: Some("vid-1".to_string()), + ..Default::default() + }; + let delete_marker_replication = ObjectOptions { + versioned: true, + delete_marker: true, + ..Default::default() + }; + let data_movement = ObjectOptions { + versioned: true, + data_movement: true, + ..Default::default() + }; + + assert!(!should_create_delete_marker_for_missing_object(&version_delete)); + assert!(!should_create_delete_marker_for_missing_object(&delete_marker_replication)); + assert!(!should_create_delete_marker_for_missing_object(&data_movement)); + } + #[test] fn resolve_decommission_target_pool_idx_result_passthrough_ok() { let idx = ECStore::resolve_decommission_target_pool_idx_result(Ok(3), "bucket", "object").unwrap(); @@ -1715,6 +1788,29 @@ mod tests { assert!(lookup_opts.skip_rebalancing); } + #[test] + fn transition_restore_pool_opts_skips_decommissioned_and_preserves_locking() { + let lookup_opts = transition_restore_pool_opts(&ObjectOptions { + no_lock: false, + skip_decommissioned: false, + ..Default::default() + }); + + assert!(lookup_opts.skip_decommissioned); + assert!(!lookup_opts.no_lock); + } + + #[test] + fn transition_restore_pool_opts_preserves_existing_no_lock() { + let lookup_opts = transition_restore_pool_opts(&ObjectOptions { + no_lock: true, + ..Default::default() + }); + + assert!(lookup_opts.skip_decommissioned); + assert!(lookup_opts.no_lock); + } + #[tokio::test] #[serial_test::serial] async fn reader_lock_is_held_when_optimization_is_disabled() { diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 106f34b1f..9f201b054 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -1080,8 +1080,11 @@ pub struct ReloadPoolMetaResponse { #[prost(string, optional, tag = "2")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } -#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] -pub struct StopRebalanceRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StopRebalanceRequest { + #[prost(string, tag = "1")] + pub expected_rebalance_id: ::prost::alloc::string::String, +} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StopRebalanceResponse { #[prost(bool, tag = "1")] diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 1c7111ceb..e0b6c9778 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -764,7 +764,9 @@ message ReloadPoolMetaResponse { optional string error_info = 2; } -message StopRebalanceRequest {} +message StopRebalanceRequest { + string expected_rebalance_id = 1; +} message StopRebalanceResponse { bool success = 1; diff --git a/docs/architecture/decommission-compatibility.md b/docs/architecture/decommission-compatibility.md new file mode 100644 index 000000000..249ee903f --- /dev/null +++ b/docs/architecture/decommission-compatibility.md @@ -0,0 +1,165 @@ +# Decommission Compatibility Scope + +This note records the current RustFS decommission contract for admin/API +compatibility reviews. + +## Current Contract + +RustFS supports queued multi-pool decommission start requests on multi-pool +deployments. + +The admin handler accepts the request shape used by the MinIO-compatible admin +API, including comma-separated pool targets. An empty target list is rejected. +Single-pool deployments reject decommission because there is no destination pool. +On multi-pool deployments, one or more valid target pools are accepted as a +single queued operation. + +### Request Semantics + +`POST /v3/pools/decommission` with comma-separated pool targets is treated as a +queue submission: + +- validate all requested pool identifiers before mutating metadata; +- reject duplicate target pools in the same request; +- reject active or queued target pools; +- reject completed decommission targets because completion means the pool can be + removed from the deployment configuration; +- allow failed or canceled targets to be retried; +- persist queued metadata before starting workers; +- start only the local-leader prefix of the queue on the receiving node. + +The local-leader-prefix rule keeps the active worker on the leader for the pool +being moved while still allowing a request to contain later targets whose leaders +are different nodes. Later queued targets are recovered or promoted by the +leader that owns that target. + +### Persisted Metadata Shape + +The queue is persisted in pool metadata and decoded with the rest of +`PoolMeta`. Each pool entry can distinguish: + +- `active`: at most one pool currently moving data; +- `queued`: validated pools waiting for the active entry to finish; +- `completed`: pools finished successfully; +- `failed`: pools whose worker reached terminal failure; +- `canceled`: pools canceled before or during execution. + +Legacy metadata without queue fields decodes as a non-queued decommission entry, +preserving restart behavior for already deployed clusters. + +### Serial Scheduling And Recovery + +Only one queued entry may own a decommission worker at a time. Startup recovery: + +- loads pool metadata before rebalance recovery; +- resumes the first local non-terminal active/queued entry; +- skips a durably completed prefix and promotes the next queued entry only after + successful completion; +- treats failed or canceled terminal entries as an automatic-promotion barrier, + leaving later queued pools visible but stopped until an operator retries, + clears, or otherwise resolves the terminal entry; +- keeps queued pools out of active worker scheduling until promotion, while still + making their future state visible in admin status. + +Promotion is persisted before worker execution. If cancellation is already +requested immediately after promotion, RustFS persists a canceled terminal state +instead of leaving the promoted pool active without a worker. + +### Cancel Semantics + +Cancel separates active and queued behavior: + +- canceling the active entry requests worker cancellation and persists terminal + metadata; +- canceling a queued entry marks that entry canceled before it becomes active; +- failed or canceled terminal entries can be cleared explicitly when the operator + chooses to abandon the decommission attempt; +- peer reload failures during cancel must be surfaced in status and logs. + +Cancel requests can be accepted on non-leader nodes as remote cancel intent; the +leader observes the pending cancel and applies it to the active worker. + +### Status Response Shape + +`GET /v3/pools/list` and `GET /v3/pools/status?pool=...` expose per-pool +machine-readable decommission state. The `status` field can report `active`, +`running`, `queued`, `complete`, `failed`, or `canceled`. + +When decommission metadata is present, `decommissionInfo` includes: + +- queue and terminal flags: `queued`, `complete`, `failed`, `canceled`; +- progress counters: `objectsDecommissioned`, + `objectsDecommissionedFailed`, `bytesDecommissioned`, and + `bytesDecommissionedFailed`; +- current location: `bucket`, `prefix`, and `object`; +- queue/history lists: `queuedBuckets` and `decommissionedBuckets`; +- `waitingReason`, currently `queued` for queued entries and + `waiting_for_worker` when metadata exists but no worker has started. + +This makes queued pools and stalled metadata visible without requiring operators +to inspect pool metadata files directly. + +## MinIO Divergence Decisions + +This section records the current product decisions for behavior that is close to +MinIO but not always byte-for-byte identical. + +### Empty Delete Markers + +MinIO decommission documentation states that empty delete markers, meaning delete +markers with no successor object versions, are not transitioned to another pool. + +RustFS follows that behavior for decommission when the bucket has no replication +configuration: a lone remaining delete marker is treated as cleanup-only metadata +and is skipped. When replication is configured, RustFS intentionally keeps the +delete marker eligible for movement so delete-marker replication and purge state +are not lost. + +RustFS rebalance uses the same predicate as decommission: skip only a lone delete +marker without replication. This is intentional even though MinIO's public +documentation calls out the decommission case more explicitly than the rebalance +case. + +Regression guards: + +- `should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication` +- `should_skip_decommission_delete_marker_characterizes_replication_configured` +- `test_should_skip_rebalance_delete_marker_characterizes_empty_marker_without_replication` +- `test_should_skip_rebalance_delete_marker_characterizes_replication_configured` + +### Lifecycle-Expired Versions During Cleanup + +MinIO decommission ignores versions that are already expired by lifecycle rules. +RustFS follows that decommission behavior by allowing safely expired versions to +count toward source cleanup completion. + +RustFS rebalance is intentionally stricter. Expired versions do not prove that a +target pool received an equivalent version, so rebalance cleanup requires actual +rebalance completion for the source entry instead of treating lifecycle-expired +versions as moved. + +Regression guards: + +- `test_should_cleanup_decommission_source_entry_accepts_migrated_and_safely_expired_versions` +- `test_should_cleanup_decommission_source_entry_accepts_versions_only_safely_expired_by_lifecycle` +- `test_should_cleanup_rebalance_source_entry_rejects_versions_only_expired_by_lifecycle` + +No migration step is required for these decisions because this note documents the +current RustFS behavior. Changing either decision later requires an operator +compatibility note and updated characterization tests. + +## Regression Guard + +The queued multi-pool contract is guarded by: + +- `test_contextualized_decommission_start_request_allows_multiple_target_pools` +- `test_decommission_start_local_leader_allows_remote_queued_pool` +- `test_local_decommission_queue_prefix_stops_at_remote_leader` +- `test_pool_meta_queued_decommission_is_not_suspended_until_promoted` +- `test_pool_meta_promoted_queued_decommission_can_be_canceled` +- `test_first_resumable_decommission_queue_indices_stops_at_failed_or_canceled_state` +- `test_first_resumable_decommission_queue_indices_allows_after_completed_prefix` +- `admin_pool_list_item_exposes_queued_decommission_state` + +These tests live in `crates/ecstore/src/pools.rs` and +`rustfs/src/app/admin_usecase.rs`. diff --git a/docs/architecture/rebalance-decommission-followup-review-plan.md b/docs/architecture/rebalance-decommission-followup-review-plan.md new file mode 100644 index 000000000..cebed5b81 --- /dev/null +++ b/docs/architecture/rebalance-decommission-followup-review-plan.md @@ -0,0 +1,1790 @@ +# Rebalance and Decommission Follow-up Review Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the remaining gaps found during the post-implementation review of F01-F14. + +**Architecture:** This plan treats the original F01-F14 work as the baseline and adds focused follow-up tasks with new `Rxx` identifiers. Each task is independently reviewable and should be committed separately. The plan favors fail-closed behavior for state propagation and strict equivalence for data movement. + +**Tech Stack:** Rust, Tokio, ECStore, admin handlers, tonic peer RPC, MessagePack metadata, `tracing`, existing crate-local unit tests. + +--- + +## Scope + +This plan covers only the gaps found during the follow-up review: + +- P1 gaps that can leave cluster state inconsistent or object metadata incomplete. +- P1 async-lock gaps that can block pool routing, status reads, and decommission progress while disk or peer operations are pending. +- P2 gaps where the implementation is safer than before but does not fully meet the written acceptance criteria. +- One startup recovery test-depth gap that should run near the distributed-state fixes. +- One P3 MinIO compatibility gap where RustFS is intentionally more conservative today but the product contract is not yet explicit. + +The following original items had no new material finding in this review and do not need a follow-up task here: + +- F02: decommission unsafe overwrite cleanup safety. +- F06: multipart streaming memory behavior. +- F10: source cleanup preflight placement and not-found idempotence. +- F14: MinIO-like rebalance completion tolerance. + +## Validated Additional Findings + +The following externally reported issues were checked against the current code and are treated as real planning items: + +- Decommission start still writes active local `pool_meta` before peer `reload_pool_meta()`. If reload fails, the admin handler returns an error before worker spawning, leaving an active decommission marker that can suspend the source pool without a running worker. Covered by upgraded R06. +- Rebalance start still performs `check -> init_rebalance_meta -> start_rebalance -> peer load_rebalance_meta(true)` without one atomic start guard. Peer failure is covered by R01; concurrent start identity races and cross-operation start races are covered by R13. +- Multiple decommission paths still hold `pool_meta.write()` across `save(...).await`, including start, cancel, failed, complete, bucket-done, and progress-save paths. Covered by new R12. +- Rebalance stats refresh holds a `rebalance_meta` read guard across `save_rebalance_meta_with_merge(...).await`. Covered by R16. +- Data movement metadata equivalence did not explicitly include user-visible `x-amz-tagging`/`expires` fields that are cleaned out of `user_defined`. Covered by expanded R03. +- Last-delete-marker skip behavior exists in both decommission and rebalance and has helper-level tests, but lacks MinIO-compatible version-listing regression coverage after migration and cleanup. Covered by new R14. +- R14 proof showed that deleting an absent key in a versioned bucket currently creates a delete-marker response without a version ID and `ListObjectVersions` does not report that only delete marker. Covered by new R17. +- The admin handler accepts comma-separated decommission targets, but store validation rejects more than one target pool. This is conservative, but the MinIO compatibility contract is not documented. Covered by R15. +- Source cleanup preflight still treats a prefix-list miss as safe cleanup. Prefix listing is not an exact object-version read, so cleanup should only proceed after exact source metadata verification, or after an explicit object/version not-found result. Covered by R18. +- Transition and restore lookup paths still use the caller's `ObjectOptions` and only contain commented-out `skip_decommissioned` guidance. MinIO skips decommissioned pools for these lifecycle operations. Covered by R19. +- Rebalance delete-marker migration still sets `skip_decommissioned` but not `skip_rebalancing`. MinIO uses `SkipRebalancing=true` for this path to avoid resolving the delete-marker write back to the source pool. Covered by R20. +- Rebalance source cleanup failures are recorded as cleanup warnings and the entry still completes. Non not-found/version-not-found cleanup failures should defer the entry/bucket so retry can converge. Covered by R21. +- Rebalance stop propagation can fail on a subset of peers, and terminal reload currently only loads metadata without cancelling any local worker that missed the stop RPC. Covered by R22 and R23. +- The earlier suggestion that stop/rollback must carry an operation ID is not a MinIO compatibility requirement. The higher-value fix is terminal reload convergence plus propagation observability. Covered by R22 and R23. + +## Execution Order + +| Order | Task | Original Fix | Priority | Main Risk | +| --- | --- | --- | --- | --- | +| 1 | R13 | F05 | P1 | Rebalance and decommission starts are not serialized across check/init/start | +| 2 | R01 | F05 | P1 | Rebalance start propagation failure can leave accepted peers running | +| 3 | R12 | F03 | P1 | Pool metadata write lock is held across async saves | +| 4 | R06 | F03 | P1 | Failed decommission reload can leave active metadata without a worker | +| 5 | R07 | F05 | P1 | Rebalance stop can release mutual exclusion before workers terminate | +| 6 | R02 | F07 | P1 | Multipart migration can drop per-part checksum metadata | +| 7 | R03 | F09 | P1 | Overwrite convergence can accept metadata-incomplete targets | +| 8 | R04 | F11 | P1 | Cleanup warning metadata can become self-inconsistent and fail decode | +| 9 | R11 | F04 | P2 | Store init recovery lacks full integration-level regression coverage | +| 10 | R16 | F05 | P2 | Rebalance metadata refresh holds a read lock across async save | +| 11 | R05 | F01 | P2 | Rebalance delete marker target write can stall when default placement selects source | +| 12 | R08 | F08/F10 | P2 | Cleanup preflight can reject safely expired versions already removed by lifecycle | +| 13 | R09 | F12 | P2 | Decommission rejected-request logs lack complete audit context | +| 14 | R10 | F13 | P2 | Legacy pool metadata fallback can bypass unknown-field hardening | +| 15 | R14 | F01 | P2 | Last delete marker migration semantics lack MinIO-compatible E2E proof | +| 16 | R17 | Versioning compatibility | P2 | Versioned delete of absent key loses only-delete-marker version metadata | +| 17 | R15 | Compatibility | P3 | Multi-pool decommission support is accepted by API shape but rejected by store | +| 18 | R18 | Source cleanup safety | P1 | Prefix-list miss can allow cleanup without exact object-version verification | +| 19 | R19 | Lifecycle pool selection | P1 | Transition/restore can select a decommissioning pool | +| 20 | R20 | Delete marker placement | P1 | Rebalance delete-marker migration can select a rebalancing source pool | +| 21 | R21 | Cleanup retry | P2 | Rebalance cleanup failure can mark an entry complete instead of retrying | +| 22 | R22 | Stop convergence | P1 | Terminal rebalance reload does not cancel local workers that missed stop RPC | +| 23 | R23 | Stop observability | P1 | Stop peer failures and terminal reload propagation are not visible enough | +| 24 | R24 | Compatibility | P3 | Multi-pool decommission queue semantics need a product-approved design | +| 25 | R25 | Compatibility | P3 | Rebalance single-coordinator semantics need a broader namespace-lock design | +| 26 | R26 | Admin API hardening | P3 | Dangerous admin query parameters are not strictly deserialized | + +## Shared Rules + +- Read this file before starting each task and restate the selected task in the commit summary or work log. +- Follow the Execution Order table rather than the numeric order of section headings. +- Implement exactly one `Rxx` task per commit. +- Keep unrelated refactors out of scope. +- Preserve the existing code shape unless a small helper is needed for the task. +- For Rust changes, run the focused tests listed under the task plus `cargo fmt --all --check`. +- Leave unrelated untracked files, such as `CLAUDE-FABLE-5.md`, untouched. + +--- + +## R01: Roll Back Cluster Rebalance Start on Propagation Failure + +### Original Fix + +F05: Rebalance distributed start/stop semantics. + +### Finding + +`rustfs/src/admin/handlers/rebalance.rs` starts the local rebalance before peer propagation. If `notification_sys.load_rebalance_meta(true)` fails, the admin API returns an error, but the local worker may continue running. The peer call is also fan-out: some peers may already have accepted `start_rebalance = true` and spawned workers before another peer fails. + +### Files + +- Modify: `rustfs/src/admin/handlers/rebalance.rs` +- Modify if needed: `crates/ecstore/src/notification_sys.rs` +- Modify if needed: `crates/ecstore/src/rebalance.rs` +- Test: `rustfs/src/admin/handlers/rebalance.rs` +- Test if needed: `crates/ecstore/src/rebalance.rs` + +### Design + +On peer propagation failure after local or peer start: + +1. Attempt local `store.stop_rebalance().await`. +2. Propagate a cluster-wide stop/rollback to peers that may have accepted the start. +3. Persist stopped metadata through the same path used by explicit stop. +4. Include `rebalance_id` in rollback checks when possible so rollback cannot stop a newer operation created after R13 releases the start guard. +5. Return an admin error that includes propagation failure and rollback failure details if rollback is incomplete. +6. Log `result = "rollback_success"`, `result = "rollback_partial"`, or `result = "rollback_failed"` with `request_id`, masked `actor`, `remote_addr`, `rebalance_id`, and peer failure details. + +Do not silently convert propagation failure into success. + +### Implementation Steps + +- [x] Add a helper in `rustfs/src/admin/handlers/rebalance.rs` such as `rollback_cluster_rebalance_start(store, notification_sys, rebalance_id)` that calls local `stop_rebalance()` and peer stop/rollback propagation. +- [x] Add a unit test for formatting rollback failure context so an error includes `failed to propagate rebalance start`, local rollback result, and peer rollback result. +- [x] Add an admin handler or helper-level test that simulates one peer accepting start and another peer failing, then verifies cluster rollback is attempted. +- [x] Update the `RebalanceStart` error path to call the rollback helper before returning. +- [x] If rollback is rebalance-id guarded, add a test where a newer operation ID is present and rollback refuses to stop it. Not applicable for the implemented rollback path because stop/rollback does not use an operation-id compatibility gate. +- [x] Run: + +```bash +cargo test -p rustfs rebalance --lib +cargo test -p rustfs-ecstore stop_rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Admin start does not leave local or peer rebalance workers running after peer propagation failure. +- Partial rollback failure is visible in the returned error, status/logs, and peer details. +- Rollback cannot stop a newer rebalance operation ID. +- Existing successful start behavior is unchanged. + +### Commit + +```bash +git add rustfs/src/admin/handlers/rebalance.rs crates/ecstore/src/rebalance.rs +git commit -m "fix(rebalance): roll back failed start propagation" +``` + +--- + +## R02: Preserve Multipart Per-part Checksums in Completed Metadata + +### Original Fix + +F07: Preserve and verify full data movement metadata. + +### Finding + +`data_movement_complete_part()` fills checksum fields on `CompletePart`, but `complete_multipart_upload` rebuilds `ObjectPartInfo` without copying `ext_part.checksums`, so migrated multipart objects can lose per-part checksum metadata. + +### Files + +- Modify: `crates/ecstore/src/set_disk.rs` +- Test: `crates/ecstore/src/data_movement.rs` +- Test if needed: `crates/ecstore/src/set_disk.rs` + +### Design + +Preserve per-part checksums when completing multipart upload: + +- Copy `ext_part.checksums.clone()` into the new `ObjectPartInfo`. +- Add a test that exercises the actual complete path, not only `CompletePart` construction. +- Ensure the test verifies the resulting target part metadata contains the original checksum map. + +### Implementation Steps + +- [x] Add a failing test that builds an uploaded multipart part with checksums and completes it through the same code that pushes `ObjectPartInfo`. +- [x] Update the `ObjectPartInfo` construction in `crates/ecstore/src/set_disk.rs` to include: + +```rust +checksums: ext_part.checksums.clone(), +``` + +- [x] Add or update a data movement test that migrates a multipart object and asserts target parts preserve checksum entries. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore multipart --lib +cargo test -p rustfs-ecstore data_movement --lib +cargo test -p rustfs-ecstore checksum --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Migrated multipart target metadata contains source per-part checksums. +- Existing multipart checksum validation still rejects mismatched checksums. +- No source cleanup path can remove the only copy of per-part checksum metadata. + +### Commit + +```bash +git add crates/ecstore/src/set_disk.rs crates/ecstore/src/data_movement.rs +git commit -m "fix(data-movement): preserve part checksums" +``` + +--- + +## R03: Strengthen Overwrite Equivalence for Metadata-complete Targets + +### Original Fix + +F09: Rebalance overwrite race equivalence. + +### Finding + +`is_equivalent_data_movement_object()` compares core object fields but does not compare multipart part metadata/checksums, replication/version purge state, or all user-visible metadata. In particular, `ObjectInfo::from_file_info()` extracts `x-amz-tagging` into `user_tags` and parses `expires`, while `clean_metadata()` removes those keys from `user_defined`. A target missing required metadata can be treated as equivalent and allow source cleanup. + +### Files + +- Modify: `crates/ecstore/src/data_movement.rs` +- Test: `crates/ecstore/src/data_movement.rs` +- Test: `crates/ecstore/src/rebalance.rs` + +### Design + +Extend overwrite equivalence to match the fields required by F07: + +- version ID +- delete marker state +- size and actual size +- ETag +- checksum +- mod time +- storage class +- user-defined metadata +- object tags / `x-amz-tagging` +- `expires` +- replication status/internal state +- version purge status/internal state +- multipart part count and per-part number, ETag, size, actual size, mod time, index, and checksums + +Use exact equality for these fields. A false negative is safer than a false positive. + +### Implementation Steps + +- [x] Add a helper such as `is_equivalent_data_movement_part(source, target)` in `crates/ecstore/src/data_movement.rs`. +- [x] Update `is_equivalent_data_movement_object()` to compare replication/version purge fields, object tags, expires, and call the part helper. +- [x] Add a failing test where source and target differ only by missing per-part checksum and assert overwrite convergence is rejected. +- [x] Add a failing test where source and target differ only by version purge status and assert overwrite convergence is rejected. +- [x] Add failing tests where source and target differ only by object tags or expires and assert overwrite convergence is rejected. +- [x] Confirm the data movement write path preserves tags and expires; if it does not, extend the write options or create a separate write-path task before allowing source cleanup. +- [x] Add a positive test where all fields match and overwrite convergence is accepted. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore DataMovementOverwriteErr --lib +cargo test -p rustfs-ecstore data_movement --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Equivalent overwrite accepts only metadata-complete targets. +- Missing multipart checksum, tags, expires, or replication/version purge state blocks convergence. +- Data movement writes preserve metadata required by the strengthened equivalence check before cleanup can run. +- Decommission remains fail-closed for unsafe overwrite. + +### Commit + +```bash +git add crates/ecstore/src/data_movement.rs crates/ecstore/src/rebalance.rs +git commit -m "fix(data-movement): require full overwrite equivalence" +``` + +--- + +## R04: Keep Cleanup Warning Merge Metadata Self-consistent + +### Original Fix + +F11: Rebalance cleanup failure reporting. + +### Finding + +`merge_rebalance_cleanup_warnings()` uses `max(count)` but merges and de-duplicates entries. Two nodes with different warning entries and count `1` can produce `count = 1` and `entries.len() = 2`, which violates the F13 decode validation. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Test: `crates/ecstore/src/rebalance.rs` + +### Design + +After merging entries: + +- Preserve total warning count semantics as much as possible. +- Guarantee `count >= entries.len()` before metadata can be saved. +- Keep bounded entries at `REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT`. + +Minimal safe rule: + +```rust +remote.count = remote.count.max(local.count); +merge_rebalance_cleanup_warning_entries(&mut remote.entries, &local.entries); +remote.count = remote.count.max(remote.entries.len() as u64); +``` + +If avoiding `as` casts, use `u64::try_from(remote.entries.len()).unwrap_or(u64::MAX)` or an error-returning helper if the function becomes fallible. + +### Implementation Steps + +- [x] Add a failing test where remote has one warning entry, local has a different warning entry, both have count `1`, and merge produces `count >= 2`. +- [x] Add a decode test that serializes the merged metadata and decodes it through `RebalanceMeta::decode_rebalance_meta_payload`. +- [x] Update `merge_rebalance_cleanup_warnings()` to normalize count after merging entries. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore cleanup_warning --lib +cargo test -p rustfs-ecstore rebalance_meta --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Merged cleanup warnings never violate metadata validation. +- Count remains at least the number of retained entries. +- Entry retention remains bounded. + +### Commit + +```bash +git add crates/ecstore/src/rebalance.rs +git commit -m "fix(rebalance): normalize cleanup warning counts" +``` + +--- + +## R05: Force Rebalance Delete Marker Writes to a Non-source Target + +### Original Fix + +F01: Rebalance delete marker and remote tiered version safety. + +### Finding + +Rebalance delete marker movement routes through `ECStore::delete_object()`. In the data-movement branch, `handle_delete_object()` computes a non-source `target_pool_idx` for equivalence checks, but the actual delete marker write still uses `selected_target_pool_idx`. If default placement resolves to the source pool, the path may return `DataMovementOverwriteErr` rather than writing the known non-source fallback target. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Modify if needed: `crates/ecstore/src/store/object.rs` +- Test: `crates/ecstore/src/rebalance.rs` + +### Design + +The data-movement delete marker path should: + +1. Exclude `src_pool_idx` from target selection. +2. Write the delete marker metadata to the resolved non-source `target_pool_idx`, not the original source-selected pool. +3. Treat overwrite as complete only after strict target equivalence is proven. +4. Return failure if no non-source target is available. + +### Implementation Steps + +- [x] Add a failing test where default placement selects the source pool but another target pool is available; rebalance delete marker migration must still write to the non-source target. +- [x] Fix or prove the `handle_delete_object()` data-movement delete marker branch so it writes through the resolved non-source fallback target. +- [x] Keep `rebalance_delete_marker()` narrow unless rebalance-specific context is required. +- [x] Preserve version ID, delete marker flag, mod time, replication delete marker state, and `src_pool_idx`. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore rebalance_delete_marker --lib +cargo test -p rustfs-ecstore rebalance_entry --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Rebalance delete markers are written to a non-source target when one exists. +- Source cleanup remains blocked on target write failure. +- Remote tiered versions remain skipped without cleanup permission. + +### Commit + +```bash +git add crates/ecstore/src/rebalance.rs crates/ecstore/src/store/object.rs +git commit -m "fix(rebalance): target delete marker movement" +``` + +--- + +## R06: Prevent Active-but-no-worker Decommission Starts + +### Original Fix + +F03: Decommission pool meta reload barrier. + +### Finding + +`start_decommission()` saves active `pool_meta` before peer reload. Reload failure returns an API error before the admin handler reaches `spawn_decommission_routines`, but the local pool already has active decommission metadata. That state can make `is_suspended()` exclude the pool and `is_decommission_running()` block rebalance while no local decommission worker is moving data until restart recovery happens. Partial peer reload success can also leave remote node memory with active decommission metadata even if the local node later rolls back. + +### Files + +- Modify: `crates/ecstore/src/pools.rs` +- Modify: `crates/ecstore/src/store/init.rs` +- Modify if status fields are added: `rustfs/src/admin/handlers/pools.rs` +- Test: `crates/ecstore/src/pools.rs` +- Test: `crates/ecstore/src/store/init.rs` + +### Design Options + +Choose one implementation before coding: + +1. **Rollback on reload failure.** + - Revert the just-written decommission state and save pool meta again. + - Propagate the rollback metadata to peers that may have accepted the active state. + - Report any rollback propagation failure as an explicit partial rollback condition. + - Safer for user-visible API semantics, but needs careful persistence handling. + - Do not use cancel semantics if cancel means "decommissioned pool remains out of ordinary placement" in MinIO-compatible behavior. + +2. **Persist degraded state.** + - Add a durable `reload_failed` or `start_degraded` marker to decommission info. + - Admin status exposes the marker. + - Store init refuses to auto-resume degraded decommission until reload succeeds or an admin action clears it. + - The admin handler may still start local workers only if the degraded state is explicit and visible in status/logs. + - Propagate the degraded marker to peers or report peer memory state as unknown/partial if propagation fails. + +Recommended first implementation: rollback if the current pool meta mutation can be reverted locally without losing unrelated state; otherwise persist degraded state and keep local workers running while surfacing the propagation failure. Do not return an error while leaving active metadata with no worker. + +### Implementation Steps + +- [x] Add a failing test where `reload_pool_meta()` fails after `pool_meta.save()` and the resulting local state is not active without a worker. +- [x] Implement the selected rollback or degraded-state behavior. +- [x] If rollback is chosen, prove `is_suspended()` and `is_decommission_running()` return normal non-decommission behavior after the failed start. +- [x] If rollback is chosen, prove peers that accepted the active state receive rollback metadata, or status reports partial rollback with affected peers. +- [x] If degraded-start-with-worker is chosen, prove status/logs expose `reload_pool_meta` failure and local worker startup proceeds. Not applicable because R06 selected rollback instead of degraded-start-with-worker. +- [x] If degraded-start-with-worker is chosen, prove peers either receive the degraded marker or status reports unknown peer state. Not applicable because R06 selected rollback instead of degraded-start-with-worker. +- [x] Add restart coverage proving failed starts do not auto-resume as successful starts unless the durable degraded-state design explicitly allows it. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore start_decommission --lib +cargo test -p rustfs-ecstore reload_pool_meta --lib +cargo test -p rustfs-ecstore init --lib +cargo test -p rustfs pools --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Admin start failure cannot leave active decommission metadata without a local worker. +- Failed propagation either rolls back local active state or starts local workers with visible degraded status. +- Partial peer reload/rollback state is surfaced; no node silently remains active-without-worker. +- Rebalance is not blocked by a hidden half-started decommission. +- Operators can see that reload failed, rollback happened, or degraded local execution is active. +- Existing successful decommission start behavior remains unchanged. + +### Commit + +```bash +git add crates/ecstore/src/pools.rs crates/ecstore/src/store/init.rs rustfs/src/admin/handlers/pools.rs +git commit -m "fix(decommission): prevent half-started state" +``` + +--- + +## R07: Distinguish Rebalance Stopping from Stopped + +### Original Fix + +F05: Rebalance distributed stop semantics. + +### Finding + +`stop_rebalance_state()` cancels the token and immediately marks started pools as `Stopped`. Admin status can report stopped while workers are still winding down. More importantly, rebalance/decommission mutual exclusion can be released too early if `is_rebalance_in_progress()` treats stop-requested metadata as terminal before worker acknowledgement is persisted. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Modify: `rustfs/src/admin/handlers/rebalance.rs` +- Test: `crates/ecstore/src/rebalance.rs` +- Test: `rustfs/src/admin/handlers/rebalance.rs` + +### Design + +Add an explicit stop-requested state without breaking existing clients: + +- Prefer an additive admin status field such as `stopping: bool` on pool status. +- Keep existing `status` values backward-compatible if adding a new enum variant would be too disruptive. +- Mark `stopping = true` after cancellation is requested and before worker terminal acknowledgement. +- Mark final stopped only after worker terminal handling records the stop event. +- Treat `stopping = true` as an active conflict for decommission/rebalance start guards until terminal acknowledgement is recorded. + +### Implementation Steps + +- [x] Add a test where stop is requested but a pool still has an active worker/cancel token and admin status reports `stopping = true`. +- [x] Add a test where terminal stop event clears `stopping` and reports stopped. +- [x] Add a test where decommission start is rejected while rebalance is stopping but not terminal. +- [x] Implement the minimal metadata/status fields required to distinguish the states. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore stop_rebalance --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo test -p rustfs rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Operators can distinguish stop requested from fully stopped. +- Decommission/rebalance mutual exclusion is not released until worker terminal acknowledgement is persisted. +- Stop failure propagation from F05 remains intact. +- Existing JSON consumers still receive the original status fields. + +### Commit + +```bash +git add crates/ecstore/src/rebalance.rs rustfs/src/admin/handlers/rebalance.rs +git commit -m "fix(rebalance): expose stopping status" +``` + +--- + +## R08: Allow Cleanup Preflight to Tolerate Confirmed Safe-expired Versions + +### Original Fix + +F08 and F10: lifecycle-expired cleanup semantics and cleanup preflight. + +### Finding + +Decommission can count safe lifecycle-expired versions toward cleanup, but cleanup preflight compares the full original `FileInfoVersions`. If lifecycle removes the safe-expired version before preflight, cleanup is incorrectly blocked. + +### Files + +- Modify: `crates/ecstore/src/data_movement.rs` +- Modify: `crates/ecstore/src/pools.rs` +- Test: `crates/ecstore/src/pools.rs` +- Test: `crates/ecstore/src/data_movement.rs` + +### Design + +Build an expected cleanup identity set that distinguishes: + +- required versions that must still match; +- versions explicitly confirmed safe-expired that may be absent; +- any new or changed version, which must still fail preflight. + +Do not allow arbitrary missing versions. Only allow absence for versions counted as safe-expired during the same entry processing. + +### Implementation Steps + +- [x] Extend the cleanup preflight helper to accept optional allowed-missing identities. +- [x] In decommission, collect identities for versions counted in `expired`. +- [x] Pass the allowed-missing set to cleanup preflight. +- [x] Add a test where a safe-expired version is absent during preflight and cleanup is allowed. +- [x] Add a negative test where a non-expired migrated version is absent or changed and cleanup is blocked. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore decommission_entry --lib +cargo test -p rustfs-ecstore cleanup --lib +cargo test -p rustfs-ecstore lifecycle --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Confirmed safe-expired versions may disappear before cleanup without blocking cleanup. +- Changed, new, or unexpectedly missing protected versions still block cleanup. +- Rebalance preflight behavior remains strict unless it explicitly tracks safe-expired versions. + +### Commit + +```bash +git add crates/ecstore/src/data_movement.rs crates/ecstore/src/pools.rs +git commit -m "fix(decommission): tolerate expired cleanup preflight" +``` + +--- + +## R09: Complete Decommission Audit Context on Rejected Requests + +### Original Fix + +F12: Structured audit fields. + +### Finding + +Some decommission rejected paths still call older helpers that log operation/reason/pool without request ID, actor, or remote address after those values are already available. + +### Files + +- Modify: `rustfs/src/admin/handlers/pools.rs` +- Test: `rustfs/src/admin/handlers/pools.rs` + +### Design + +Normalize all decommission start/cancel reject logs after authentication to include: + +- `event` +- `component` +- `subsystem` +- `operation` +- `action` +- `result = "rejected"` +- `reason` +- `request_id` +- `actor` +- `remote_addr` +- target `pool` or `pool_index` when available + +### Implementation Steps + +- [x] Add helper overloads or a small `PoolAuditContext` struct containing `request_id`, `actor`, and `remote_addr`. +- [x] Update invalid query, invalid pool, pool not found, and pool index out-of-range paths in decommission start/cancel to use contextual logging after authentication. +- [x] Add log-capture tests if an existing tracing test helper is available; otherwise add helper-level tests for field construction. +- [x] Run: + +```bash +cargo test -p rustfs pools --lib +scripts/check_logging_guardrails.sh +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Rejected decommission admin operations can be searched by actor, request ID, remote address, and target. +- Logs still avoid raw credentials and authorization headers. +- Existing error responses remain unchanged. + +### Commit + +```bash +git add rustfs/src/admin/handlers/pools.rs +git commit -m "fix(admin): complete decommission audit context" +``` + +--- + +## R10: Harden Legacy Pool Metadata Fallback + +### Original Fix + +F13: Persisted metadata decode hardening. + +### Finding + +If strict `PersistedPoolMeta` decode fails, code falls back to lenient `PoolMeta` decode. Legacy-shaped payloads containing `version`, `pools`, `dont_save`, and unexpected fields can be silently accepted. + +### Files + +- Modify: `crates/ecstore/src/pools.rs` +- Test: `crates/ecstore/src/pools.rs` + +### Design + +Keep legacy compatibility but make it explicit: + +- Introduce a private `LegacyPoolMeta` DTO with `#[serde(deny_unknown_fields)]`. +- Include exactly the legacy fields that are known to be supported: `version`, `pools`, and `dont_save`. +- Convert `LegacyPoolMeta` into `PoolMeta`. +- Reset `dont_save` to `false` after decode. +- Preserve terminal-state validation. + +### Implementation Steps + +- [x] Add a failing test for a legacy-shaped payload with an extra `unexpected` field and assert decode fails. +- [x] Add a passing test for a legacy-shaped payload with only supported legacy fields. +- [x] Add `LegacyPoolMeta`, `LegacyPoolStatus` if needed, and `LegacyPoolDecommissionInfo` if needed. +- [x] Replace lenient `rmp_serde::from_slice::` fallback with strict legacy DTO decode. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore pool_meta --lib +cargo test -p rustfs-ecstore metadata --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Supported legacy metadata still decodes. +- Unknown legacy-shaped fields fail with actionable decode context. +- Runtime-only `dont_save` is never restored from disk. + +### Commit + +```bash +git add crates/ecstore/src/pools.rs +git commit -m "fix(metadata): harden legacy pool meta decode" +``` + +--- + +## R11: Add Store Init Integration Regression for Decommission/Rebalance Conflict + +### Original Fix + +F04: Store init recovery ordering. + +### Finding + +The startup ordering fix has helper tests but lacks an integration-level test that exercises persisted active decommission metadata plus persisted rebalance metadata through `ECStore::init()`. + +### Files + +- Modify: `crates/ecstore/src/store/init.rs` +- Test: `crates/ecstore/src/store/init.rs` +- Possibly use existing test helpers in: `crates/ecstore/src/pools.rs`, `crates/ecstore/src/rebalance.rs` + +### Design + +Add a store-init-level regression test that proves: + +- active decommission metadata is installed before rebalance auto-start is considered; +- rebalance auto-start is skipped when decommission is active; +- rebalance still auto-starts when only rebalance metadata exists. + +Avoid broad test infrastructure rewrites. Use existing init helper seams if full `ECStore::init()` setup is too expensive. + +### Implementation Steps + +- [x] Inspect existing `store::init::tests` helpers and identify the narrowest integration seam that loads persisted pool meta and rebalance meta. +- [x] Add a test for active decommission plus started rebalance metadata and assert rebalance is not started. +- [x] Add or keep a positive test where only rebalance metadata allows auto-start. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore init --lib +cargo test -p rustfs-ecstore rebalance_meta --lib +cargo test -p rustfs-ecstore decommission --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Future refactors cannot reorder init recovery without failing tests. +- Decommission resume behavior remains unchanged except for blocking concurrent rebalance. +- No production behavior changes are introduced by this test-only task. + +### Commit + +```bash +git add crates/ecstore/src/store/init.rs +git commit -m "test(init): cover decommission rebalance recovery" +``` + +--- + +## R12: Remove Async Saves from Pool Metadata Write Guards + +### Original Fix + +F03: Decommission pool metadata durability and reload semantics. + +### Finding + +Several decommission paths call `pool_meta.save(...).await` while holding `self.pool_meta.write().await`. Confirmed locations include start, cancel, failed, complete, bucket-done, and progress-save paths in `crates/ecstore/src/pools.rs`. This can block `is_suspended()`, pool routing, admin status, and progress updates while disk or peer metadata work is pending. It also violates `crates/AGENTS.md`, which forbids holding Tokio write guards across `.await` unless bounded and unavoidable. + +The fix must not simply clone whole `PoolMeta` snapshots and save them concurrently. `PoolMeta::save()` persists the full metadata document; if an older lock-free snapshot finishes after a newer one, it can overwrite bucket-done, progress, failed, or complete state. R12 must preserve save ordering or use a merge/epoch strategy. + +### Files + +- Modify: `crates/ecstore/src/pools.rs` +- Test: `crates/ecstore/src/pools.rs` + +### Design + +Convert each affected path to a short lock section: + +1. Acquire the write guard. +2. Validate and mutate in-memory metadata. +3. Clone the minimal save snapshot or full `PoolMeta` snapshot needed for persistence. +4. Release the write guard before awaiting disk save or peer reload. +5. Serialize full-document saves through a dedicated save mutex, generation/epoch check, or equivalent merge-before-save helper so stale snapshots cannot overwrite newer in-memory state. +6. On save failure, reacquire a short write guard only for conditional handling: roll back if current memory still matches this mutation, otherwise preserve newer state and report the save failure. + +Mutation-specific save failure policy: + +- `start_decommission`: rollback or mark degraded only if the active start still matches this operation. +- `decommission_cancel`: keep the in-memory cancel request visible; return/report save failure so restart risk is known. +- `decommission_failed` and `complete_decommission`: do not silently revert terminal memory state; report durable-state mismatch if persistence fails. +- bucket-done and progress-save: do not roll back counters or bucket progress; retry or report persistence lag without losing newer progress. + +Do not broaden this task into a metadata rewrite. Keep the change local to existing decommission save paths. + +### Implementation Steps + +- [x] Inventory all `pool_meta.write()` sections in `crates/ecstore/src/pools.rs` that await `save()`. +- [x] Add a helper only if it materially reduces repeated snapshot/save/error handling across affected paths. +- [x] Add a save serialization, generation, or merge-before-save mechanism for full `PoolMeta` persistence. +- [x] Refactor `start_decommission`, `decommission_cancel`, `decommission_failed`, `complete_decommission`, bucket-done, and progress-save paths so no Tokio write guard lives across `save(...).await`. +- [x] Add a regression test with an instrumented or delayed save seam proving a concurrent read-side operation such as `is_suspended()` or decommission status lookup is not blocked by an in-flight save. +- [x] Add a regression test where two saves complete out of order and the older snapshot cannot overwrite newer durable state. +- [x] Add tests for save failure handling in start and terminal-state paths. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore decommission --lib +cargo test -p rustfs-ecstore pool_meta --lib +cargo test -p rustfs pools --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- No `self.pool_meta.write().await` guard in decommission paths is held across `pool_meta.save(...).await`. +- Lock-free saves preserve persistence order or merge with current state so stale snapshots cannot overwrite newer metadata. +- Save failures remain visible and do not silently lose required metadata state. +- Save failure handling is mutation-specific and conditional; it does not roll back unrelated or newer updates. +- Pool routing/status reads are not blocked for the duration of slow metadata saves. +- Existing decommission start/cancel/complete behavior remains unchanged except for improved lock scope. + +### Commit + +```bash +git add crates/ecstore/src/pools.rs +git commit -m "fix(decommission): save pool meta outside write lock" +``` + +--- + +## R13: Serialize Rebalance and Decommission Start Gates + +### Original Fix + +F05: Rebalance distributed start semantics. + +### Finding + +The admin handler checks `is_rebalance_conflicting_with_decommission()`, then separately calls `init_rebalance_meta()` and `start_rebalance()`. `init_rebalance_meta()` writes a fresh operation ID after async storage-info and metadata-save work. Two concurrent rebalance starts can both pass the initial check, generate different IDs, and return an ID that no longer matches the effective metadata. A decommission start can also race between rebalance metadata initialization and worker start, leaving active-looking rebalance metadata without workers if `start_rebalance()` later rejects the decommission conflict. + +### Files + +- Modify: `rustfs/src/admin/handlers/rebalance.rs` +- Modify: `crates/ecstore/src/rebalance.rs` +- Modify if needed: `crates/ecstore/src/pools.rs` +- Test if needed: `crates/ecstore/src/pools.rs` +- Test: `crates/ecstore/src/rebalance.rs` +- Test if needed: `rustfs/src/admin/handlers/rebalance.rs` + +### Design + +Make rebalance and decommission starts a single serialized conflict domain: + +- Prefer a dedicated start mutex or metadata namespace lock shared by rebalance start and decommission start. +- For rebalance, the guard covers check, `init_rebalance_meta()`, local `start_rebalance()`, and the point where R01 propagation rollback becomes responsible. +- For decommission, the same guard covers decommission/rebalance conflict checks through active metadata save/reload handling. +- Alternatively, make `init_rebalance_meta()` reject existing active metadata with `OperationAborted` semantics before saving a new ID. +- If `start_rebalance()` fails after `init_rebalance_meta()` saved active-looking metadata, roll back or mark that metadata failed before releasing the guard. +- Keep R01 propagation rollback behavior compatible with the chosen guard. + +The guard should prevent duplicate starts, stale returned operation IDs, and cross-operation decommission/rebalance start races. It should not block unrelated status reads longer than necessary. + +### Implementation Steps + +- [x] Add a concurrent-start regression test where two start attempts race and only one operation ID can be accepted. +- [x] Add a rebalance-vs-decommission race regression test where decommission attempts to start between rebalance metadata init and worker start. +- [x] Add or reuse a shared start guard around rebalance check, `init_rebalance_meta()`, and `start_rebalance()`. +- [x] Apply the same conflict domain to decommission start checks and active-state transition. +- [x] Ensure the loser receives `OperationAborted` or an equivalent existing admin error instead of a stale success ID. +- [x] If local rebalance worker start fails after metadata init, roll back or persist a failed state before releasing the guard. +- [x] Verify R01 rollback path releases the guard and leaves later starts possible. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore start_rebalance --lib +cargo test -p rustfs-ecstore start_decommission --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo test -p rustfs rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Concurrent admin rebalance start requests cannot both return successful different IDs. +- The accepted operation ID matches persisted and in-memory rebalance metadata. +- Rebalance and decommission cannot both pass start checks through separate non-atomic windows. +- Rebalance metadata init followed by worker-start failure cannot leave active-looking metadata without workers. +- Failed or rolled-back starts do not permanently block later starts. + +### Commit + +```bash +git add crates/ecstore/src/rebalance.rs crates/ecstore/src/pools.rs rustfs/src/admin/handlers/rebalance.rs +git commit -m "fix(rebalance): serialize start gates" +``` + +--- + +## R14: Prove Last-delete-marker Migration Semantics Against MinIO Behavior + +### Original Fix + +F01: Delete marker and object-version safety. + +### Finding + +Both decommission and rebalance skip the last remaining delete marker when replication is not configured, then count that version as complete and may clean up the source entry. The helper-level tests prove the predicate, but they do not prove the user-visible behavior after migration: `ListObjectVersions`, current-object `GET`, and version-specific `GET` can expose whether version metadata was lost. Because this behavior was intentionally hardened earlier, treat it as a compatibility proof task first, not an immediate bug fix. + +### Files + +- Test: `crates/e2e_test/src/` +- Test if cheaper seam exists: `crates/ecstore/src/pools.rs` +- Test if cheaper seam exists: `crates/ecstore/src/rebalance.rs` + +### Design + +Add MinIO-compatible regression coverage for versioned buckets: + +- Case A: object has only a delete marker. +- Case B: object has a delete marker plus historical data version. +- Case C: repeat A and B with delete-marker replication configured or explicitly absent. + +For each case, exercise decommission and rebalance when feasible, then verify: + +- `ListObjectVersions` still reports the expected versions/delete markers. +- Current-object `GET` preserves delete-marker not-found semantics. +- Version-specific `GET` preserves access to historical versions. + +This is a mandatory proof gate. If the test proves RustFS loses user-visible version metadata or intentionally differs from MinIO, stop after the failing/characterization test and create a separate implementation or product-decision task. Do not hide behavior changes inside R14. + +### Implementation Steps + +- [x] Identify the narrowest existing E2E harness that can create multi-pool RustFS and call admin decommission/rebalance. +- [x] Add versioned-bucket scenarios for only-delete-marker and delete-marker-plus-history. +- [x] Add assertions for `ListObjectVersions`, current `GET`, and version-specific `GET` after migration and source cleanup. +- [x] If the current behavior loses required version metadata, commit the proof/characterization test as R14 and add a new follow-up implementation task before changing behavior. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore rebalance_delete_marker --lib +cargo test -p rustfs-ecstore decommission --lib +cargo test -p e2e_test versioning -- --nocapture +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Last-delete-marker migration semantics are covered by user-visible versioning assertions. +- R14 contains tests only; behavior fixes or product documentation are split into a separate task if needed. +- Source cleanup cannot remove the only user-visible version metadata without a failing test. + +### Commit + +```bash +git add crates/e2e_test/src +git commit -m "test(data-movement): cover delete marker migration" +``` + +--- + +## R17: Preserve Only-delete-marker Version Metadata + +### Original Fix + +Compatibility follow-up discovered by R14. + +### Finding + +R14 proved a MinIO-incompatible versioning gap before exercising migration: in a versioned bucket, deleting an absent key returns success but does not expose a durable delete-marker version. The delete response lacks `version_id`, and `ListObjectVersions` does not report the only delete marker. Because decommission and rebalance intentionally skip a last remaining delete marker without replication, this baseline gap means migration cleanup can remove or ignore the only user-visible version metadata without an active failing default test. + +### Files + +- Modify: `crates/ecstore/src/store/object.rs` +- Modify if needed: `crates/ecstore/src/set_disk.rs` +- Test: `crates/e2e_test/src/delete_marker_migration_semantics_test.rs` +- Test if cheaper seam exists: `crates/ecstore/src/store/object.rs` + +### Design + +Make versioned deletion of an absent key persist and expose an S3/MinIO-compatible delete marker: + +1. When bucket versioning is enabled and the target key has no existing data version, create a delete marker with a real version ID. +2. Ensure `ListObjectVersions` reports that delete marker as the latest version. +3. Preserve current-object `GET` delete-marker not-found semantics. +4. Keep delete-marker-plus-history behavior unchanged. +5. Unignore the R14 only-delete-marker compatibility test once the behavior is fixed. + +Do not change decommission/rebalance skip policy in this task; first make the underlying versioning semantics compatible and verifiable. + +### Implementation Steps + +- [x] Reproduce the R14 ignored e2e by running it with `--ignored` and confirm it fails on missing only-delete-marker metadata. +- [x] Trace the versioned delete path for absent keys and identify where delete marker metadata is skipped or not assigned a version ID. +- [x] Add a focused unit/helper test if a cheap seam exists, then fix the delete path to persist the delete marker. +- [x] Unignore `test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof`. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore delete_marker --lib +cargo test -p e2e_test delete_marker_migration_semantics -- --nocapture +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Deleting an absent key in an enabled versioned bucket returns a delete-marker version ID. +- `ListObjectVersions` reports the only delete marker as latest. +- Current-object `GET` still returns delete-marker not-found semantics. +- The R14 only-delete-marker compatibility proof runs by default and passes. + +### Commit + +```bash +git add crates/ecstore/src/store/object.rs crates/e2e_test/src/delete_marker_migration_semantics_test.rs +git commit -m "fix(versioning): persist only delete markers" +``` + +--- + +## R15: Document Single-pool Decommission Compatibility Scope + +> Superseded by R24. RustFS now supports queued multi-pool decommission start +> requests on multi-pool deployments. This section is retained as historical +> context for the earlier conservative contract. + +### Original Fix + +Compatibility follow-up for decommission admin behavior. + +### Finding + +Historical finding: `rustfs/src/admin/handlers/pools.rs` parsed +comma-separated pool targets while `crates/ecstore/src/pools.rs` rejected more +than one index with "decommission supports one target pool at a time". That was +the earlier conservative contract. R24 supersedes it with persisted queued +multi-pool decommission. + +### Files + +- Modify: `docs/` +- Test if needed: `crates/ecstore/src/pools.rs` +- Test if needed: `rustfs/src/admin/handlers/pools.rs` + +### Design + +Document single-pool support for this remediation scope: + +- Keep current store behavior. +- Update admin/API/architecture documentation to state that RustFS accepts only one target pool per decommission request. +- Keep the existing reject test as the compatibility guard. +- If product requirements demand MinIO-like queued multi-pool decommission, create a new design task covering persisted queues, restart recovery, active-vs-queued status, and one-active-pool-at-a-time semantics. + +### Implementation Steps + +- [x] Record the historical conservative decommission contract. +- [x] Keep or add a test that comma-separated multiple pools return the documented error. +- [x] Add a short note that MinIO-like queued multi-pool decommission is out of scope for R15 and requires a separate product-approved task. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore start_decommission --lib +cargo test -p rustfs pools --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Superseded by the queued multi-pool contract in + `docs/architecture/decommission-compatibility.md`. + +### Commit + +```bash +git add docs crates/ecstore/src/pools.rs rustfs/src/admin/handlers/pools.rs +git commit -m "docs(decommission): define multi-pool support" +``` + +--- + +## R16: Save Rebalance Metadata Refresh Outside Read Guard + +### Original Fix + +F05: Rebalance metadata lifecycle and distributed state. + +### Finding + +`update_rebalance_stats()` takes a `rebalance_meta` read guard and then calls `save_rebalance_meta_with_merge(...).await` while the guard is still live. This is less severe than the decommission `pool_meta.write()` cases, but it can still delay writers that need to update rebalance state. Other rebalance save paths already clone metadata inside a short lock section and save after releasing the guard. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Test: `crates/ecstore/src/rebalance.rs` + +### Design + +Match the existing `save_rebalance_stats()` pattern: + +1. Acquire the read or write guard only long enough to clone the metadata snapshot that needs saving. +2. Release the guard before calling `save_rebalance_meta_with_merge(...).await`. +3. Preserve existing merge behavior and error context. + +Do not combine this with R13 unless the same helper naturally covers both paths without broad refactoring. + +### Implementation Steps + +- [x] Add a regression test or helper-level assertion showing `update_rebalance_stats()` saves after cloning metadata outside the guard. +- [x] Refactor `update_rebalance_stats()` so no `rebalance_meta` read guard is held across async save. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore update_rebalance_stats --lib +cargo test -p rustfs-ecstore rebalance_meta --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- `update_rebalance_stats()` does not hold a `rebalance_meta` guard across `save_rebalance_meta_with_merge(...).await`. +- Existing merge-on-save behavior remains unchanged. +- Rebalance metadata writers are not blocked by a long read guard during disk/network save. + +### Commit + +```bash +git add crates/ecstore/src/rebalance.rs +git commit -m "fix(rebalance): save metadata refresh outside lock" +``` + +--- + +## R18: Require Exact Source Version Verification Before Cleanup + +### Original Fix + +Source cleanup safety follow-up for decommission and rebalance. + +### Finding + +`ensure_source_cleanup_versions_unchanged()` uses `list_path()` with an object prefix and returns success when the exact object entry is not observed. A prefix-list miss is not the same as an explicit object not-found/version-not-found result. If listing truncation, filtering, cache behavior, or an unexpected prefix collision hides the exact entry, decommission or rebalance cleanup can delete the source without proving that the source versions still match the migrated snapshot. + +### Files + +- Modify: `crates/ecstore/src/data_movement.rs` +- Modify if needed: `crates/ecstore/src/set_disk.rs` +- Test: `crates/ecstore/src/data_movement.rs` +- Test if needed: `crates/ecstore/src/set_disk.rs` + +### Design + +Use exact object metadata resolution for cleanup preflight: + +1. Read the exact object's version metadata from the source `SetDisks`. +2. Treat only explicit object-not-found/version-not-found as already cleaned. +3. Treat read quorum, decode, unexpected list/cache, and other errors as cleanup preflight failures. +4. Compare the exact current versions against the expected migrated snapshot, allowing only the already-modeled lifecycle-expired identities. +5. Keep decommission and rebalance call sites using the same helper. + +Do not rely on prefix listing to prove exact object absence. + +### Implementation Steps + +- [x] Add an exact source metadata read helper that returns `Ok(None)` only for explicit object-not-found/version-not-found. +- [x] Replace the prefix-list based `load_source_cleanup_versions()` path with the exact helper. +- [x] Add a helper-level test that a miss from a non-exact source would fail closed instead of succeeding. +- [x] Add or update tests where explicit not-found remains idempotent. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore source_cleanup --lib +cargo test -p rustfs-ecstore data_movement --lib +cargo test -p rustfs-ecstore decommission --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Source cleanup can proceed only after exact metadata equivalence or explicit not-found/version-not-found. +- Prefix-list absence is no longer a success signal. +- Existing lifecycle-expired allowed-missing behavior is preserved. + +### Commit + +```bash +git add crates/ecstore/src/data_movement.rs crates/ecstore/src/set_disk.rs +git commit -m "fix(data-movement): require exact cleanup preflight" +``` + +--- + +## R19: Skip Decommissioned Pools for Transition and Restore + +### Original Fix + +MinIO compatibility follow-up for lifecycle transition and restore. + +### Finding + +`handle_transition_object()` and `handle_restore_transitioned_object()` still use the caller's `ObjectOptions` when locating the object across pools. The code contains commented-out `skip_decommissioned` guidance, but the option is not applied. MinIO explicitly uses `SkipDecommissioned=true` for these lifecycle operations, so RustFS can route transition/restore work to a pool that is being decommissioned. + +### Files + +- Modify: `crates/ecstore/src/store/object.rs` +- Test: `crates/ecstore/src/store/object.rs` + +### Design + +Clone the incoming options for multi-pool lookup and operation dispatch: + +1. Set `skip_decommissioned = true`. +2. Set `no_lock = true` only if this matches the existing MinIO-compatible lifecycle path and does not bypass a required RustFS lock; otherwise preserve caller `no_lock`. +3. Use the same adjusted options for the pool lookup and the selected pool operation, so lookup and mutation target the same eligibility rules. +4. Keep single-pool behavior unchanged. + +### Implementation Steps + +- [x] Add a small helper that prepares transition/restore options for pool lookup. +- [x] Update `handle_transition_object()` to use the adjusted options on multi-pool paths. +- [x] Update `handle_restore_transitioned_object()` to use the adjusted options on multi-pool paths. +- [x] Add tests asserting transition and restore options set `skip_decommissioned` and preserve the intended `no_lock` behavior. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore transition --lib +cargo test -p rustfs-ecstore restore --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Transition and restore do not select decommissioned pools in multi-pool deployments. +- Lookup and operation dispatch use consistent options. +- Single-pool behavior is unchanged. + +### Commit + +```bash +git add crates/ecstore/src/store/object.rs +git commit -m "fix(lifecycle): skip decommissioned transition pools" +``` + +--- + +## R20: Mark Rebalance Delete-marker Writes as Skip-rebalancing + +### Original Fix + +F01/R05: Rebalance delete-marker movement. + +### Finding + +`rebalance_delete_marker_opts()` sets `skip_decommissioned = true` but not `skip_rebalancing = true`. MinIO uses `SkipRebalancing=true` for rebalance delete-marker movement so placement does not resolve back to a source pool that is actively rebalancing. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Test: `crates/ecstore/src/rebalance.rs` + +### Design + +Add `skip_rebalancing = true` to rebalance delete-marker options while preserving the existing `skip_decommissioned = true` guard. This is a narrow placement fix and should not change delete-marker payload or replication metadata. + +### Implementation Steps + +- [x] Update `rebalance_delete_marker_opts()` to set `skip_rebalancing = true`. +- [x] Extend the existing helper test to assert both `skip_rebalancing` and `skip_decommissioned`. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore rebalance_delete_marker --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Rebalance delete-marker movement skips pools that are currently rebalancing. +- Existing decommission skip and delete-replication state are preserved. + +### Commit + +```bash +git add crates/ecstore/src/rebalance.rs +git commit -m "fix(rebalance): skip rebalancing delete marker targets" +``` + +--- + +## R21: Defer Rebalance Entries on Source Cleanup Failure + +### Original Fix + +F14/R04: Rebalance cleanup tolerance and warning accounting. + +### Finding + +`rebalance_entry()` records non not-found source cleanup delete failures as cleanup warnings and then returns `Completed`. That prevents the bucket retry queue from converging cleanup later. For data safety and operational convergence, only object-not-found/version-not-found should be treated as already cleaned. Other cleanup failures should defer the entry and bucket. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Test: `crates/ecstore/src/rebalance.rs` + +### Design + +Change cleanup delete result handling from warning-only completion to retryable deferral: + +1. Keep `Ok(_)`, object-not-found, and version-not-found as successful cleanup outcomes. +2. For all other cleanup delete errors, record the last error and return `RebalanceEntryOutcome::Deferred`. +3. Preserve cleanup warning accounting only for observability if it remains useful, but do not mark the entry complete. +4. Ensure `wait_rebalance_entry_tasks()` and `defer_rebalance_bucket()` continue to put the bucket back on the queue. + +### Implementation Steps + +- [x] Change cleanup result resolution to return a typed outcome instead of optional warning text, or adjust the call site to treat optional warning as deferral. +- [x] Update tests that currently expect cleanup failures to be ignored. +- [x] Add a test that a transient cleanup failure returns `Deferred` and records the last error. +- [x] Add a test that not-found/version-not-found cleanup remains `Completed`. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore resolve_rebalance_entry_cleanup --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Non not-found cleanup failures do not mark entries or buckets complete. +- Cleanup failure is retried through the existing deferred bucket path. +- Cleanup warning/status remains visible without hiding incomplete cleanup. + +### Commit + +```bash +git add crates/ecstore/src/rebalance.rs +git commit -m "fix(rebalance): defer failed source cleanup" +``` + +--- + +## R22: Cancel Local Rebalance Workers on Terminal Reload + +### Original Fix + +F05/R07: Rebalance distributed stop semantics. + +### Finding + +If a peer misses `stop_rebalance()` but later receives `load_rebalance_meta(start=false)`, the RPC handler loads stopped/stopping metadata but does not cancel the already-running local worker token. `next_rebal_bucket()` also does not explicitly stop bucket selection for stopped/stopping metadata. This can leave a peer worker running after terminal metadata has propagated. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Modify: `rustfs/src/storage/rpc/node_service.rs` +- Test: `crates/ecstore/src/rebalance.rs` +- Test: `rustfs/src/storage/rpc/node_service.rs` + +### Design + +Make terminal reload authoritative for local workers: + +1. After loading rebalance metadata, detect terminal stop state (`stopped_at` set or pool status stopping/stopped). +2. If terminal stop state is present, cancel any local in-memory worker token and keep metadata in stopping/stopped state. +3. In `resolve_next_rebalance_bucket()`, return `Ok(None)` for stopped/stopping metadata before selecting any bucket. +4. Keep `start_rebalance=true` behavior unchanged for active metadata. + +### Implementation Steps + +- [x] Add a helper that applies terminal loaded metadata to the local cancel token. +- [x] Call it from `load_rebalance_meta()` after replacing in-memory metadata. +- [x] Harden `resolve_next_rebalance_bucket()` so stopped/stopping metadata returns no bucket. +- [x] Add tests for terminal reload cancelling an in-memory token. +- [x] Add tests for `next_rebal_bucket()` returning `None` when `stopped_at` or stopping status is present. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore load_rebalance_meta --lib +cargo test -p rustfs-ecstore next_rebal_bucket --lib +cargo test -p rustfs rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- A peer that missed stop RPC stops its local worker after terminal metadata reload. +- Stopped/stopping rebalance metadata cannot hand out more buckets. +- Active reload/start behavior remains unchanged. + +### Commit + +```bash +git add crates/ecstore/src/rebalance.rs rustfs/src/storage/rpc/node_service.rs +git commit -m "fix(rebalance): cancel workers on terminal reload" +``` + +--- + +## R23: Expose Rebalance Stop Propagation Failures + +### Original Fix + +F05/R07: Rebalance distributed stop/status observability. + +### Finding + +`RebalanceStop` and `NotificationSys::stop_rebalance()` surface aggregate peer errors, but status does not expose enough detail about failed peers, last propagation time, or pending terminal reload. When stop partially fails, operators need to see which peers failed and whether terminal reload propagation has been attempted. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Modify: `crates/ecstore/src/notification_sys.rs` +- Modify: `rustfs/src/admin/handlers/rebalance.rs` +- Test: `crates/ecstore/src/rebalance.rs` +- Test: `rustfs/src/admin/handlers/rebalance.rs` + +### Design + +Add minimal propagation observability without changing stop semantics: + +1. Persist local stopped/stopping metadata even when peer stop propagation has failures. +2. Attempt terminal `load_rebalance_meta(false)` after local stop is persisted, even if some peer stop RPCs failed. +3. Record the last stop propagation attempt time and peer failure strings in rebalance metadata or status response. +4. Expose pending/failed propagation in admin status. + +Do not add operation-id requirements for stop/rollback as a compatibility gate. + +### Implementation Steps + +- [x] Add fields or a status-only structure for last stop propagation time and failed peer details. +- [x] Update stop handler to persist local stopped metadata before returning peer propagation errors. +- [x] Ensure terminal reload broadcast is attempted after local stop persistence. +- [x] Extend rebalance status serialization to include propagation failure details. +- [x] Add tests for partial peer failure visibility. +- [x] Run: + +```bash +cargo test -p rustfs rebalance --lib +cargo test -p rustfs-ecstore stop_rebalance --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Partial stop propagation failures remain visible to clients/operators. +- Local stopped/stopping metadata is persisted before stop returns. +- Terminal reload propagation is attempted after stop persistence. +- No operation-id compatibility requirement is introduced for stop. + +### Commit + +```bash +git add crates/ecstore/src/rebalance.rs crates/ecstore/src/notification_sys.rs rustfs/src/admin/handlers/rebalance.rs +git commit -m "fix(rebalance): expose stop propagation failures" +``` + +--- + +## R24: Design Multi-pool Decommission Queue Compatibility + +### Original Fix + +R15 compatibility follow-up. + +### Finding + +RustFS previously rejected multiple decommission target pools in one request. +MinIO supports submitting multiple pools and processing them serially. RustFS +now implements a persisted queued multi-pool contract with restart recovery, +active/queued status, cancellation semantics, and one-active-pool-at-a-time +worker scheduling. + +### Files + +- Modify: `docs/architecture/decommission-compatibility.md` +- Modify: `docs/architecture/rebalance-decommission-followup-review-plan.md` +- Modify: `crates/ecstore/src/pools.rs` +- Modify: `rustfs/src/app/admin_usecase.rs` +- Modify if query contract changes: `rustfs/src/admin/handlers/pools.rs` + +### Design + +Current queued multi-pool contract: + +1. Comma-separated targets are accepted after all targets are validated. +2. Duplicate target pools in the same request are rejected. +3. Queue state is persisted in pool metadata with legacy decode defaults. +4. Startup skips completed queue prefixes, but failed or canceled entries block + automatic promotion until the operator resolves the terminal entry. +5. Active and queued entries are visible through pool admin status. + +### Implementation Steps + +- [x] Write or update a compatibility design section for queued multi-pool decommission. +- [x] Implement queued multi-pool scheduling and recovery. +- [x] Expose queued/progress state in admin pool status. +- [x] Run: + +```bash +cargo test -p rustfs-ecstore first_resumable_decommission_queue_indices --lib +cargo test -p rustfs admin_pool_list_item --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Multi-pool decommission behavior is explicit and tested. +- Failed or canceled entries do not allow later queued pools to start + automatically. +- Queued pools are visible in admin status. + +### Commit + +```bash +git add docs/architecture/decommission-compatibility.md docs/architecture/rebalance-decommission-followup-review-plan.md +git commit -m "docs(decommission): plan multi-pool queue support" +``` + +--- + +## R25: Design Rebalance Single-coordinator Start Semantics + +### Original Fix + +R13 rebalance start serialization. + +### Finding + +R13 serializes local start gates, but a stronger MinIO-like distributed single-coordinator model may require reading and writing `rebalance.bin` under a namespace lock immediately before start on every coordinator path. This is larger than the current local start guard and should be designed separately from P1 stop convergence. + +### Files + +- Modify: `docs/architecture/rebalance-decommission-followup-review-plan.md` +- Code changes only after design approval. + +### Design + +Define whether RustFS should: + +1. Keep local start gate plus metadata merge as the supported contract. +2. Add a `rebalance.bin` namespace-lock guarded check/init/start path. +3. Elect or require a single admin coordinator for distributed rebalance starts. + +### Design Draft + +Current R13 behavior provides a local start gate around check/init/start and +prevents duplicate operation IDs on one coordinator process. It also preserves +the existing rollback behavior when peer start propagation fails. The remaining +race is distributed: two admin coordinators can still enter the local start gate +on different nodes at nearly the same time, each observing no active local +metadata before one of the writes wins the shared `rebalance.bin` merge. + +The stronger design option is to guard the rebalance start decision with the +same namespace that persists `rebalance.bin`: + +1. Acquire a cluster-visible namespace lock for `rebalance.bin`. +2. Reload the current persisted rebalance metadata while holding that lock. +3. Reject start if the loaded metadata is active, stopping, stopped-but-not + terminally acknowledged, or conflicts with decommission metadata. +4. Compute storage information and the new operation ID. +5. Persist the new metadata before releasing the lock. +6. After the lock is released, start the local worker and broadcast + `load_rebalance_meta(true)` to peers. + +The lock should cover only the metadata decision and write. Worker spawn and +peer RPC fan-out should remain outside the lock so a slow or unreachable peer +does not block other metadata readers longer than necessary. + +Failure semantics need explicit review before implementation: + +- If storage-info collection fails before the metadata write, return an error + without changing persisted state. +- If the metadata save fails, return an error without spawning workers. +- If local worker start fails after metadata save, mark the operation failed or + roll back the metadata under the same namespace lock before returning. +- If peer start propagation fails after metadata save and local worker start, + use the existing R01 rollback path or a product-approved degraded-start state. +- If rollback fails, status must expose the active/stale metadata and the peer + propagation failures. + +A stricter single-coordinator model is a separate option: only one elected admin +coordinator may accept rebalance start requests, and non-coordinators forward or +reject start attempts. That can simplify distributed races but adds availability +and leader-election behavior outside the current remediation scope. + +Approval questions before code changes: + +- Is namespace-lock guarded start sufficient, or is a single admin coordinator + required for compatibility? +- Should rebalance start reject when any peer cannot confirm current metadata + before the write, or only after the write during start propagation? +- Should rollback keep the current R01 semantics, or should failed propagation + become a durable degraded state? +- Which namespace lock implementation is authoritative for metadata objects + stored in internal buckets? + +### Implementation Steps + +- [x] Document the current R13 guarantee and the remaining distributed race model. +- [x] Propose the namespace-lock guarded start design with failure/rollback semantics. +- [x] Do not change start behavior until the design is approved. +- [x] Run: + +```bash +cargo fmt --all --check +``` + +### Acceptance Criteria + +- The stronger coordinator model is scoped and reviewable before code changes. +- P1 stop/reload fixes are not blocked by this broader compatibility design. + +### Commit + +```bash +git add docs/architecture/rebalance-decommission-followup-review-plan.md +git commit -m "docs(rebalance): plan coordinator start semantics" +``` + +--- + +## R26: Strict Query Parsing for Dangerous Admin APIs + +### Original Fix + +Admin API hardening follow-up. + +### Finding + +Dangerous admin API query parameters can silently fall back when misspelled or unknown. Tightening this behavior may be compatibility-visible, so it should be designed and rolled out with clear endpoint coverage and error semantics. + +### Files + +- Modify: `docs/architecture/rebalance-decommission-followup-review-plan.md` +- Modify: `rustfs/src/admin/handlers/pools.rs` +- Modify: `rustfs/src/admin/handlers/rebalance.rs` +- Modify: `rustfs/src/app/admin_usecase.rs` + +### Design + +Define strict parsing for high-risk admin APIs: + +1. Identify rebalance/decommission endpoints with destructive or stateful side effects. +2. Define allowed query keys and unknown-key errors. +3. Preserve compatibility for read-only status endpoints unless explicitly approved. +4. Add tests for misspelled dangerous parameters. + +### Design Draft + +Strict query parsing should apply only to admin APIs that mutate distributed +state or start/stop background workers: + +| Endpoint | Method | Query Contract | +| --- | --- | --- | +| `/v3/rebalance/start` | `POST` | No query parameters allowed. | +| `/v3/rebalance/stop` | `POST` | No query parameters allowed. | +| `/v3/pools/decommission` | `POST` | Allow only `pool` and `by-id`. | +| `/v3/pools/cancel` | `POST` | Allow only `pool` and `by-id`. | +| `/v3/pools/clear` | `POST` | Allow only `pool` and `by-id`. | + +Read-only endpoints such as `/v3/rebalance/status`, `/v3/pools/status`, and +`/v3/pools/list` should retain the existing permissive behavior unless product +approval explicitly includes them. This avoids breaking dashboards or scripts +that attach harmless tracking parameters to status requests. + +Current implementation keeps `/v3/pools/status` compatible for unknown query +keys, but rejects duplicate known keys and invalid `by-id` values because those +make pool selection ambiguous. + +The proposed error contract for dangerous endpoints: + +- Unknown query keys return `InvalidArgument`. +- Misspelled known keys, such as `pooll` or `byid`, are treated as unknown keys + and return `InvalidArgument`. +- Repeated keys are rejected unless the endpoint explicitly documents list + semantics. For the current decommission/cancel endpoints, repeated `pool` or + `by-id` should return `InvalidArgument`. +- Empty required values continue to use the existing endpoint-specific invalid + pool or missing target errors after strict key validation passes. +- Rejection logs must include request ID, actor, remote address, and operation + when authentication has already succeeded. + +Implementation uses small local helpers rather than endpoint-specific ad hoc +fallbacks: + +1. Parse the raw query into key/value pairs while preserving duplicate keys. +2. Validate the key set against the endpoint allow-list. +3. Validate duplicate policy. +4. Only then deserialize into the existing typed query struct. + +Tests should cover: + +- `POST /v3/pools/decommission?pooll=...` rejects before defaulting to an empty + `pool`; +- `POST /v3/pools/cancel?byid=true` rejects instead of silently using + `by-id=false`; +- `POST /v3/rebalance/start?dryRun=true` rejects because start accepts no query + keys; +- read-only status endpoints keep current behavior unless included in the + approved endpoint list. + +Resolved implementation decisions: + +- `by-id` accepts only `true` or `false`. +- Repeated `pool` and `by-id` are rejected; comma-separated `pool` remains the + list form for decommission start. +- Unknown query rejection happens after authentication/authorization for these + admin endpoints, preserving contextual audit logs. +- `GET /v3/pools/status?by-id=true&pool=abc` is rejected instead of falling + back to pool index 0. + +### Implementation Steps + +- [x] Document endpoint scope and error contract. +- [x] Harden pool mutation query parsing. +- [x] Harden rebalance start/stop query parsing. +- [x] Harden pool status `by-id` index parsing. +- [x] Run: + +```bash +cargo test -p rustfs pools_handler_tests --lib +cargo test -p rustfs rebalance_handler_tests --lib +cargo test -p rustfs admin_query_pool_status_by_id --lib +cargo fmt --all --check +``` + +### Acceptance Criteria + +- Strict query parsing is scoped to dangerous admin APIs. +- Read-only pool status keeps harmless unknown-key compatibility while rejecting + ambiguous known-key input. +- Invalid `by-id` status queries no longer fall back to pool 0. + +### Commit + +```bash +git add docs/architecture/rebalance-decommission-followup-review-plan.md +git commit -m "docs(admin): align query hardening contract" +``` + +--- + +## Follow-up Test Matrix + +Run after all R01-R23 implementation tasks are complete: + +```bash +cargo test -p rustfs-ecstore rebalance --lib +cargo test -p rustfs-ecstore decommission --lib +cargo test -p rustfs-ecstore data_movement --lib +cargo test -p rustfs-ecstore multipart --lib +cargo test -p rustfs-ecstore metadata --lib +cargo test -p rustfs-ecstore pool_meta --lib +cargo test -p rustfs-ecstore update_rebalance_stats --lib +cargo test -p rustfs-ecstore source_cleanup --lib +cargo test -p rustfs rebalance --lib +cargo test -p rustfs pools --lib +cargo test -p e2e_test versioning -- --nocapture +scripts/check_logging_guardrails.sh +cargo fmt --all --check +``` + +Before opening a PR: + +```bash +cargo fmt --all +cargo fmt --all --check +make pre-commit +``` + +After build-based verification, clean generated build artifacts to avoid unnecessary disk usage. + +## Review Notes + +- R13 should be completed before R01 because rollback must not stop a newer concurrent rebalance operation. +- R01 should be completed before R07 because rollback behavior and stopping status share stop semantics. +- R12 should be completed before R06 because decommission start rollback/degraded handling should use the corrected pool-meta save discipline. +- R11 should run near R06/R13 because it protects startup recovery ordering while distributed start semantics change. +- R02 should be completed before R03 because overwrite equivalence should compare the metadata that migration actually persists. +- R04 should be completed before any wider metadata hardening because it prevents newly strict decode from rejecting metadata produced by the current merge path. +- R06 may require a product decision between rollback and durable degraded state. If that decision cannot be made during implementation, stop before code changes and record the trade-off. +- R14 is a proof task first. If it demonstrates data loss or MinIO-incompatible behavior, create a separate implementation task instead of hiding behavior changes inside the test task. +- R15 was superseded by R24; the current contract is queued multi-pool + decommission on multi-pool deployments. +- R18-R21 are data-safety fixes and should be completed before R22/R23 stop convergence unless a stop-specific regression is blocking the cluster. +- R22 should precede R23 because terminal reload cancellation is the actual convergence mechanism; R23 makes partial failure visible. +- R24 and R26 are now compatibility implementations with focused regression + coverage; R25 remains a larger design task. diff --git a/docs/architecture/rebalance-decommission-implementation-plan-index.md b/docs/architecture/rebalance-decommission-implementation-plan-index.md new file mode 100644 index 000000000..f3a7262f5 --- /dev/null +++ b/docs/architecture/rebalance-decommission-implementation-plan-index.md @@ -0,0 +1,76 @@ +# Rebalance and Decommission Implementation Plan Index + +> This index is based on `docs/architecture/rebalance-decommission-remediation-plan.md`. The remediation scope is intentionally split into smaller implementation plans because the fixes touch independent risk areas: object-version safety, distributed operation semantics, data movement internals, and operational hardening. + +## Why Split the Work + +The remediation backlog has fourteen fix blocks. Implementing them in one PR would make review risky and would mix unrelated failure modes. The safer path is: + +1. Fix the data semantics that can lose or corrupt object-version meaning. +2. Fix distributed start/stop/recovery semantics so operators can trust cluster state. +3. Fix resource and metadata correctness in shared data movement. +4. Add observability and compatibility hardening. + +Each plan below should be reviewed and executed independently unless the plan explicitly says two fixes must share the same implementation. + +## Plan Set + +| Plan | Fixes | Status | Purpose | +| --- | --- | --- | --- | +| `rebalance-decommission-phase1-safety-plan.md` | F01, F02, F03, F04, F05 | Drafted | Protect object-version semantics and cluster operation safety | +| `rebalance-decommission-phase2-data-movement-plan.md` | F06, F07, F08, F09, F10 | Drafted | Stream multipart migration, preserve metadata, and improve convergence | +| `rebalance-decommission-phase3-hardening-plan.md` | F11, F12, F13, F14 | Drafted | Improve cleanup reporting, auditability, metadata decoding, and threshold docs | +| `rebalance-decommission-followup-review-plan.md` | R01-R16 | Reviewed | Close post-implementation review gaps found after F01-F14 | + +## Execution Recommendation + +Start with `rebalance-decommission-phase1-safety-plan.md`. Phase 1 contains the highest-risk issues and defines semantics that later fixes depend on. + +Within Phase 1: + +1. F01 should be analyzed first because rebalance delete marker and remote tiered behavior determines whether RustFS should move or skip these versions. +2. F02 can be implemented independently once the expected decommission overwrite semantics are confirmed. +3. F03, F04, and F05 can be designed together but should remain separate PRs unless a shared peer-failure helper is introduced. + +For a long-running implementation task, use the phase plans as checkpoints: + +1. Implement one fix block at a time. +2. Run the focused tests listed in that block. +3. Review the diff before moving to the next fix block. +4. Run the phase-level test matrix before considering the phase complete. + +After the initial F01-F14 implementation pass, continue with `rebalance-decommission-followup-review-plan.md`. That follow-up plan is ordered by remaining risk and should be executed before treating the remediation as complete. + +## Upstream Change Impact Notes + +### 2026-06-17: `ed55857b refactor: move bucket operations contract (#3507)` + +This upstream change moved `BucketOperations` from `crates/ecstore/src/store_api/traits.rs` into `crates/storage-api/src/bucket.rs` and re-exported it from `rustfs_storage_api`. + +Impact on this remediation plan: + +- No F01-F14 risk item is fixed by this upstream change. +- No planned fix block needs priority changes because of this upstream change. +- Implementation code that imports or bounds `BucketOperations` must now use `rustfs_storage_api::BucketOperations`. +- Generic implementations that need the ECStore error type should use an explicit associated error bound such as `BucketOperations`. +- The only touched planning-relevant files are import/boundary changes in paths such as `crates/ecstore/src/pools.rs`, `crates/ecstore/src/store.rs`, `crates/ecstore/src/set_disk.rs`, and `rustfs/src/admin/handlers/rebalance.rs`; the rebalance/decommission safety logic remains unchanged. + +## Review Gates Before Implementation + +Before any code patch starts for a fix block: + +- Confirm the selected behavior in the corresponding plan. +- Identify the minimal set of files for that fix. +- Write or update failing tests first. +- Keep unrelated refactors out of scope. +- Run focused tests for the touched crate before broader checks. + +## Verification Baseline + +For any code PR generated from these plans: + +- Run focused `cargo test` commands for touched crates and modules. +- Run `cargo fmt --all`. +- Run `cargo fmt --all --check`. +- Run `make pre-commit` before opening a PR when the implementation is ready. +- Clean generated build artifacts after build-based verification. diff --git a/docs/architecture/rebalance-decommission-phase1-safety-plan.md b/docs/architecture/rebalance-decommission-phase1-safety-plan.md new file mode 100644 index 000000000..aca9bd8e6 --- /dev/null +++ b/docs/architecture/rebalance-decommission-phase1-safety-plan.md @@ -0,0 +1,376 @@ +# Rebalance and Decommission Phase 1 Safety Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix the highest-risk rebalance/decommission safety defects so object-version semantics and distributed operation state cannot silently diverge. + +**Architecture:** Phase 1 keeps the existing ECStore/Rebalance/PoolMeta architecture and introduces narrowly scoped behavior changes. The plan favors fail-closed semantics for unsafe migration, peer propagation, and startup conflicts. Each fix block should be implemented as a separate PR unless a shared helper is explicitly called out. + +**Tech Stack:** Rust, Tokio, ECStore, Axum admin handlers, tonic peer RPC, MessagePack metadata, existing RustFS test modules. + +--- + +## Scope + +This plan covers: + +- F01: rebalance delete marker and remote tiered version migration semantics. +- F02: decommission `DataMovementOverwriteErr` cleanup safety. +- F03: decommission `pool_meta` reload as a start barrier. +- F04: store startup recovery ordering for pool meta and rebalance. +- F05: rebalance distributed start/stop semantics and `stopping` visibility. + +This plan does not cover multipart streaming, checksum preservation, lifecycle-expired cleanup, overwrite convergence, cleanup warning UX, audit fields, or metadata decoding hardening. Those belong to later phase plans. + +## Shared Principles + +- Treat unknown target equivalence as unsafe. +- Do not clean a source entry unless the target state is proven complete or the version is intentionally skipped by a documented policy. +- Do not report distributed admin success when required peers failed. +- Do not auto-run rebalance and decommission together after restart. +- Prefer small local helpers near existing logic over broad refactors. + +## F01: Rebalance Delete Marker and Remote Tiered Version Safety + +### Decision Needed + +Remote tiered versions need a product decision before coding: + +- **Recommended short-term behavior:** Match MinIO and skip remote tiered versions during rebalance. This avoids changing remote-tier metadata during a balancing operation and removes the immediate source-cleanup data-loss risk. +- **Later behavior:** Implement explicit cross-pool target metadata movement for remote tiered versions only after tests prove remote references, lifecycle state, and cleanup semantics. + +Delete markers should not be skipped. They preserve versioned delete semantics and must be copied to the selected target pool before source cleanup. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Possibly modify: `crates/ecstore/src/data_movement.rs` +- Avoid unless necessary: `crates/ecstore/src/set_disk.rs` +- Test: existing rebalance unit tests in `crates/ecstore/src/rebalance.rs` + +### Design + +1. Add an ECStore-level helper for rebalance delete marker movement. + - Input: source pool index, bucket, object name, source `FileInfo`, version ID. + - Behavior: choose a target pool using the same placement rules as normal data movement, excluding source/rebalancing/decommissioned pools. + - Write the delete marker metadata to the target pool with the original version ID, mod time, delete marker flag, and replication state. + - Return success only when the target write succeeds or an equivalent target delete marker is confirmed. + +2. Change the delete marker branch in `migrate_entry_version_with_retry_wait`. + - Do not call `SetDisks::delete_object_for_migration` for rebalance delete markers. + - Call the ECStore-level helper via the `transfer`/backend abstraction or split delete marker handling out of the `SetDisks` backend path. + - Do not set `moved = true` unless the target helper succeeds. + +3. Change remote tiered branch. + - Short term: return `ignored = true`, `cleanup_ignored = false`, `moved = false` for remote tiered versions so they do not contribute to full source cleanup. + - If skipping a remote tiered version prevents source cleanup, record a clear status reason so operators understand why the entry remains. + +4. Source cleanup must remain blocked if any version was skipped without cleanup permission. + +### Implementation Tasks + +- [ ] Write a failing test proving a rebalance delete marker currently does not require target-pool metadata before counting complete. +- [ ] Write a failing test proving remote tiered versions do not allow source cleanup under the short-term skip policy. +- [ ] Implement the delete marker target write helper. +- [ ] Replace the rebalance delete marker source-set branch with the target write helper. +- [ ] Replace the remote tiered branch with the skip-without-cleanup policy. +- [ ] Run focused rebalance tests. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore rebalance_delete_marker --lib +cargo test -p rustfs-ecstore remote_tier --lib +cargo test -p rustfs-ecstore rebalance_entry --lib +``` + +### Acceptance Criteria + +- Rebalance does not clean a source entry when a delete marker target write fails. +- Rebalance preserves versioned delete behavior after source cleanup. +- Remote tiered versions are either safely skipped without source cleanup or explicitly moved with verified target metadata. +- Existing non-delete-marker rebalance behavior remains unchanged. + +### Risks + +- The existing `MigrationBackend` trait is source-set oriented. If adapting it becomes invasive, prefer an ECStore-specific path in `rebalance_entry` rather than widening the trait for one branch. +- Skipping remote tiered versions may leave more source entries behind. That is safer than deleting unverified metadata and can be improved in a later PR. + +--- + +## F02: Decommission `DataMovementOverwriteErr` Cleanup Safety + +### Decision + +`DataMovementOverwriteErr` must not be cleanup-safe by default. It can count as complete only after explicit target equivalence verification. + +### Files + +- Modify: `crates/ecstore/src/pools.rs` +- Possibly modify: `crates/ecstore/src/data_movement.rs` +- Test: decommission tests in `crates/ecstore/src/pools.rs` + +### Design + +1. Update delete marker and remote tiered decommission error handling. + - Keep object-not-found and version-not-found handling as cleanup-safe only when existing semantics justify it. + - Remove `is_err_data_movement_overwrite(&err)` from branches that set `cleanup_ignored = true`. + +2. Add a narrow equivalence helper only if an existing target version can be inspected cheaply. + - Compare bucket, object, version ID, delete marker state, ETag where applicable, mod time, and key metadata. + - If comparison cannot be done reliably in the current code path, treat overwrite as failure for Phase 1. + +3. Preserve source entry on unsafe overwrite. + - Set `failure = true`. + - Include a clear error stage in logs/status. + - Do not increment the decommissioned count. + +### Implementation Tasks + +- [ ] Write a failing unit test for delete marker decommission where `DataMovementOverwriteErr` must not count complete. +- [ ] Write a failing unit test for remote tiered decommission where `DataMovementOverwriteErr` must not count complete. +- [ ] Remove `is_err_data_movement_overwrite` from cleanup-safe conditions in those branches. +- [ ] Add an explicit helper or comment documenting why overwrite is unsafe without equivalence. +- [ ] Run focused decommission tests. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore decommission --lib +cargo test -p rustfs-ecstore DataMovementOverwriteErr --lib +``` + +### Acceptance Criteria + +- `DataMovementOverwriteErr` does not set `cleanup_ignored = true` by default. +- Source cleanup does not run when overwrite equivalence is unknown. +- Logs make unsafe overwrite distinguishable from not-found cleanup-safe cases. + +### Risks + +- Tightening this behavior may cause more decommission retries/failures in clusters that previously masked unsafe state. That is intended; status must make the failure actionable. + +--- + +## F03: Decommission Pool Meta Reload Barrier + +### Decision + +A successful decommission start must mean all required peers have acknowledged the updated pool meta or the API must return an explicit failure/degraded result. + +### Files + +- Modify: `crates/ecstore/src/pools.rs` +- Modify: `crates/ecstore/src/notification_sys.rs` +- Modify: `rustfs/src/admin/handlers/pools.rs` +- Possibly modify: `crates/ecstore/src/rpc/peer_rest_client.rs` +- Possibly modify: `rustfs/src/storage/rpc/node_service.rs` + +### Design + +1. Change `start_decommission` propagation behavior. + - After `pool_meta.save`, call `reload_pool_meta`. + - If peer reload returns an aggregate error, return that error to the admin handler. + - Do not silently proceed. + +2. Decide rollback behavior. + - Preferred minimal Phase 1: fail the API and leave persisted `pool_meta` in decommission state only if rollback is unsafe or unavailable; status must show reload failure. + - Stronger option: persist a reverted pool meta before returning failure. This needs careful lock and persistence review and should not be attempted without tests. + +3. Ensure decommission workers do not spawn after reload failure. + - If the caller starts workers after `start_decommission` returns, returning `Err` is sufficient. + - If any background path can resume from persisted metadata after a failed reload, status must clearly show pending/degraded state. + +### Implementation Tasks + +- [ ] Write a failing test for `start_decommission` with peer reload failure. +- [ ] Change reload failure from `warn` to returned error. +- [ ] Update admin handler error mapping so clients do not receive success. +- [ ] Add status/log context for peer reload failure. +- [ ] Run focused pool/decommission tests. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore start_decommission --lib +cargo test -p rustfs-ecstore reload_pool_meta --lib +cargo test -p rustfs decommission --lib +``` + +### Acceptance Criteria + +- Admin start decommission does not return success when peer reload fails. +- Decommission workers do not start after reload failure through the admin path. +- Failure output includes enough peer context for operators. + +### Risks + +- Persisted pool meta may already be updated before reload failure is detected. Avoid adding rollback until the exact persistence safety is reviewed. Fail-closed reporting is the minimum safe fix. + +--- + +## F04: Store Init Recovery Ordering + +### Decision + +Store init must load and install pool meta before deciding whether rebalance can auto-start. + +### Files + +- Modify: `crates/ecstore/src/store/init.rs` +- Possibly modify: `crates/ecstore/src/rebalance.rs` +- Possibly modify: `crates/ecstore/src/pools.rs` + +### Design + +1. Reorder init sequence. + - Initialize boot time. + - Load pool meta. + - Validate and install pool meta. + - Resolve resumable decommission pools. + - Load rebalance meta. + - Start rebalance only if no decommission is active or resumable. + +2. Define conflict behavior. + - If active decommission and started rebalance metadata coexist, decommission wins for Phase 1. + - Rebalance should not start. + - Record or expose a clear deferred/conflict reason if existing status structures can carry it without broad schema changes. + +3. Preserve existing happy path. + - Clusters with only rebalance metadata should still auto-start rebalance. + - Clusters with only decommission metadata should still resume decommission after the existing delay. + +### Implementation Tasks + +- [ ] Write a failing init test for active decommission plus rebalance metadata. +- [ ] Move pool meta load/validate/install before `load_rebalance_meta` and `start_rebalance`. +- [ ] Add a helper such as `should_auto_start_rebalance_after_init`. +- [ ] Ensure decommission resume still uses the installed pool meta. +- [ ] Run focused init tests. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore init --lib +cargo test -p rustfs-ecstore rebalance_meta --lib +cargo test -p rustfs-ecstore decommission --lib +``` + +### Acceptance Criteria + +- Rebalance does not start when active decommission is loaded from pool meta. +- Rebalance still starts when no decommission is active. +- Decommission resume behavior is unchanged except that it no longer races with an already-started rebalance. + +### Risks + +- Init ordering can affect existing startup failure behavior. Keep error handling and stage names explicit so operators can identify which stage failed. + +--- + +## F05: Rebalance Distributed Start/Stop Semantics + +### Decision + +Admin start/stop must not report plain success for partial cluster state. If stop remains asynchronous, status must expose that distinction. + +### Files + +- Modify: `rustfs/src/admin/handlers/rebalance.rs` +- Modify: `rustfs/src/storage/rpc/node_service.rs` +- Modify: `crates/ecstore/src/notification_sys.rs` +- Modify: `crates/ecstore/src/rpc/peer_rest_client.rs` +- Possibly modify: `crates/ecstore/src/rebalance.rs` + +### Design + +1. Fix peer RPC stop first. + - `node_service::stop_rebalance` must return `success=false` or gRPC error when `store.stop_rebalance().await` fails. + - `notification_sys.stop_rebalance` must propagate aggregate peer failure instead of warning and returning `Ok`. + +2. Fix start propagation. + - Avoid returning success from peer `load_rebalance_meta(start=true)` before local start validation has completed. + - If fully synchronous worker startup is too invasive, split semantics: + - metadata loaded; + - start accepted; + - worker running observable through status. + - Admin start should fail when any required peer fails load/start acceptance. + +3. Define rollback/degraded semantics. + - Preferred Phase 1: return failure on propagation failure and attempt local stop if local rebalance was already started. + - If rollback fails, return a degraded error and expose status. + +4. Add status language for stop. + - If cancellation is requested but workers may still be running, status should not claim fully stopped. + - Minimal approach: add a `stopping` or equivalent field if existing response types allow it. + +### Implementation Tasks + +- [ ] Write a failing test for peer stop returning success despite local stop error. +- [ ] Update `node_service::stop_rebalance` to return failure details. +- [ ] Write a failing test for `notification_sys.stop_rebalance` aggregate peer failure. +- [ ] Update `notification_sys.stop_rebalance` to return aggregate error. +- [ ] Write a failing admin start propagation test. +- [ ] Update admin start to fail or rollback on propagation failure. +- [ ] Add status distinction for requested stop versus completed stop if response schema permits. +- [ ] Run focused admin/RPC/rebalance tests. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore stop_rebalance --lib +cargo test -p rustfs-ecstore load_rebalance_meta --lib +cargo test -p rustfs rebalance --lib +``` + +### Acceptance Criteria + +- Peer stop failure is visible to callers. +- Admin stop fails or reports degraded state on peer failure. +- Admin start fails or reports degraded state on peer load/start failure. +- Stop status does not mislead operators while workers are still winding down. + +### Risks + +- API response schema changes may affect clients. Prefer backward-compatible additions where possible, and keep error responses compatible with existing admin error handling. + +--- + +## Phase 1 Test Matrix + +Run these once the individual fix tests pass: + +```bash +cargo test -p rustfs-ecstore rebalance --lib +cargo test -p rustfs-ecstore decommission --lib +cargo test -p rustfs-ecstore pools --lib +cargo test -p rustfs rebalance --lib +cargo test -p rustfs pools --lib +cargo fmt --all --check +``` + +Before opening a PR with code changes: + +```bash +cargo fmt --all +cargo fmt --all --check +make pre-commit +``` + +After build-based verification, clean generated build artifacts to avoid unnecessary disk usage. + +## Suggested PR Split + +1. PR 1: F02 only. Smallest fail-closed fix. +2. PR 2: F04 only. Startup ordering and conflict tests. +3. PR 3: F03 only. Decommission peer reload barrier. +4. PR 4: F05 stop semantics first, then start semantics if review scope remains manageable. +5. PR 5: F01 delete marker fix and remote tiered skip policy. + +F01 is highest risk, but it may need the most design review. F02 and F04 are good first implementation candidates because they reduce safety risk with narrower code changes. + +## Open Questions Before Coding + +1. For F01 remote tiered versions, should Phase 1 match MinIO and skip them during rebalance, or should RustFS implement cross-pool remote metadata movement now? +2. For F03 reload failure, should the implementation attempt to rollback persisted pool meta, or fail closed and expose degraded metadata state? +3. For F05 start propagation, is a backward-compatible degraded error response acceptable, or must the API stay strictly compatible with current success/error shapes? diff --git a/docs/architecture/rebalance-decommission-phase2-data-movement-plan.md b/docs/architecture/rebalance-decommission-phase2-data-movement-plan.md new file mode 100644 index 000000000..0e4ccb0e8 --- /dev/null +++ b/docs/architecture/rebalance-decommission-phase2-data-movement-plan.md @@ -0,0 +1,363 @@ +# Rebalance and Decommission Phase 2 Data Movement Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make shared data movement safe for large objects, metadata-preserving, and resilient to overwrite and cleanup races. + +**Architecture:** Phase 2 keeps `crates/ecstore/src/data_movement.rs` as the shared migration core, but tightens its streaming, metadata, and equivalence behavior. Rebalance and decommission should share small helpers where their safety checks are identical, while keeping operation-specific policy decisions in `rebalance.rs` and `pools.rs`. + +**Tech Stack:** Rust, Tokio async readers, ECStore object APIs, multipart APIs, lifecycle evaluator, filemeta metadata types, existing crate-local unit tests. + +--- + +## Scope + +This plan covers: + +- F06: stream multipart data movement instead of buffering whole parts. +- F07: preserve and verify checksum, replication, version purge, and object-lock metadata. +- F08: align or explicitly define decommission lifecycle-expired cleanup behavior. +- F09: handle rebalance overwrite races with explicit target equivalence checks. +- F10: add final source cleanup verification before prefix deletion. + +This plan assumes Phase 1 either has landed or its semantics are stable enough to build on. + +## Shared Principles + +- Streaming must keep memory bounded by buffer size, not part size. +- Metadata equivalence should be explicit and testable. +- A version is complete only when copied, intentionally skipped by policy, or proven equivalent on target. +- Source cleanup must be the last step and must verify the source version set has not changed. +- Fail closed when equivalence cannot be proven. + +## F06: Stream Multipart Data Movement + +### Decision + +Multipart migration must not allocate a buffer equal to `part.size`. It should stream each part into `put_object_part`. + +### Files + +- Modify: `crates/ecstore/src/data_movement.rs` +- Possibly modify: `crates/ecstore/src/store/multipart.rs` +- Possibly modify: `crates/ecstore/src/store_api/readers.rs` +- Test: data movement tests in `crates/ecstore/src/data_movement.rs` + +### Design + +1. Replace per-part `Vec` allocation. + - Current behavior reads each part with `read_exact` into `chunk`. + - New behavior should create a bounded reader over the shared object stream for exactly `part.size` bytes. + +2. Preserve part boundaries. + - Each `put_object_part` call must receive a reader that ends exactly at the part boundary. + - If a part stream ends early, return a staged `read_part` error and abort the multipart upload. + +3. Preserve part index behavior. + - Continue decoding `part.index`. + - Preserve indexed reader behavior so erasure/indexed metadata remains compatible. + +4. Preserve abort behavior. + - `abort_multipart_upload` must still run if any part upload or complete call fails before completion is marked. + +### Implementation Tasks + +- [ ] Add a failing data movement test with a fake large multipart stream that panics or fails if code tries to allocate/read the full part into memory. +- [ ] Add a test proving part boundary reads consume exactly the configured part sizes. +- [ ] Introduce a bounded async reader helper for a single multipart part. +- [ ] Replace `vec![0u8; part.size]` and `read_exact` in multipart migration with the bounded reader helper. +- [ ] Ensure part upload still wraps data in `PutObjReader` with correct size, actual size, and index. +- [ ] Keep existing abort-on-error behavior and add a focused test for failure during streamed part upload. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore data_movement --lib +cargo test -p rustfs-ecstore multipart --lib +``` + +### Acceptance Criteria + +- Data movement memory is bounded and not proportional to multipart part size. +- Multipart part boundaries, sizes, ETags, and indexes remain correct. +- Failed streamed part upload aborts the temporary multipart upload. + +### Risks + +- Async reader composition can accidentally over-read into the next part. The part-boundary test is mandatory. +- If `put_object_part` requires a concrete reader type, introduce the smallest adapter needed rather than rewriting multipart internals. + +--- + +## F07: Preserve and Verify Full Data Movement Metadata + +### Decision + +Checksum preservation is a confirmed fix. Replication, version purge, and object-lock behavior must be tested first; production changes should follow only where tests reveal loss or mismatch. + +### Files + +- Modify: `crates/ecstore/src/data_movement.rs` +- Possibly modify: `crates/ecstore/src/store_api/types.rs` +- Possibly modify: `crates/ecstore/src/set_disk.rs` +- Possibly modify: `crates/filemeta/src/fileinfo.rs` +- Possibly modify: `crates/filemeta/src/replication.rs` +- Test: data movement tests in `crates/ecstore/src/data_movement.rs` + +### Design + +1. Define metadata equivalence for data movement. + - Required fields: version ID, ETag, size, actual size, mod time, user metadata, storage class, checksum, multipart checksum, replication state, version purge state, object-lock mode/date, legal hold. + - Use a helper local to data movement tests first. Promote to production only if F09/F10 need it. + +2. Preserve multipart checksum. + - When source part checksum exists, populate the corresponding `CompletePart` checksum fields. + - Preserve object-level multipart checksum metadata when complete metadata is written. + +3. Validate replication state preservation. + - Current code copies `user_defined` and `set_disk` reconstructs `replication_state_internal` from metadata. + - Add tests to prove this works for replication status and version purge status. + - If tests fail, extend `ObjectOptions` or write path metadata handling narrowly. + +4. Validate object lock preservation. + - Current code copies `user_defined`, which should carry object-lock headers. + - Add tests for retention mode/date and legal hold. + - If tests fail, fix the metadata copy path without changing normal user copy semantics. + +### Implementation Tasks + +- [ ] Add a metadata equivalence assertion helper in data movement tests. +- [ ] Add a failing multipart checksum preservation test. +- [ ] Populate `CompletePart` checksum fields from source part metadata. +- [ ] Add single-part checksum preservation test and fix object-level checksum metadata if needed. +- [ ] Add replication state and version purge state preservation tests. +- [ ] Add object-lock retention and legal hold preservation tests. +- [ ] Apply the minimal production fixes required by those tests. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore data_movement --lib +cargo test -p rustfs-ecstore checksum --lib +cargo test -p rustfs-ecstore replication --lib +cargo test -p rustfs-ecstore object_lock --lib +``` + +### Acceptance Criteria + +- Migrated single-part and multipart objects preserve checksum behavior. +- Migrated multipart parts preserve per-part checksum metadata where RustFS stores it. +- Replication state and version purge state are equivalent after migration. +- Object-lock retention and legal hold remain unchanged after migration. + +### Risks + +- Some metadata may be derived rather than stored directly. Tests should compare externally observable object info, not only internal fields. +- Avoid broad `ObjectOptions` expansion unless current metadata copy cannot preserve a required field. + +--- + +## F08: Lifecycle-Expired Version Cleanup Semantics + +### Decision + +RustFS should either align with MinIO by counting lifecycle-expired versions as decommission-complete, or explicitly document and expose retained expired source entries. The recommended behavior is to align with MinIO when lifecycle/object-lock/replication checks say the version is safe to expire. + +### Files + +- Modify: `crates/ecstore/src/pools.rs` +- Possibly modify: lifecycle evaluator under `crates/ecstore/src/bucket/lifecycle/` +- Test: decommission tests in `crates/ecstore/src/pools.rs` + +### Design + +1. Preserve existing lifecycle safety checks. + - Do not bypass object-lock or replication constraints. + - Keep `should_skip_lifecycle_for_data_movement` or equivalent evaluator as the source of truth. + +2. Count safe expired versions as complete for source cleanup. + - Replace `expired == 0 && decommissioned == total_versions` with semantics that allow `decommissioned + expired == total_versions` when expired versions are safe. + +3. Keep status honest. + - If a version is retained because expiration is not safe, status should show it as remaining, not completed. + +### Implementation Tasks + +- [ ] Write a failing test where one migrated version plus one lifecycle-expired version should allow source cleanup. +- [ ] Write a test where object-lock protected expired-looking version must not allow cleanup. +- [ ] Write a test where replication-pending version must not allow cleanup. +- [ ] Update source cleanup predicate to count safe expired versions. +- [ ] Update counters/status if existing fields distinguish expired from decommissioned. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore decommission --lib +cargo test -p rustfs-ecstore lifecycle --lib +cargo test -p rustfs-ecstore object_lock --lib +``` + +### Acceptance Criteria + +- Source entry is cleaned when every version is migrated or safely lifecycle-expired. +- Protected versions still block cleanup. +- Decommission status remains understandable for migrated, expired, and blocked versions. + +### Risks + +- Counting expired versions as complete can hide lifecycle evaluator bugs. Tests must include protected negative cases. + +--- + +## F09: Rebalance Overwrite Race Equivalence + +### Decision + +Unsafe `DataMovementOverwriteErr` remains a failure. Equivalent target overwrite can be counted complete only after explicit comparison. + +### Files + +- Modify: `crates/ecstore/src/data_movement.rs` +- Modify: `crates/ecstore/src/rebalance.rs` +- Possibly modify: `crates/ecstore/src/store/object.rs` +- Test: data movement and rebalance tests + +### Design + +1. Define target equivalence helper. + - Compare version ID, ETag, size, actual size, mod time, checksum, delete marker state, and selected user metadata. + - Reuse the test helper from F07 if appropriate, but production helper must be limited to fields needed for safety. + +2. Apply helper only to overwrite race cases. + - If `DataMovementOverwriteErr` occurs and target pool differs from source pool, inspect target object/version. + - If equivalent, count migration as complete. + - If not equivalent or cannot inspect target, return failure. + +3. Keep source-equals-target unsafe. + - Do not convert source-equals-target into success without independent target proof. + +### Implementation Tasks + +- [ ] Add a failing rebalance test where equivalent target version exists and overwrite should converge. +- [ ] Add a failing rebalance test where target version differs and overwrite must fail. +- [ ] Implement a minimal target equivalence helper. +- [ ] Wire the helper into data movement overwrite handling for rebalance. +- [ ] Keep decommission behavior aligned with F02. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore DataMovementOverwriteErr --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo test -p rustfs-ecstore data_movement --lib +``` + +### Acceptance Criteria + +- Equivalent overwrite does not block rebalance convergence. +- Non-equivalent overwrite does not clean source. +- Logs/status distinguish equivalent overwrite from unsafe overwrite. + +### Risks + +- Comparing too few fields can mark a corrupt target complete. Prefer strict comparison in Phase 2. +- Comparing too many volatile fields can prevent convergence. Use tests to calibrate. + +--- + +## F10: Source Cleanup Preflight Verification + +### Decision + +Before deleting a source prefix, rebalance and decommission should verify the source version set still matches the migrated or intentionally skipped set. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Modify: `crates/ecstore/src/pools.rs` +- Possibly modify: `crates/ecstore/src/set_disk.rs` +- Test: rebalance and decommission cleanup tests + +### Design + +1. Capture source version identity at scan time. + - Version identity should include object name, version ID, delete marker flag, and enough metadata to detect a new or changed version. + +2. Re-read before cleanup. + - Immediately before `delete_prefix`, read current source metadata. + - Compare current identity set to the expected cleanup set. + +3. Defer on mismatch. + - Rebalance should defer/retry the entry with a clear last error. + - Decommission should fail the entry or retry according to existing decommission retry policy. + +4. Keep not-found idempotent. + - If source entry is already gone, cleanup remains successful. + +### Implementation Tasks + +- [ ] Add helper to build a stable version identity set from `FileInfoVersions`. +- [ ] Add rebalance test where source metadata changes between migration and cleanup. +- [ ] Add decommission test where source metadata changes between migration and cleanup. +- [ ] Add cleanup preflight before rebalance source `delete_prefix`. +- [ ] Add cleanup preflight before decommission source `delete_prefix`. +- [ ] Return deferred/failure status with a clear reason on mismatch. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore rebalance_entry --lib +cargo test -p rustfs-ecstore decommission_entry --lib +cargo test -p rustfs-ecstore cleanup --lib +``` + +### Acceptance Criteria + +- Cleanup does not delete a source entry if the version set changed after migration started. +- Already-deleted source remains idempotent. +- Mismatch errors are visible in status or last-error fields. + +### Risks + +- Extra metadata reads can add I/O to hot migration paths. Keep this as a cleanup-only check. + +--- + +## Phase 2 Test Matrix + +Run after individual fix tests pass: + +```bash +cargo test -p rustfs-ecstore data_movement --lib +cargo test -p rustfs-ecstore multipart --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo test -p rustfs-ecstore decommission --lib +cargo test -p rustfs-ecstore lifecycle --lib +cargo fmt --all --check +``` + +Before PR: + +```bash +cargo fmt --all +cargo fmt --all --check +make pre-commit +``` + +Clean generated build artifacts after build-based verification. + +## Suggested PR Split + +1. PR 1: F06 streaming multipart migration. +2. PR 2: F07 checksum and metadata preservation tests/fixes. +3. PR 3: F08 lifecycle-expired cleanup semantics. +4. PR 4: F09 overwrite equivalence. +5. PR 5: F10 cleanup preflight verification. + +F06 and F07 may combine only if checksum preservation requires the new streaming reader shape. F09 should wait for F07 if it depends on metadata equivalence helpers. + +## Open Questions Before Coding + +1. Which metadata fields are considered mandatory for equivalence in Phase 2: strict internal fields or externally observable fields only? +2. Should F08 fully match MinIO cleanup behavior, or should RustFS keep expired source entries but expose them as residual state? +3. Should F10 apply to both rebalance and decommission in the same PR, or should rebalance land first as the higher-risk cleanup path? diff --git a/docs/architecture/rebalance-decommission-phase3-hardening-plan.md b/docs/architecture/rebalance-decommission-phase3-hardening-plan.md new file mode 100644 index 000000000..dda5ca2f3 --- /dev/null +++ b/docs/architecture/rebalance-decommission-phase3-hardening-plan.md @@ -0,0 +1,320 @@ +# Rebalance and Decommission Phase 3 Hardening Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Improve operator visibility, auditability, metadata robustness, and documented compatibility after the Phase 1 and Phase 2 safety fixes. + +**Architecture:** Phase 3 should not change core data movement semantics unless a hardening test exposes a safety bug. It adds bounded status data, structured logs, metadata validation tests, and an explicit decision around rebalance completion tolerance. + +**Tech Stack:** Rust, tracing structured logs, admin status DTOs, serde/rmp metadata decoding, existing guardrail scripts, crate-local unit tests. + +--- + +## Scope + +This plan covers: + +- F11: improve rebalance cleanup failure reporting. +- F12: add structured audit fields for rebalance/decommission operations. +- F13: harden persisted rebalance and pool metadata decoding. +- F14: decide and document rebalance completion threshold behavior. + +This plan assumes the safety semantics from Phase 1 and the data movement behavior from Phase 2 are stable. + +## Shared Principles + +- Hardening must not hide safety failures. +- Status fields should be bounded to avoid unbounded metadata growth. +- Logs must be structured, searchable, and free of secrets. +- Metadata compatibility must be explicit: strict where possible, legacy-compatible where necessary. + +## F11: Rebalance Cleanup Failure Reporting + +### Decision + +Rebalance cleanup failures should not be invisible completion details. Admin status should expose whether completion had cleanup warnings and provide bounded object-level detail. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Modify: `rustfs/src/admin/handlers/rebalance.rs` +- Possibly modify: admin response DTOs in the same module or related madmin types +- Test: rebalance status and metadata tests + +### Design + +1. Extend cleanup warning metadata. + - Keep existing count. + - Add a bounded list of recent cleanup failures, for example last 10 entries. + - Each entry should include bucket, object, message, and timestamp. + +2. Keep metadata bounded. + - Do not store every cleanup failure unboundedly in `rebalance.bin`. + - Use a ring-buffer style helper or truncate older entries. + +3. Make terminal state visible. + - If rebalance completes with cleanup warnings, status should show warning count and details. + - Avoid reporting a clean completion when source cleanup failed. + +### Implementation Tasks + +- [ ] Add a cleanup warning entry struct with bounded retention. +- [ ] Add metadata compatibility defaults for older `rebalance.bin` without the new list. +- [ ] Update `record_rebalance_cleanup_warning_in_meta` to append bounded entries and update count/last fields. +- [ ] Update admin status serialization to expose warning count and entries. +- [ ] Add tests for one warning, multiple warnings, and bounded truncation. +- [ ] Add legacy metadata decode test. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore cleanup_warning --lib +cargo test -p rustfs-ecstore rebalance_meta --lib +cargo test -p rustfs rebalance --lib +``` + +### Acceptance Criteria + +- Multiple cleanup failures are visible through count and bounded details. +- Legacy rebalance metadata still decodes. +- Status distinguishes clean completion from completion with cleanup warnings. + +### Risks + +- Adding status fields may affect clients if response schemas are strict. Prefer additive fields with defaults. + +--- + +## F12: Structured Audit Fields + +### Decision + +Rebalance and decommission admin operations should have consistent structured logs for success, rejection, propagation failure, and partial/degraded state. + +### Files + +- Modify: `rustfs/src/admin/handlers/rebalance.rs` +- Modify: `rustfs/src/admin/handlers/pools.rs` +- Modify: `crates/ecstore/src/notification_sys.rs` +- Modify: `crates/ecstore/src/rpc/peer_rest_client.rs` +- Possibly modify: `scripts/check_logging_guardrails.sh` or related guardrails if logging rules require updates + +### Design + +1. Define standard fields. + - `event` + - `component` + - `subsystem` + - `action` + - `result` + - `request_id` when available + - `actor` or masked access key when available + - `remote_addr` when available + - `pool_indices` or `rebalance_id` + - `peer` + - `error` + +2. Apply to admin handlers. + - Start, stop, cancel, status-affecting operations. + - Authorization failures should remain handled by existing auth paths but should have enough context if already logged. + +3. Apply to peer propagation. + - Peer success and failure should use stable event names. + - Partial failure must be warn/error, not info-only. + +4. Avoid secret leakage. + - Never log raw credentials, signatures, tokens, or full auth headers. + +### Implementation Tasks + +- [ ] Inventory existing rebalance/decommission log events and field names. +- [ ] Define a small local helper or convention for masked actor/request fields if one already exists. +- [ ] Update rebalance admin start/stop/status logs. +- [ ] Update decommission start/cancel/status logs. +- [ ] Update notification peer propagation failure logs. +- [ ] Add log-capture tests for success, partial failure, and rejected operation where feasible. +- [ ] Run logging guardrail script if the repository has one. + +### Focused Test Commands + +```bash +cargo test -p rustfs rebalance --lib +cargo test -p rustfs pools --lib +cargo test -p rustfs-ecstore notification --lib +scripts/check_logging_guardrails.sh +``` + +### Acceptance Criteria + +- Operators can answer who requested an operation, what resource it targeted, and which peers failed. +- Partial failures are searchable through stable structured fields. +- Logs contain no secrets. + +### Risks + +- Logging changes can be noisy. Keep high-volume per-object logs out of admin audit events. + +--- + +## F13: Persisted Metadata Decode Hardening + +### Decision + +Persisted rebalance and pool metadata should reject or surface unknown/corrupt fields where compatibility allows. Legacy compatibility must be tested explicitly. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Modify: `crates/ecstore/src/pools.rs` +- Possibly add fixtures under an existing test fixture directory if one exists +- Test: metadata decode tests in `rebalance.rs` and `pools.rs` + +### Design + +1. Identify persisted structs. + - Rebalance metadata and stats structs. + - Pool metadata and decommission info structs. + +2. Choose strictness per struct. + - Use strict unknown-field rejection where metadata is not expected to carry forward-compatible fields. + - For legacy-compatible structs, keep decode lenient but log or validate unknown/unsupported state where possible. + +3. Add state validation. + - Reject conflicting terminal states such as complete plus canceled plus running. + - Reject impossible counters or invalid pool indices where existing validation can catch them. + +4. Preserve legacy fixtures. + - Existing old metadata must still decode if compatibility is required. + +### Implementation Tasks + +- [ ] List all rebalance and pool metadata structs currently deserialized from persisted bytes. +- [ ] Add fixture tests for unknown fields, missing critical fields, and conflicting terminal states. +- [ ] Add strict serde attributes only where fixtures show compatibility is safe. +- [ ] Add post-decode validation helpers for state conflicts. +- [ ] Update load paths to surface validation errors with actionable messages. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore rebalance_meta --lib +cargo test -p rustfs-ecstore pool_meta --lib +cargo test -p rustfs-ecstore metadata --lib +``` + +### Acceptance Criteria + +- Unknown/corrupt metadata does not silently become an apparently valid operation state. +- Legacy metadata fixtures continue to decode through documented paths. +- Conflicting terminal state is rejected or quarantined. + +### Risks + +- Strict decode can block startup if old metadata contains benign extra fields. Start with tests and post-decode validation before broad strict attributes. + +--- + +## F14: Rebalance Completion Threshold Decision + +### Decision + +RustFS must explicitly choose whether to match MinIO's tolerance-based rebalance completion or keep strict completion. The recommended default is to match MinIO unless RustFS has a documented reason to move more data. + +### Files + +- Modify: `crates/ecstore/src/rebalance.rs` +- Possibly modify: admin docs if present +- Test: rebalance goal/completion tests in `rebalance.rs` + +### Design Options + +1. **Match MinIO tolerance.** + - Complete when pool free-space ratio is within the MinIO-like tolerance of the goal. + - Reduces movement and operational time. + - Best for compatibility. + +2. **Keep strict behavior and document it.** + - No code behavior change. + - Add tests proving strict behavior is intentional. + - Status/docs should avoid implying MinIO-compatible tolerance. + +3. **Configurable tolerance.** + - Most flexible but not recommended unless operators need it. + - Adds configuration and support burden. + +### Recommended Plan + +Start with option 1 if compatibility is the goal. If maintainers prefer strictness, implement option 2 with explicit tests and documentation. + +### Implementation Tasks + +- [ ] Add tests that capture current strict behavior around the completion goal. +- [ ] Add tests for MinIO-like tolerance behavior. +- [ ] Confirm maintainers choose MinIO-compatible or strict behavior. +- [ ] Implement the selected threshold behavior. +- [ ] Update status/docs comments so completion semantics are clear. + +### Focused Test Commands + +```bash +cargo test -p rustfs-ecstore rebalance_goal --lib +cargo test -p rustfs-ecstore check_if_rebalance_done --lib +cargo test -p rustfs-ecstore rebalance --lib +``` + +### Acceptance Criteria + +- Completion threshold behavior is covered by deterministic tests. +- Operators can understand whether RustFS matches MinIO tolerance. +- Empty-queue completion path remains valid. + +### Risks + +- Changing threshold may alter operational expectations for existing users. Document the change if behavior changes. + +--- + +## Phase 3 Test Matrix + +Run after individual fix tests pass: + +```bash +cargo test -p rustfs-ecstore rebalance_meta --lib +cargo test -p rustfs-ecstore pool_meta --lib +cargo test -p rustfs-ecstore rebalance --lib +cargo test -p rustfs rebalance --lib +cargo test -p rustfs pools --lib +cargo fmt --all --check +``` + +If logging guardrails are relevant: + +```bash +scripts/check_logging_guardrails.sh +``` + +Before PR: + +```bash +cargo fmt --all +cargo fmt --all --check +make pre-commit +``` + +Clean generated build artifacts after build-based verification. + +## Suggested PR Split + +1. PR 1: F11 cleanup warning status. +2. PR 2: F12 structured audit fields. +3. PR 3: F13 metadata decode hardening. +4. PR 4: F14 completion threshold decision and tests. + +F11 may depend on Phase 2 cleanup semantics if F10 changes retry behavior. F12 can be done anytime after Phase 1 defines distributed failure semantics. + +## Open Questions Before Coding + +1. What is the maximum number of cleanup warning entries to keep in persisted rebalance metadata? +2. Which actor identity is safe and useful to log for admin requests? +3. Should metadata unknown fields fail startup, warn and continue, or quarantine only the affected operation? +4. Should RustFS match MinIO's rebalance completion tolerance by default? diff --git a/docs/architecture/rebalance-decommission-post-remediation-review-plan.md b/docs/architecture/rebalance-decommission-post-remediation-review-plan.md new file mode 100644 index 000000000..ae8b94516 --- /dev/null +++ b/docs/architecture/rebalance-decommission-post-remediation-review-plan.md @@ -0,0 +1,132 @@ +# Rebalance and Decommission Post-Remediation Review + +> **Status:** Updated for branch `cxymds/rebalance-decommission-remediation`. +> +> **Scope:** Rebalance, decommission, shared data-movement helpers, source cleanup, +> delete-marker handling, lifecycle expiry, restart/cancel recovery, and CI coverage. +> +> **Conclusion:** The main code-level P1/P2 remediation set has been implemented on +> this branch. The remaining release gate is verification, not another known +> correctness rewrite: full CI must pass, `make pre-commit` must pass before the +> final PR state, and the data-movement e2e proof added to CI must complete in the +> GitHub runner environment. + +## Review Corrections + +The previous report needed two important corrections. + +First, the R28 decommission permit issue is real but was overstated. The worker +permit was released before entry migration work, so `Workers` did not account for +entry lifetime and `wk.wait()` could not prove entry work had drained. However, +the current listing path awaits entry callbacks inline and bucket processing is +bounded, so the old "unbounded object movement" wording was not supported by the +code. The fix remains correct because permit lifetime now matches entry lifetime +and prevents future spawn-based regressions. + +Second, SSE-C migration is related to raw data movement but has a distinct +failure mode. Before the raw migration path, SSE-C objects did not silently +corrupt through an empty `HeaderMap`; migration reads failed while resolving +SSE-C material because the customer SSE-C headers were missing. The raw internal +read path avoids requiring request customer headers for data movement. + +## Implemented Remediation + +| Area | Current Branch State | Representative Guards | +| --- | --- | --- | +| Raw migration reads | Rebalance and decommission use raw data-movement read options instead of normal transformed GET semantics. | `test_rebalance_object_migration_read_opts_are_raw_data_movement`, `test_decommission_object_migration_read_opts_are_raw_data_movement` | +| Raw write invariants | Data-movement writes preserve source version, ETag, tags, expires, part metadata, and checksums consistently. Single-part raw ETag validation no longer rejects preserved source ETags. | `test_data_movement_put_object_opts_preserves_version_and_etag`, `test_data_movement_opts_preserve_tags_and_expires`, `test_data_movement_single_part_raw_reader_does_not_validate_source_etag` | +| Terminal decommission recovery | Cancel/failed/complete paths cancel in-memory workers before terminal persistence, and terminal-save failure no longer leaves ghost cancelers. | decommission canceler unit tests in `crates/ecstore/src/pools.rs` | +| Rebalance operation identity | Rebalance merge/save/stop paths are id-aware and do not let stale operation state clobber a newer operation. | rebalance id-gate and merge tests | +| Start coordination | Rebalance and decommission starts cross-check persisted peer operation state before starting conflicting work. | start cross-check tests | +| Decommission worker accounting | Entry permits are held through entry completion. R28 is no longer a pending worker-accounting bug. | decommission worker permit tests | +| Rebalance rollback | Failed start rollback finalizes metadata instead of leaving `Started` plus `stopping=true` without a worker. | rollback tests | +| Decommission progress save | Periodic progress-save errors are best-effort; terminal saves remain strict. | progress-save tests | +| Queue semantics | Multi-pool decommission is queued, local-leader prefix scheduling is supported, completed restart is rejected, completed queue prefixes can advance, failed/canceled entries block automatic promotion, failed/canceled retry preserves bucket progress, and promoted queued pools are canceled if cancellation arrives before work starts. | queue/promotion/retry tests in `pools.rs` | +| Admin status and query hardening | Pool admin status exposes queued/progress state, dangerous pool/rebalance mutation queries reject unknown or ambiguous parameters, and pool status `by-id` parsing no longer falls back to pool 0. | `admin_pool_list_item`, `pools_handler_tests`, `rebalance_handler_tests` | +| Cancel/clear operations | Non-leader cancel intent is accepted and failed/canceled terminal decommission can be explicitly cleared. | remote cancel and clear tests | +| Resume equivalence | Delete-marker replication state, tiered-object metadata, multipart part numbers, tags, expires, and version counts are covered more strictly. | data-movement equivalence tests | +| Lifecycle expiry | Data movement no longer treats failed lifecycle expiry application as a successful skip. | `resolve_data_movement_lifecycle_expiry_result_rejects_apply_failure` | +| CI coverage | Existing RustFS e2e delete-marker migration proof is wired into the `e2e-tests` CI job with `--test-threads=1`, reusing the downloaded debug binary. | `.github/workflows/ci.yml` | + +## Compatibility Decisions + +The current RustFS decommission contract is no longer single-pool only. Multi-pool +requests are supported as queued operations on multi-pool deployments. The +current behavior is documented in +`docs/architecture/decommission-compatibility.md`. + +Delete-marker behavior is intentionally characterized rather than guessed: + +- a lone delete marker without replication is cleanup-only metadata and can be + skipped; +- delete markers with replication metadata remain eligible for movement; +- rebalance and decommission share the same predicate. + +Lifecycle-expired versions are also explicit: + +- decommission can count safely expired versions toward source cleanup only when + lifecycle expiry application succeeds; +- rebalance remains stricter and requires actual data-movement completion for + cleanup. + +## Remaining Risks and Gates + +There are no remaining known P1/P2 code changes in this remediation set, but the +branch is not final-release-ready until verification completes: + +1. `make pre-commit` must pass before the PR is marked ready. +2. GitHub CI must pass, including the newly wired delete-marker migration e2e + proof. +3. Local full e2e verification was attempted but the machine ran out of disk + space while building `rustfs`; this is an environment blocker, not a test + assertion failure. The `target/` directory was cleaned afterward. +4. If product requires stronger end-to-end proof for encrypted and compressed + migration, add those scenarios to `crates/e2e_test` before release. Current + branch coverage is strongest at the internal option/invariant layer, with CI + e2e coverage focused on versioning/delete-marker semantics. + +## Focused Verification Run During Implementation + +Focused checks were run per task, including: + +```bash +cargo test -p rustfs-ecstore data_movement_single_part_raw_reader --lib +cargo test -p rustfs-ecstore test_merge_rebalance_meta_preserves_stopping_stop_snapshot --lib +cargo test -p rustfs-ecstore test_merge_rebalance_pool_stats_clears_stopping_for_terminal_status --lib +cargo test -p rustfs-ecstore first_resumable_decommission_queue_indices --lib +cargo test -p rustfs-ecstore test_return_resumable_pools_skips_failed_decommission --lib +cargo test -p rustfs-ecstore resolve_data_movement_lifecycle_expiry_result --lib +cargo test -p rustfs-ecstore lifecycle_action_removes_data_movement_version --lib +cargo test -p rustfs-ecstore test_pool_meta_promoted_queued_decommission_can_be_canceled --lib +cargo test -p rustfs admin_pool_list_item --lib +cargo test -p rustfs pools_handler_tests --lib +cargo test -p rustfs rebalance_handler_tests --lib +cargo test -p rustfs admin_query_pool_status_by_id --lib +cargo fmt --all --check +``` + +The attempted local e2e command was: + +```bash +cargo test -p e2e_test delete_marker_migration_semantics -- --nocapture +``` + +It failed because the local filesystem reached `No space left on device` while +building the RustFS debug binary. CI now runs the proof in the e2e job after the +debug binary artifact has been downloaded, and uses `--test-threads=1` to avoid +parallel test-server startup for that proof. + +## Final PR Gate + +Before marking the draft PR ready: + +```bash +cargo fmt --all +cargo fmt --all --check +make pre-commit +``` + +If CI reports failures after the branch is pushed, inspect the failing job logs +instead of weakening the e2e gate. The delete-marker migration proof was added +because skipping it would leave the exact versioning regression class invisible +to CI. diff --git a/docs/architecture/rebalance-decommission-remediation-plan.md b/docs/architecture/rebalance-decommission-remediation-plan.md new file mode 100644 index 000000000..4d60b041f --- /dev/null +++ b/docs/architecture/rebalance-decommission-remediation-plan.md @@ -0,0 +1,546 @@ +# Rebalance and Decommission Remediation Plan + +> This document turns `docs/architecture/expert-review-analysis.md` into an actionable remediation backlog. It is intentionally split into independent fix blocks so each item can be analyzed, assigned, implemented, and verified separately. + +## Scope + +This plan covers the confirmed and calibrated issues from the rebalance/decommission expert review: + +- Critical and high-risk data integrity issues. +- Distributed state propagation and recovery semantics. +- Data movement resource usage and metadata preservation. +- Operational visibility, auditability, and hardening gaps. + +This file is not a code patch. Each block below should be expanded into a focused implementation plan before code changes begin. + +## Priority Order + +| Order | Fix ID | Source | Priority | Main Risk | +| --- | --- | --- | --- | --- | +| 1 | F01 | P1.1 | Critical | Rebalance may drop delete marker or remote tiered metadata | +| 2 | F02 | P1.5 | High | Decommission may treat unsafe overwrite as cleanup-safe | +| 3 | F03 | P1.4 | High | Retiring pool may remain writable on stale peers | +| 4 | F04 | P1.2 | High | Rebalance and decommission may both resume after restart | +| 5 | F05 | P1.3, A4 | High | Admin start/stop may report success for partial cluster state | +| 6 | F06 | P2.1 | High | Multipart migration can allocate whole parts in memory | +| 7 | F07 | P2.2, A3 | Medium | Migrated objects may lose checksum or other metadata semantics | +| 8 | F08 | P2.4 | Medium | Decommission leaves lifecycle-expired source entries behind | +| 9 | F09 | P2.3 | Medium | Rebalance overwrite races may fail instead of converging | +| 10 | F10 | A2 | Medium | Source cleanup lacks a final version-set guard | +| 11 | F11 | A1 | Medium | Rebalance cleanup failures are completion warnings only | +| 12 | F12 | P3.2 | Low | Admin operations are not sufficiently auditable | +| 13 | F13 | P3.3 | Low | Persisted metadata accepts unknown or inconsistent fields | +| 14 | F14 | P3.4 | Low | Rebalance completion threshold differs from MinIO | + +## Phase 1: Data Integrity and Safety + +### F01: Fix rebalance delete marker and remote tiered version migration + +**Source:** P1.1 +**Decision:** Confirmed fix required. +**Target priority:** Critical. + +**Problem:** `rebalance_entry` marks delete markers and remote tiered versions as moved, but those branches operate through the source `SetDisks` path rather than a confirmed cross-pool target write. Source cleanup can then remove the only metadata that preserves tombstone or tiered-object semantics. + +**Primary files:** +- `crates/ecstore/src/rebalance.rs` +- `crates/ecstore/src/data_movement.rs` +- `crates/ecstore/src/set_disk.rs` +- Tests near existing rebalance migration tests in `crates/ecstore/src/rebalance.rs` + +**Preferred fix direction:** +- Route rebalance delete marker movement through an `ECStore`-level data movement path that chooses a non-source target pool and writes the exact version metadata to that target. +- For remote tiered versions, either: + - align with MinIO and skip tiered versions during rebalance, or + - implement a target-pool metadata move that preserves the remote tier pointer and lifecycle state. +- Do not count the version as rebalanced until the target metadata is confirmed. + +**Acceptance criteria:** +- A versioned object with a delete marker remains deleted after rebalance source cleanup. +- A remote tiered version remains readable or correctly listed after rebalance source cleanup. +- Rebalance does not mark delete marker or tiered versions as complete when target metadata was not written. + +**Required tests:** +- Multi-pool rebalance test for a versioned object with a delete marker. +- Multi-pool rebalance test for a remote tiered version. +- Failure test where target metadata write fails and source cleanup must not happen. + +**Dependencies:** None. This is the first safety fix. + +--- + +### F02: Stop treating `DataMovementOverwriteErr` as cleanup-safe in decommission + +**Source:** P1.5 +**Decision:** Confirmed fix required. +**Target priority:** High. + +**Problem:** Decommission treats `DataMovementOverwriteErr` like object-not-found/version-not-found and sets `cleanup_ignored = true`. That error only means source and destination pool are the same; it does not prove an equivalent target version exists. + +**Primary files:** +- `crates/ecstore/src/pools.rs` +- `crates/ecstore/src/data_movement.rs` +- `crates/ecstore/src/store/object.rs` + +**Preferred fix direction:** +- Remove `DataMovementOverwriteErr` from cleanup-safe branches for delete marker and remote tiered decommission paths. +- If overwrite occurs because an equivalent target version already exists, perform an explicit equivalence check before counting the version complete. +- Otherwise return a migration failure and keep the source entry. + +**Acceptance criteria:** +- `DataMovementOverwriteErr` does not increment the decommissioned count unless equivalence is proven. +- Source cleanup is blocked when target equivalence is unknown. +- Logs/status clearly state whether overwrite was equivalent-complete or unsafe. + +**Required tests:** +- Delete marker decommission returns `DataMovementOverwriteErr` and does not clean source. +- Remote tiered decommission returns `DataMovementOverwriteErr` and does not clean source. +- Equivalent target version exists; overwrite can be counted complete only after metadata equality check passes. + +**Dependencies:** F01 clarifies remote/tombstone movement semantics, but F02 can be implemented independently for decommission. + +--- + +### F03: Make decommission `pool_meta` reload a cluster-wide start barrier + +**Source:** P1.4 +**Decision:** Confirmed fix required. +**Target priority:** High. + +**Problem:** `start_decommission` saves pool meta and then calls peer reload. Reload failure is logged but the operation still returns success, leaving stale peers able to write into the retiring pool. + +**Primary files:** +- `crates/ecstore/src/pools.rs` +- `crates/ecstore/src/notification_sys.rs` +- `crates/ecstore/src/rpc/peer_rest_client.rs` +- `rustfs/src/admin/handlers/pools.rs` +- `rustfs/src/storage/rpc/node_service.rs` + +**Preferred fix direction:** +- Treat reload failure as a failed start unless the system has an explicit partial/degraded operation state. +- Return peer failure information to the admin handler. +- Do not spawn or resume decommission workers until required peers acknowledge the updated pool meta. + +**Acceptance criteria:** +- Admin decommission start fails or returns a clearly degraded response when peer reload fails. +- No peer can select the retiring pool for new writes after decommission start is reported successful. +- Status exposes peer reload failure details. + +**Required tests:** +- Unit test for `start_decommission` with mocked `reload_pool_meta` failure. +- RPC/admin test showing start does not return plain success on peer failure. +- Write-placement test verifying retiring pools are skipped only after successful cluster reload. + +**Dependencies:** None. + +--- + +### F04: Reorder store init recovery to restore pool meta before rebalance + +**Source:** P1.2 +**Decision:** Confirmed fix required. +**Target priority:** High. + +**Problem:** Store init loads and starts rebalance before loading persisted pool meta. If unfinished decommission exists on disk, `start_rebalance` can miss it and both processes may resume. + +**Primary files:** +- `crates/ecstore/src/store/init.rs` +- `crates/ecstore/src/rebalance.rs` +- `crates/ecstore/src/pools.rs` + +**Preferred fix direction:** +- Load and install `PoolMeta` before attempting to start rebalance. +- After pool meta is loaded, explicitly decide: + - if decommission is active, do not auto-start rebalance; + - if rebalance metadata exists but conflicts with decommission, mark rebalance as blocked/deferred and expose it in status. +- Keep existing decommission resume delay, but ensure rebalance is not already running when decommission resumes. + +**Acceptance criteria:** +- Restart with active decommission metadata and rebalance metadata does not run both workers. +- Startup status explains which operation was deferred and why. +- Existing startup without decommission still resumes rebalance normally. + +**Required tests:** +- Init test with persisted active decommission plus started rebalance metadata. +- Init test with only rebalance metadata still resumes rebalance. +- Init test with corrupt or missing pool meta follows existing error policy. + +**Dependencies:** None, but coordinate with F05 status semantics. + +--- + +### F05: Make rebalance start/stop distributed semantics explicit and fail-safe + +**Source:** P1.3, A4 +**Decision:** Confirmed fix required. +**Target priority:** High. + +**Problem:** Rebalance start can return success even when peer propagation fails or peer background start later fails. Stop can return success while peer stop failed or local stop errors were ignored. Stop is also asynchronous, but admin semantics do not clearly expose `stopping` versus `stopped`. + +**Primary files:** +- `rustfs/src/admin/handlers/rebalance.rs` +- `rustfs/src/storage/rpc/node_service.rs` +- `crates/ecstore/src/notification_sys.rs` +- `crates/ecstore/src/rpc/peer_rest_client.rs` +- `crates/ecstore/src/rebalance.rs` + +**Preferred fix direction:** +- For start: + - propagate rebalance metadata to peers before returning success; + - have peer RPC synchronously validate/start or return a structured failure; + - on partial failure, rollback local start or report a degraded state. +- For stop: + - never ignore `store.stop_rebalance()` errors; + - aggregate peer stop failures and return them to admin; + - expose `stop_requested`, `stopping`, and `stopped` in status if worker shutdown remains asynchronous. + +**Acceptance criteria:** +- Admin start does not return plain success when any required peer fails to load/start. +- Admin stop does not return plain success when any required peer fails to stop. +- Status can distinguish a requested stop from all workers fully stopped. + +**Required tests:** +- Peer `load_rebalance_meta(true)` failure. +- Peer background `start_rebalance` failure. +- Peer `stop_rebalance` local save failure. +- Stop requested while a worker is in a long migration operation. + +**Dependencies:** Coordinate with F04 so startup and admin semantics use the same conflict model. + +--- + +## Phase 2: Data Movement Correctness and Resource Control + +### F06: Stream multipart data movement instead of buffering whole parts + +**Source:** P2.1 +**Decision:** Confirmed fix required. +**Target priority:** High. + +**Problem:** Multipart migration allocates `Vec` for each part and reads the whole part into memory. Large parts plus concurrent workers can OOM the process. + +**Primary files:** +- `crates/ecstore/src/data_movement.rs` +- `crates/ecstore/src/store/multipart.rs` +- `crates/ecstore/src/store/object.rs` + +**Preferred fix direction:** +- Replace whole-part buffering with bounded streaming. +- Preserve the current part index and checksum/etag behavior through the streaming reader. +- Ensure multipart abort still runs on any failed part or complete call. + +**Acceptance criteria:** +- Migrating a large multipart object does not allocate memory proportional to part size. +- Part ETag, size, actual size, and index handling remain correct. +- Failed migration aborts the temporary multipart upload. + +**Required tests:** +- Fake large multipart reader proving memory is bounded. +- Multipart migration preserves part order and ETags. +- Failure during part upload aborts the multipart upload. + +**Dependencies:** Coordinate with F07 for checksum preservation. + +--- + +### F07: Preserve and verify full data movement metadata + +**Source:** P2.2, A3 +**Decision:** Confirmed fix required for checksum; additional metadata requires verification and likely fixes. +**Target priority:** Medium. + +**Problem:** Data movement preserves basic metadata but does not fully prove checksum, multipart checksum, replication state, version purge status, or object-lock metadata equivalence after migration. + +**Primary files:** +- `crates/ecstore/src/data_movement.rs` +- `crates/ecstore/src/store_api/types.rs` +- `crates/ecstore/src/set_disk.rs` +- `crates/filemeta/src/fileinfo.rs` +- `crates/filemeta/src/replication.rs` + +**Preferred fix direction:** +- For multipart migration, populate `CompletePart` checksum fields when source part checksum exists. +- Preserve or reconstruct object-level checksum metadata. +- Add migration equivalence checks for: + - ETag; + - checksum and multipart checksum; + - replication status and version purge status; + - object lock retention mode/date and legal hold; + - version ID and mod time. +- Only change production metadata handling after tests show a real loss path. + +**Acceptance criteria:** +- Migrated object metadata matches source for the fields above. +- `GetObjectAttributes` checksum behavior remains correct after migration. +- Object lock retention/legal hold remains unchanged across migration. + +**Required tests:** +- Single-part object with checksum metadata. +- Multipart object with per-part checksum metadata. +- Object with replication state and version purge state. +- Object with governance/compliance retention and legal hold. + +**Dependencies:** F06 should land first or in the same PR if checksum streaming needs shared reader changes. + +--- + +### F08: Align decommission lifecycle-expired version cleanup semantics + +**Source:** P2.4 +**Decision:** Confirmed fix required or explicit design decision required. +**Target priority:** Medium. + +**Problem:** Decommission skips lifecycle-expired versions but does not clean the source entry if any expired version exists. MinIO counts expired versions as completed and can clean the source entry. + +**Primary files:** +- `crates/ecstore/src/pools.rs` +- Lifecycle evaluator code under `crates/ecstore/src/bucket/lifecycle/` + +**Preferred fix direction:** +- Treat lifecycle-expired versions as completed for decommission source cleanup when they are not required to remain accessible. +- Keep lifecycle/object-lock/replication checks in place before treating a version as expired. +- If RustFS intentionally keeps expired source versions, expose that state in status and exclude the pool from being considered fully clean. + +**Acceptance criteria:** +- Decommission can clean a source entry when all remaining versions are either migrated or lifecycle-expired. +- Object-lock or replication-protected versions are not wrongly treated as cleanup-safe. +- Status accurately reflects residual expired-source entries if they are intentionally retained. + +**Required tests:** +- Bucket with one migrated version plus one lifecycle-expired version. +- Expired version protected by object lock must not be cleaned. +- Replication-pending version must not be treated as cleanup-safe. + +**Dependencies:** Coordinate with F07 metadata tests for object lock and replication. + +--- + +### F09: Handle rebalance overwrite races using explicit equivalence checks + +**Source:** P2.3 +**Decision:** Fix likely required; first confirm exact race cases with tests. +**Target priority:** Medium. + +**Problem:** RustFS treats `DataMovementOverwriteErr` as non-transient during rebalance, while MinIO ignores acceptable overwrite races. A strict failure can prevent rebalance convergence when the target already has an equivalent version. + +**Primary files:** +- `crates/ecstore/src/rebalance.rs` +- `crates/ecstore/src/data_movement.rs` +- `crates/ecstore/src/store/object.rs` + +**Preferred fix direction:** +- Keep unsafe source-equals-target cases as failures. +- If the target version already exists, compare version ID, ETag, size, mod time, checksum, and key metadata. +- Count as complete only when equivalence passes. +- Add last-error/status text when overwrite is unsafe. + +**Acceptance criteria:** +- Equivalent target version allows rebalance to continue. +- Non-equivalent target version fails and does not clean source. +- Behavior is documented as compatible with MinIO's acceptable overwrite race handling. + +**Required tests:** +- Rebalance overwrite with equivalent target version. +- Rebalance overwrite with mismatched checksum or metadata. +- Rebalance overwrite where source and target pool are the same and no equivalent target exists. + +**Dependencies:** F07 defines the metadata equivalence fields. + +--- + +### F10: Add source cleanup preflight verification before deleting migrated entries + +**Source:** A2 +**Decision:** Fix recommended as defense-in-depth. +**Target priority:** Medium. + +**Problem:** After versions are migrated, source cleanup deletes the source prefix based on the version list read before migration. A final verification would protect against state propagation failures, recovery races, or repair processes changing metadata before cleanup. + +**Primary files:** +- `crates/ecstore/src/rebalance.rs` +- `crates/ecstore/src/pools.rs` +- `crates/ecstore/src/set_disk.rs` + +**Preferred fix direction:** +- Before source cleanup, re-read source metadata or a generation marker. +- Verify the version set scheduled for deletion equals the version set that was migrated or intentionally ignored. +- If the version set changed, defer/retry the entry instead of deleting the prefix. + +**Acceptance criteria:** +- Cleanup does not delete versions created or discovered after the migration scan. +- Changed source metadata causes a retry/defer state with a clear status error. +- The guard applies to rebalance and decommission where source prefix deletion is used. + +**Required tests:** +- Simulate source metadata changing between migration and cleanup. +- Simulate peer state mismatch where a write lands on a rebalancing/decommissioning pool. +- Confirm unchanged metadata still allows cleanup. + +**Dependencies:** Works best after F03 and F05 reduce state propagation failures. + +--- + +## Phase 3: Cleanup Semantics, Observability, and Hardening + +### F11: Improve rebalance cleanup failure handling and reporting + +**Source:** A1 +**Decision:** Fix recommended. +**Target priority:** Medium. + +**Problem:** Rebalance source cleanup failure is converted to a warning and the task can still complete. Only a count and last warning are preserved. + +**Primary files:** +- `crates/ecstore/src/rebalance.rs` +- `rustfs/src/admin/handlers/rebalance.rs` +- Rebalance status DTOs and serialization paths + +**Preferred fix direction:** +- Preserve a bounded list of cleanup failures, not only the last one. +- Add a clear terminal or partial terminal state when cleanup warnings exist. +- Decide whether cleanup failure should: + - defer the entry for retry, or + - allow completion but mark the pool as `completed_with_cleanup_warnings`. + +**Acceptance criteria:** +- Admin status reports cleanup warning count and representative object list. +- Operators can identify which objects need manual cleanup or retry. +- Rebalance terminal state is not misleading when source data remains. + +**Required tests:** +- Multiple cleanup failures preserve count and bounded object details. +- Status response includes cleanup warning data. +- Completion state reflects cleanup warnings. + +**Dependencies:** Coordinate with F10 if cleanup failures become retryable. + +--- + +### F12: Add structured audit fields for rebalance and decommission operations + +**Source:** P3.2 +**Decision:** Fix recommended. +**Target priority:** Low. + +**Problem:** Critical admin operations do not consistently log actor, remote address, request ID, pool indices, peer host, result, and partial failure state. + +**Primary files:** +- `rustfs/src/admin/handlers/rebalance.rs` +- `rustfs/src/admin/handlers/pools.rs` +- `crates/ecstore/src/notification_sys.rs` +- `crates/ecstore/src/rpc/peer_rest_client.rs` +- Logging guardrail scripts if applicable + +**Preferred fix direction:** +- Standardize structured fields for start, stop, cancel, status-affecting propagation, and peer RPC failures. +- Mask sensitive actor material. +- Use warn/error for partial failure rather than info-only logs. + +**Acceptance criteria:** +- Start/stop/cancel logs include actor, request ID, pool/rebalance ID, peer, result, and error when present. +- Logs do not expose secrets. +- Partial failures are searchable by stable event fields. + +**Required tests:** +- Log-capture tests for success, rejection, and partial failure. +- Guardrail check if logging conventions are enforced by script. + +**Dependencies:** F03 and F05 define partial-failure semantics. + +--- + +### F13: Harden persisted rebalance and pool metadata decoding + +**Source:** P3.3 +**Decision:** Fix recommended after compatibility review. +**Target priority:** Low. + +**Problem:** Persisted rebalance and pool metadata can accept unknown fields silently. This may be acceptable for compatibility, but it weakens corruption and typo detection. + +**Primary files:** +- `crates/ecstore/src/rebalance.rs` +- `crates/ecstore/src/pools.rs` +- Metadata fixture tests near existing metadata tests + +**Preferred fix direction:** +- For formats that can be strict, add `deny_unknown_fields`. +- Where backward compatibility requires leniency, detect and warn on unknown fields during legacy decode. +- Add explicit validation for conflicting terminal states. + +**Acceptance criteria:** +- Unknown fields either fail decode or produce a warning through a documented compatibility path. +- Missing critical fields fail safely. +- Conflicting states such as completed plus running are rejected. + +**Required tests:** +- Unknown field fixture. +- Missing critical field fixture. +- Conflicting terminal state fixture. +- Legacy fixture proving backward compatibility. + +**Dependencies:** None. + +--- + +### F14: Decide and document rebalance completion threshold behavior + +**Source:** P3.4 +**Decision:** Design decision required. +**Target priority:** Low. + +**Problem:** RustFS uses a stricter rebalance completion goal than MinIO's tolerance-based behavior. This may increase movement work and operational time. + +**Primary files:** +- `crates/ecstore/src/rebalance.rs` +- Admin status documentation if present +- Tests around rebalance goal calculation + +**Preferred fix direction:** +- Decide whether RustFS should: + - match MinIO tolerance; + - keep strict behavior and document it; + - make tolerance configurable with a conservative default. +- Avoid configurability unless a real operational requirement exists. + +**Acceptance criteria:** +- Completion behavior is intentional and covered by tests. +- Status/ETA reporting does not imply MinIO-compatible tolerance if strict behavior remains. +- Large-pool behavior is covered by a deterministic unit test. + +**Required tests:** +- Rebalance goal just below target. +- Rebalance goal within MinIO-like tolerance. +- Empty queue completion path remains valid. + +**Dependencies:** None. + +--- + +## Cross-Cutting Test Matrix + +Each fix should add focused tests, but the following end-to-end scenarios should also be covered before enabling production use: + +1. Rebalance a versioned object with normal versions, delete marker, and source cleanup. +2. Rebalance a remote tiered version or verify it is intentionally skipped. +3. Decommission a pool while peer reload fails. +4. Restart with both active decommission metadata and rebalance metadata. +5. Stop rebalance while a large multipart migration is in progress. +6. Migrate a large multipart object without memory proportional to part size. +7. Migrate objects with checksum, replication state, retention, and legal hold metadata. +8. Trigger source cleanup failure and verify admin status exposes it. +9. Trigger overwrite race with equivalent and non-equivalent target versions. +10. Decommission lifecycle-expired versions with object-lock and replication constraints. + +## Suggested Execution Strategy + +1. Start with F01 and F02 because they protect object version semantics. +2. Implement F03, F04, and F05 before broader rollout because they define distributed operation safety. +3. Implement F06 before running large-scale data movement tests. +4. Implement F07 through F11 as compatibility, convergence, and observability hardening. +5. Keep F12 through F14 as final hardening unless release criteria require them earlier. + +Each fix should be developed in a separate PR unless two adjacent fixes share the same core implementation. In particular: + +- F01 and F09 should not be merged together unless the equivalence helper is clearly isolated. +- F06 and F07 may be combined only if streaming part migration and checksum preservation share the same reader changes. +- F03 and F05 may share peer failure aggregation utilities, but their admin behavior should be tested separately. diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 20cfcd14d..5a9fd6c8a 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -34,6 +34,7 @@ use rustfs_utils::path::path_join; use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE}; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; use std::path::PathBuf; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::sync::mpsc; @@ -64,21 +65,28 @@ fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) -> validate_heal_target(&hip.bucket, &hip.obj_prefix)?; if let Some(query) = uri.query() { - let params: Vec<&str> = query.split('&').collect(); - for param in params { - let mut parts = param.split('='); - if let Some(key) = parts.next() { - if key == "clientToken" - && let Some(value) = parts.next() - { - hip.client_token = value.to_string(); + let mut seen = HashSet::with_capacity(3); + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "clientToken" => { + if !seen.insert("clientToken") { + return Err(s3_error!(InvalidArgument, "duplicate heal query parameter")); + } + hip.client_token = value.into_owned(); } - if key == "forceStart" && parts.next().is_some() { - hip.force_start = true; + "forceStart" => { + if !seen.insert("forceStart") { + return Err(s3_error!(InvalidArgument, "duplicate heal query parameter")); + } + hip.force_start = parse_heal_query_bool(value.as_ref())?; } - if key == "forceStop" && parts.next().is_some() { - hip.force_stop = true; + "forceStop" => { + if !seen.insert("forceStop") { + return Err(s3_error!(InvalidArgument, "duplicate heal query parameter")); + } + hip.force_stop = parse_heal_query_bool(value.as_ref())?; } + _ => return Err(s3_error!(InvalidArgument, "unknown heal query parameter")), } } } @@ -109,6 +117,14 @@ fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) -> Ok(hip) } +fn parse_heal_query_bool(value: &str) -> S3Result { + match value { + "true" => Ok(true), + "false" => Ok(false), + _ => Err(s3_error!(InvalidArgument, "invalid heal query boolean")), + } +} + fn validate_heal_target(bucket: &str, obj_prefix: &str) -> S3Result<()> { if bucket.is_empty() && !obj_prefix.is_empty() { return Err(s3_error!(InvalidRequest, "invalid bucket name")); @@ -817,6 +833,28 @@ mod tests { assert!(parsed.force_stop); } + #[test] + fn test_extract_heal_init_params_rejects_unknown_duplicate_and_invalid_bool() { + let mut router = Router::new(); + router + .insert("/rustfs/admin/v3/heal/{bucket}", ()) + .expect("route should insert"); + + for query in [ + "forceStart=yes", + "forceStart=true&forceStart=false", + "clientToken=token&unexpected=true", + ] { + let uri: Uri = format!("/rustfs/admin/v3/heal/test-bucket?{query}") + .parse() + .expect("uri should parse"); + let matched = router.at("/rustfs/admin/v3/heal/test-bucket").expect("route should match"); + let err = extract_heal_init_params(&Bytes::new(), &uri, matched.params) + .expect_err("strict heal query should reject malformed input"); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + } + } + #[test] fn test_heal_channel_request_preserves_admin_heal_options() { let hip = HealInitParams { diff --git a/rustfs/src/admin/handlers/pools.rs b/rustfs/src/admin/handlers/pools.rs index 5e4cecd32..91d7206b3 100644 --- a/rustfs/src/admin/handlers/pools.rs +++ b/rustfs/src/admin/handlers/pools.rs @@ -12,12 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use http::{HeaderMap, StatusCode}; +use http::{HeaderMap, StatusCode, Uri}; use matchit::Params; use rustfs_policy::policy::action::{Action, AdminAction}; +use rustfs_utils::{ + MaskedAccessKey, + http::{AMZ_REQUEST_ID, REQUEST_ID_HEADER}, +}; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::Deserialize; -use serde_urlencoded::from_bytes; use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; @@ -38,10 +41,114 @@ use std::collections::HashSet; const LOG_COMPONENT_ADMIN_API: &str = "admin_api"; const LOG_SUBSYSTEM_POOL_ADMIN: &str = "pool_admin"; +const EVENT_ADMIN_REQUEST_STATE: &str = "admin_request_state"; const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected"; const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed"; const EVENT_ADMIN_RESPONSE_EMITTED: &str = "admin_response_emitted"; +fn admin_request_id(headers: &HeaderMap) -> Option<&str> { + headers + .get(REQUEST_ID_HEADER) + .or_else(|| headers.get(AMZ_REQUEST_ID)) + .and_then(|value| value.to_str().ok()) +} + +fn admin_remote_addr(req: &S3Request) -> Option { + req.extensions + .get::>() + .and_then(|opt| opt.map(|addr| addr.0.to_string())) +} + +fn log_pool_request_rejected_with_context(operation: &str, reason: &str, request_id: &str, actor: &str, remote_addr: &str) { + warn!( + event = EVENT_ADMIN_REQUEST_REJECTED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation, + action = operation, + result = "rejected", + reason, + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + "admin request rejected" + ); +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct PoolAuditContext<'a> { + request_id: &'a str, + actor: &'a str, + remote_addr: &'a str, +} + +impl<'a> PoolAuditContext<'a> { + fn new(request_id: &'a str, actor: &'a str, remote_addr: &'a str) -> Self { + Self { + request_id, + actor, + remote_addr, + } + } +} + +fn log_pool_request_rejected_with_audit(operation: &str, reason: &str, audit: PoolAuditContext<'_>) { + warn!( + event = EVENT_ADMIN_REQUEST_REJECTED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation, + action = operation, + result = "rejected", + reason, + request_id = %audit.request_id, + actor = %audit.actor, + remote_addr = %audit.remote_addr, + "admin request rejected" + ); +} + +fn log_pool_request_rejected_with_pool_audit(operation: &str, reason: &str, pool: &str, audit: PoolAuditContext<'_>) { + warn!( + event = EVENT_ADMIN_REQUEST_REJECTED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation, + action = operation, + result = "rejected", + reason, + request_id = %audit.request_id, + actor = %audit.actor, + remote_addr = %audit.remote_addr, + pool, + "admin request rejected" + ); +} + +fn log_pool_request_rejected_with_index_audit( + operation: &str, + reason: &str, + idx: usize, + pool_count: usize, + audit: PoolAuditContext<'_>, +) { + warn!( + event = EVENT_ADMIN_REQUEST_REJECTED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation, + action = operation, + result = "rejected", + reason, + request_id = %audit.request_id, + actor = %audit.actor, + remote_addr = %audit.remote_addr, + pool_index = idx, + pool_count, + "admin request rejected" + ); +} + macro_rules! log_pool_request_rejected { ($operation:expr, $reason:expr) => { warn!( @@ -49,6 +156,7 @@ macro_rules! log_pool_request_rejected { component = LOG_COMPONENT_ADMIN_API, subsystem = LOG_SUBSYSTEM_POOL_ADMIN, operation = $operation, + action = $operation, result = "rejected", reason = $reason, "admin request rejected" @@ -56,21 +164,6 @@ macro_rules! log_pool_request_rejected { }; } -macro_rules! log_pool_request_rejected_with_pool { - ($operation:expr, $reason:expr, $pool:expr) => { - warn!( - event = EVENT_ADMIN_REQUEST_REJECTED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_POOL_ADMIN, - operation = $operation, - result = "rejected", - reason = $reason, - pool = $pool, - "admin request rejected" - ); - }; -} - macro_rules! log_pool_request_failed { ($operation:expr, $reason:expr, $err:expr) => { error!( @@ -78,6 +171,7 @@ macro_rules! log_pool_request_failed { component = LOG_COMPONENT_ADMIN_API, subsystem = LOG_SUBSYSTEM_POOL_ADMIN, operation = $operation, + action = $operation, result = "failed", reason = $reason, error = %$err, @@ -89,10 +183,11 @@ macro_rules! log_pool_request_failed { macro_rules! log_pool_response_emitted { ($operation:expr) => { info!( - event = EVENT_ADMIN_RESPONSE_EMITTED, + event = EVENT_ADMIN_REQUEST_STATE, component = LOG_COMPONENT_ADMIN_API, subsystem = LOG_SUBSYSTEM_POOL_ADMIN, operation = $operation, + action = $operation, result = "success", "admin response emitted" ); @@ -130,11 +225,20 @@ fn contextualize_admin_pool_api_error( } } -fn decommission_admin_not_initialized_error(operation: &str) -> S3Error { - log_pool_request_failed!( - operation_to_event(operation), - "object_layer_not_initialized", - "object layer not initialized" +fn decommission_admin_not_initialized_error_with_audit(operation: &str, audit: PoolAuditContext<'_>) -> S3Error { + error!( + event = EVENT_ADMIN_REQUEST_FAILED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation = operation_to_event(operation), + action = operation_to_event(operation), + result = "failed", + reason = "object_layer_not_initialized", + request_id = %audit.request_id, + actor = %audit.actor, + remote_addr = %audit.remote_addr, + error = "object layer not initialized", + "admin request failed" ); S3Error::with_message(S3ErrorCode::InternalError, format!("Failed to {operation}: object layer not initialized")) } @@ -144,36 +248,52 @@ fn pool_admin_missing_credentials_error(operation: &str) -> S3Error { S3Error::with_message(S3ErrorCode::InvalidRequest, format!("Failed to {operation}: missing credentials")) } +fn pool_admin_missing_credentials_error_with_request(operation: &str, request_id: &str, remote_addr: &str) -> S3Error { + warn!( + event = EVENT_ADMIN_REQUEST_REJECTED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation = operation_to_event(operation), + action = operation_to_event(operation), + result = "rejected", + reason = "missing_credentials", + request_id = %request_id, + remote_addr = %remote_addr, + "admin request rejected" + ); + S3Error::with_message(S3ErrorCode::InvalidRequest, format!("Failed to {operation}: missing credentials")) +} + fn pool_admin_query_parse_error(operation: &str) -> S3Error { log_pool_request_rejected!(operation_to_event(operation), "invalid_query_parameters"); S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid query parameters")) } -fn pool_admin_pool_parse_error(operation: &str, pool: &str) -> S3Error { - log_pool_request_rejected_with_pool!(operation_to_event(operation), "invalid_pool", pool); +fn pool_admin_query_parse_error_with_audit(operation: &str, audit: PoolAuditContext<'_>) -> S3Error { + log_pool_request_rejected_with_audit(operation_to_event(operation), "invalid_query_parameters", audit); + S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid query parameters")) +} + +fn pool_admin_pool_parse_error_with_audit(operation: &str, pool: &str, audit: PoolAuditContext<'_>) -> S3Error { + log_pool_request_rejected_with_pool_audit(operation_to_event(operation), "invalid_pool", pool, audit); S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Failed to {operation}: invalid pool `{pool}`")) } -fn pool_admin_pool_not_found_error(operation: &str, pool: &str) -> S3Error { - log_pool_request_rejected_with_pool!(operation_to_event(operation), "pool_not_found", pool); +fn pool_admin_pool_not_found_error_with_audit(operation: &str, pool: &str, audit: PoolAuditContext<'_>) -> S3Error { + log_pool_request_rejected_with_pool_audit(operation_to_event(operation), "pool_not_found", pool, audit); S3Error::with_message( S3ErrorCode::InvalidArgument, format!("Failed to {operation}: pool `{pool}` was not found"), ) } -fn pool_admin_pool_index_error(operation: &str, idx: usize, pool_count: usize) -> S3Error { - warn!( - event = EVENT_ADMIN_REQUEST_REJECTED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_POOL_ADMIN, - operation = operation_to_event(operation), - result = "rejected", - reason = "pool_index_out_of_range", - pool_index = idx, - pool_count, - "admin request rejected" - ); +fn pool_admin_pool_index_error_with_audit( + operation: &str, + idx: usize, + pool_count: usize, + audit: PoolAuditContext<'_>, +) -> S3Error { + log_pool_request_rejected_with_index_audit(operation_to_event(operation), "pool_index_out_of_range", idx, pool_count, audit); S3Error::with_message( S3ErrorCode::InvalidArgument, format!("Failed to {operation}: pool index {idx} is out of range for {pool_count} pools"), @@ -186,6 +306,7 @@ fn operation_to_event(operation: &str) -> &'static str { "load pool status" => "query_pool_status", "start decommission" => "start_decommission", "cancel decommission" => "cancel_decommission", + "clear decommission" => "clear_decommission", _ => "pool_admin", } } @@ -195,15 +316,14 @@ fn parse_pool_idx_by_id(pool: &str, endpoint_count: usize) -> Option { (idx < endpoint_count).then_some(idx) } -fn dedup_indices(indices: &[usize]) -> Vec { +fn has_duplicate_indices(indices: &[usize]) -> bool { let mut seen = HashSet::with_capacity(indices.len()); - let mut output = Vec::with_capacity(indices.len()); for idx in indices { - if seen.insert(*idx) { - output.push(*idx); + if !seen.insert(*idx) { + return true; } } - output + false } pub fn register_pool_route(r: &mut S3Router) -> std::io::Result<()> { @@ -231,6 +351,12 @@ pub fn register_pool_route(r: &mut S3Router) -> std::io::Result< AdminOperation(&CancelDecommission {}), )?; + r.insert( + Method::POST, + format!("{}{}", ADMIN_PREFIX, "/v3/pools/clear").as_str(), + AdminOperation(&ClearDecommission {}), + )?; + Ok(()) } @@ -285,6 +411,52 @@ pub struct StatusPoolQuery { pub by_id: String, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PoolQueryMode { + Status, + Mutation, +} + +fn parse_status_pool_query(uri: &Uri) -> Result { + parse_pool_query(uri, PoolQueryMode::Status) +} + +fn parse_mutation_pool_query(uri: &Uri) -> Result { + parse_pool_query(uri, PoolQueryMode::Mutation) +} + +fn parse_pool_query(uri: &Uri, mode: PoolQueryMode) -> Result { + let mut parsed = StatusPoolQuery::default(); + let mut seen = HashSet::with_capacity(2); + let Some(query) = uri.query() else { + return Ok(parsed); + }; + + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "pool" => { + if !seen.insert("pool") { + return Err(()); + } + parsed.pool = value.into_owned(); + } + "by-id" => { + if !seen.insert("by-id") { + return Err(()); + } + match value.as_ref() { + "true" | "false" => parsed.by_id = value.into_owned(), + _ => return Err(()), + } + } + _ if mode == PoolQueryMode::Status => {} + _ => return Err(()), + } + } + + Ok(parsed) +} + pub struct StatusPool {} #[async_trait::async_trait] @@ -312,15 +484,7 @@ impl Operation for StatusPool { ) .await?; - let query = { - if let Some(query) = req.uri.query() { - let input: StatusPoolQuery = - from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("load pool status"))?; - input - } else { - StatusPoolQuery::default() - } - }; + let query = parse_status_pool_query(&req.uri).map_err(|_| pool_admin_query_parse_error("load pool status"))?; let usecase = DefaultAdminUsecase::from_global(); let pools_status = usecase @@ -351,12 +515,31 @@ impl Operation for StartDecommission { // POST //pools/decommission?pool=http://server{1...4}/disk{1...4} #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string(); + let remote_addr = admin_remote_addr(&req).unwrap_or_default(); + info!( + event = EVENT_ADMIN_REQUEST_STATE, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation = "start_decommission", + action = "start_decommission", + state = "requested", + request_id = %request_id, + remote_addr = %remote_addr, + "admin pool request state" + ); + let Some(input_cred) = req.credentials else { - return Err(pool_admin_missing_credentials_error("start decommission")); + return Err(pool_admin_missing_credentials_error_with_request( + "start decommission", + &request_id, + &remote_addr, + )); }; let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + let actor = MaskedAccessKey(&input_cred.access_key).to_string(); validate_admin_request( &req.headers, @@ -367,32 +550,51 @@ impl Operation for StartDecommission { req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), ) .await?; + let audit = PoolAuditContext::new(&request_id, &actor, &remote_addr); let Some(endpoints) = endpoints_from_context() else { - log_pool_request_rejected!("start_decommission", "not_implemented"); + log_pool_request_rejected_with_context("start_decommission", "not_implemented", &request_id, &actor, &remote_addr); return Err(s3_error!(NotImplemented)); }; if endpoints.legacy() { - log_pool_request_rejected!("start_decommission", "legacy_endpoints_not_supported"); + log_pool_request_rejected_with_context( + "start_decommission", + "legacy_endpoints_not_supported", + &request_id, + &actor, + &remote_addr, + ); return Err(s3_error!(NotImplemented)); } let Some(store) = resolve_object_store_handle() else { - return Err(decommission_admin_not_initialized_error("start decommission")); + return Err(decommission_admin_not_initialized_error_with_audit("start decommission", audit)); }; - validate_start_decommission_guards(store.is_decommission_running().await, store.is_rebalance_started().await)?; + let decommission_running = store.is_decommission_running().await; + let rebalance_running = store.is_rebalance_started().await; + if decommission_running { + log_pool_request_rejected_with_context( + "start_decommission", + "decommission_already_running", + &request_id, + &actor, + &remote_addr, + ); + } else if rebalance_running { + log_pool_request_rejected_with_context( + "start_decommission", + "rebalance_in_progress", + &request_id, + &actor, + &remote_addr, + ); + } + validate_start_decommission_guards(decommission_running, rebalance_running)?; - let query = { - if let Some(query) = req.uri.query() { - let input: StatusPoolQuery = - from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("start decommission"))?; - input - } else { - StatusPoolQuery::default() - } - }; + let query = parse_mutation_pool_query(&req.uri) + .map_err(|_| pool_admin_query_parse_error_with_audit("start decommission", audit))?; let is_byid = query.by_id.as_str() == "true"; let pools: Vec<&str> = query.pool.split(",").collect(); @@ -404,33 +606,53 @@ impl Operation for StartDecommission { let idx = { if is_byid { parse_pool_idx_by_id(pool, endpoints.as_ref().len()) - .ok_or_else(|| pool_admin_pool_parse_error("start decommission", pool))? + .ok_or_else(|| pool_admin_pool_parse_error_with_audit("start decommission", pool, audit))? } else { let Some(idx) = endpoints.get_pool_idx(pool) else { - return Err(pool_admin_pool_parse_error("start decommission", pool)); + return Err(pool_admin_pool_parse_error_with_audit("start decommission", pool, audit)); }; idx } }; if idx >= store.pools.len() { - return Err(pool_admin_pool_index_error("start decommission", idx, store.pools.len())); + return Err(pool_admin_pool_index_error_with_audit( + "start decommission", + idx, + store.pools.len(), + audit, + )); } parsed_indices.push(idx); } - let pools_indices = dedup_indices(&parsed_indices); + if has_duplicate_indices(&parsed_indices) { + return Err(pool_admin_query_parse_error_with_audit("start decommission", audit)); + } + let pools_indices = parsed_indices; if !pools_indices.is_empty() { let pool_context = format!("pools {:?}", &pools_indices); store - .decommission(ctx.clone(), pools_indices) + .decommission(ctx.clone(), pools_indices.clone()) .await .map_err(ApiError::from) .map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?; } - log_pool_response_emitted!("start_decommission"); + info!( + event = EVENT_ADMIN_RESPONSE_EMITTED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation = "start_decommission", + action = "start_decommission", + result = "success", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + pool_indices = ?pools_indices, + "admin response emitted" + ); Ok(S3Response::new((StatusCode::OK, Body::default()))) } } @@ -442,12 +664,31 @@ impl Operation for CancelDecommission { // POST //pools/cancel?pool=http://server{1...4}/disk{1...4} #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string(); + let remote_addr = admin_remote_addr(&req).unwrap_or_default(); + info!( + event = EVENT_ADMIN_REQUEST_STATE, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation = "cancel_decommission", + action = "cancel_decommission", + state = "requested", + request_id = %request_id, + remote_addr = %remote_addr, + "admin pool request state" + ); + let Some(input_cred) = req.credentials else { - return Err(pool_admin_missing_credentials_error("cancel decommission")); + return Err(pool_admin_missing_credentials_error_with_request( + "cancel decommission", + &request_id, + &remote_addr, + )); }; let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + let actor = MaskedAccessKey(&input_cred.access_key).to_string(); validate_admin_request( &req.headers, @@ -458,26 +699,26 @@ impl Operation for CancelDecommission { req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), ) .await?; + let audit = PoolAuditContext::new(&request_id, &actor, &remote_addr); let Some(endpoints) = endpoints_from_context() else { - log_pool_request_rejected!("cancel_decommission", "not_implemented"); + log_pool_request_rejected_with_context("cancel_decommission", "not_implemented", &request_id, &actor, &remote_addr); return Err(s3_error!(NotImplemented)); }; if endpoints.legacy() { - log_pool_request_rejected!("cancel_decommission", "legacy_endpoints_not_supported"); + log_pool_request_rejected_with_context( + "cancel_decommission", + "legacy_endpoints_not_supported", + &request_id, + &actor, + &remote_addr, + ); return Err(s3_error!(NotImplemented)); } - let query = { - if let Some(query) = req.uri.query() { - let input: StatusPoolQuery = - from_bytes(query.as_bytes()).map_err(|_e| pool_admin_query_parse_error("cancel decommission"))?; - input - } else { - StatusPoolQuery::default() - } - }; + let query = parse_mutation_pool_query(&req.uri) + .map_err(|_| pool_admin_query_parse_error_with_audit("cancel decommission", audit))?; let is_byid = query.by_id.as_str() == "true"; @@ -490,11 +731,11 @@ impl Operation for CancelDecommission { }; let Some(idx) = has_idx else { - return Err(pool_admin_pool_not_found_error("cancel decommission", &query.pool)); + return Err(pool_admin_pool_not_found_error_with_audit("cancel decommission", &query.pool, audit)); }; let Some(store) = resolve_object_store_handle() else { - return Err(decommission_admin_not_initialized_error("cancel decommission")); + return Err(decommission_admin_not_initialized_error_with_audit("cancel decommission", audit)); }; store @@ -503,7 +744,124 @@ impl Operation for CancelDecommission { .map_err(ApiError::from) .map_err(|err| contextualize_admin_pool_api_error(err, "cancel decommission", format!("pool {idx}")))?; - log_pool_response_emitted!("cancel_decommission"); + info!( + event = EVENT_ADMIN_RESPONSE_EMITTED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation = "cancel_decommission", + action = "cancel_decommission", + result = "success", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + pool_index = idx, + "admin response emitted" + ); + Ok(S3Response::new((StatusCode::OK, Body::default()))) + } +} + +pub struct ClearDecommission {} + +#[async_trait::async_trait] +impl Operation for ClearDecommission { + // POST //pools/clear?pool=http://server{1...4}/disk{1...4} + // Clears failed/canceled decommission metadata only; already moved data is not rolled back. + #[tracing::instrument(skip_all)] + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string(); + let remote_addr = admin_remote_addr(&req).unwrap_or_default(); + info!( + event = EVENT_ADMIN_REQUEST_STATE, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation = "clear_decommission", + action = "clear_decommission", + state = "requested", + request_id = %request_id, + remote_addr = %remote_addr, + "admin pool request state" + ); + + let Some(input_cred) = req.credentials else { + return Err(pool_admin_missing_credentials_error_with_request( + "clear decommission", + &request_id, + &remote_addr, + )); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + let actor = MaskedAccessKey(&input_cred.access_key).to_string(); + + validate_admin_request( + &req.headers, + &cred, + owner, + false, + vec![Action::AdminAction(AdminAction::DecommissionAdminAction)], + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ) + .await?; + let audit = PoolAuditContext::new(&request_id, &actor, &remote_addr); + + let Some(endpoints) = endpoints_from_context() else { + log_pool_request_rejected_with_context("clear_decommission", "not_implemented", &request_id, &actor, &remote_addr); + return Err(s3_error!(NotImplemented)); + }; + + if endpoints.legacy() { + log_pool_request_rejected_with_context( + "clear_decommission", + "legacy_endpoints_not_supported", + &request_id, + &actor, + &remote_addr, + ); + return Err(s3_error!(NotImplemented)); + } + + let query = parse_mutation_pool_query(&req.uri) + .map_err(|_| pool_admin_query_parse_error_with_audit("clear decommission", audit))?; + + let is_byid = query.by_id.as_str() == "true"; + + let has_idx = { + if is_byid { + parse_pool_idx_by_id(&query.pool, endpoints.as_ref().len()) + } else { + endpoints.get_pool_idx(&query.pool) + } + }; + + let Some(idx) = has_idx else { + return Err(pool_admin_pool_not_found_error_with_audit("clear decommission", &query.pool, audit)); + }; + + let Some(store) = resolve_object_store_handle() else { + return Err(decommission_admin_not_initialized_error_with_audit("clear decommission", audit)); + }; + + store + .clear_decommission(idx) + .await + .map_err(ApiError::from) + .map_err(|err| contextualize_admin_pool_api_error(err, "clear decommission", format!("pool {idx}")))?; + + info!( + event = EVENT_ADMIN_RESPONSE_EMITTED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_POOL_ADMIN, + operation = "clear_decommission", + action = "clear_decommission", + result = "success", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + pool_index = idx, + "admin response emitted" + ); Ok(S3Response::new((StatusCode::OK, Body::default()))) } } @@ -511,9 +869,12 @@ impl Operation for CancelDecommission { #[cfg(test)] mod pools_handler_tests { use super::{ - contextualize_admin_pool_api_error, decommission_admin_not_initialized_error, dedup_indices, parse_pool_idx_by_id, - pool_admin_missing_credentials_error, pool_admin_pool_index_error, pool_admin_pool_not_found_error, - pool_admin_pool_parse_error, pool_admin_query_parse_error, validate_start_decommission_guards, + PoolAuditContext, contextualize_admin_pool_api_error, decommission_admin_not_initialized_error_with_audit, + has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id, parse_status_pool_query, + pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request, + pool_admin_pool_index_error_with_audit, pool_admin_pool_not_found_error_with_audit, + pool_admin_pool_parse_error_with_audit, pool_admin_query_parse_error, pool_admin_query_parse_error_with_audit, + validate_start_decommission_guards, }; #[test] @@ -526,6 +887,52 @@ mod pools_handler_tests { assert_eq!(parse_pool_idx_by_id("4", 4), None); } + #[test] + fn test_parse_status_pool_query_ignores_unknown_but_rejects_duplicate_and_invalid_bool() { + let unknown = "/rustfs/admin/v3/pools/status?pool=0&force=true" + .parse() + .expect("uri should parse"); + let query = parse_status_pool_query(&unknown).expect("status query should ignore unknown keys"); + assert_eq!(query.pool, "0"); + + let duplicate = "/rustfs/admin/v3/pools/status?pool=0&pool=1" + .parse() + .expect("uri should parse"); + assert!(parse_status_pool_query(&duplicate).is_err()); + + let invalid_bool = "/rustfs/admin/v3/pools/status?by-id=yes".parse().expect("uri should parse"); + assert!(parse_status_pool_query(&invalid_bool).is_err()); + } + + #[test] + fn test_parse_mutation_pool_query_rejects_unknown_duplicate_and_invalid_bool() { + let unknown = "/rustfs/admin/v3/pools/decommission?pool=0&force=true" + .parse() + .expect("uri should parse"); + assert!(parse_mutation_pool_query(&unknown).is_err()); + + let duplicate = "/rustfs/admin/v3/pools/decommission?pool=0&pool=1" + .parse() + .expect("uri should parse"); + assert!(parse_mutation_pool_query(&duplicate).is_err()); + + let invalid_bool = "/rustfs/admin/v3/pools/decommission?by-id=yes" + .parse() + .expect("uri should parse"); + assert!(parse_mutation_pool_query(&invalid_bool).is_err()); + } + + #[test] + fn test_parse_status_pool_query_accepts_expected_keys() { + let uri = "/rustfs/admin/v3/pools/status?pool=pool-a&by-id=true" + .parse() + .expect("uri should parse"); + let query = parse_status_pool_query(&uri).expect("valid query should parse"); + + assert_eq!(query.pool, "pool-a"); + assert_eq!(query.by_id, "true"); + } + #[test] fn test_parse_pool_idx_by_id_rejects_empty_pool_count() { assert_eq!(parse_pool_idx_by_id("0", 0), None); @@ -592,21 +999,14 @@ mod pools_handler_tests { } #[test] - fn test_decommission_admin_not_initialized_error_formats_start_context() { - let err = decommission_admin_not_initialized_error("start decommission"); + fn test_decommission_admin_not_initialized_error_with_audit_preserves_response_contract() { + let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000"); + let err = decommission_admin_not_initialized_error_with_audit("start decommission", audit); assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError); assert_eq!(err.message(), Some("Failed to start decommission: object layer not initialized")); } - #[test] - fn test_decommission_admin_not_initialized_error_formats_cancel_context() { - let err = decommission_admin_not_initialized_error("cancel decommission"); - - assert_eq!(err.code(), &s3s::S3ErrorCode::InternalError); - assert_eq!(err.message(), Some("Failed to cancel decommission: object layer not initialized")); - } - #[test] fn test_pool_admin_missing_credentials_error_formats_list_context() { let err = pool_admin_missing_credentials_error("list pools"); @@ -623,6 +1023,14 @@ mod pools_handler_tests { assert_eq!(err.message(), Some("Failed to start decommission: missing credentials")); } + #[test] + fn test_pool_admin_missing_credentials_error_with_request_preserves_response_contract() { + let err = pool_admin_missing_credentials_error_with_request("cancel decommission", "req-1", "127.0.0.1:9000"); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("Failed to cancel decommission: missing credentials")); + } + #[test] fn test_pool_admin_query_parse_error_formats_status_context() { let err = pool_admin_query_parse_error("load pool status"); @@ -632,16 +1040,36 @@ mod pools_handler_tests { } #[test] - fn test_pool_admin_pool_parse_error_formats_pool_context() { - let err = pool_admin_pool_parse_error("start decommission", "pool-x"); + fn test_pool_audit_context_keeps_request_actor_and_remote_addr() { + let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000"); + + assert_eq!(audit.request_id, "req-1"); + assert_eq!(audit.actor, "access-key"); + assert_eq!(audit.remote_addr, "127.0.0.1:9000"); + } + + #[test] + fn test_pool_admin_query_parse_error_with_audit_preserves_response_contract() { + let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000"); + let err = pool_admin_query_parse_error_with_audit("start decommission", audit); + + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument); + assert_eq!(err.message(), Some("Failed to start decommission: invalid query parameters")); + } + + #[test] + fn test_pool_admin_pool_parse_error_with_audit_preserves_response_contract() { + let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000"); + let err = pool_admin_pool_parse_error_with_audit("start decommission", "pool-x", audit); assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument); assert_eq!(err.message(), Some("Failed to start decommission: invalid pool `pool-x`")); } #[test] - fn test_pool_admin_pool_index_error_formats_range_context() { - let err = pool_admin_pool_index_error("start decommission", 4, 2); + fn test_pool_admin_pool_index_error_with_audit_preserves_response_contract() { + let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000"); + let err = pool_admin_pool_index_error_with_audit("start decommission", 4, 2, audit); assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument); assert_eq!( @@ -651,21 +1079,23 @@ mod pools_handler_tests { } #[test] - fn test_pool_admin_pool_not_found_error_formats_cancel_context() { - let err = pool_admin_pool_not_found_error("cancel decommission", "pool-x"); + fn test_pool_admin_pool_not_found_error_with_audit_preserves_response_contract() { + let audit = PoolAuditContext::new("req-1", "access-key", "127.0.0.1:9000"); + let err = pool_admin_pool_not_found_error_with_audit("cancel decommission", "pool-x", audit); assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument); assert_eq!(err.message(), Some("Failed to cancel decommission: pool `pool-x` was not found")); } #[test] - fn test_dedup_indices_removes_duplicates_preserving_order() { - assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]); + fn test_has_duplicate_indices_detects_duplicate_indices() { + assert!(has_duplicate_indices(&[0, 2, 1, 2, 3])); } #[test] - fn test_dedup_indices_handles_empty_input() { + fn test_has_duplicate_indices_allows_unique_and_empty_input() { let empty: Vec = Vec::new(); - assert!(dedup_indices(&empty).is_empty()); + assert!(!has_duplicate_indices(&empty)); + assert!(!has_duplicate_indices(&[0, 2, 1, 3])); } } diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index 58d1989a8..e04670488 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -13,7 +13,8 @@ // limitations under the License. use crate::admin::handlers::storage_compat::{ - DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, StorageError, get_global_notification_sys, + DiskStat, ECStore, NotificationSys, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStopPropagationRecord, + StorageError, decode_rebalance_stop_propagation_record, get_global_notification_sys, }; use crate::{ admin::{ @@ -24,25 +25,154 @@ use crate::{ auth::{check_key_valid, get_session_token}, server::{ADMIN_PREFIX, RemoteAddr}, }; -use http::{HeaderMap, HeaderValue, StatusCode}; +use http::{HeaderMap, HeaderValue, StatusCode, Uri}; use hyper::Method; use matchit::Params; use rustfs_policy::policy::action::{Action, AdminAction}; use rustfs_storage_api::{BucketOperations, BucketOptions, StorageAdminApi}; +use rustfs_utils::{ + MaskedAccessKey, + http::{AMZ_REQUEST_ID, REQUEST_ID_HEADER}, +}; use s3s::{ Body, S3Request, S3Response, S3Result, header::{CONTENT_LENGTH, CONTENT_TYPE}, s3_error, }; use serde::{Deserialize, Serialize}; -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use time::OffsetDateTime; -use tracing::info; +use tracing::{error, info, warn}; const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance"; const EVENT_ADMIN_REBALANCE_STATE: &str = "admin_rebalance_state"; +fn admin_request_id(headers: &HeaderMap) -> Option<&str> { + headers + .get(REQUEST_ID_HEADER) + .or_else(|| headers.get(AMZ_REQUEST_ID)) + .and_then(|value| value.to_str().ok()) +} + +fn admin_remote_addr(req: &S3Request) -> Option { + req.extensions + .get::>() + .and_then(|opt| opt.map(|addr| addr.0.to_string())) +} + +fn log_rebalance_request_rejected(action: &str, reason: &str, request_id: &str, actor: &str, remote_addr: &str) { + warn!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action, + result = "rejected", + reason, + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + "admin rebalance state" + ); +} + +fn rebalance_query_present(uri: &Uri) -> bool { + uri.query().is_some_and(|query| !query.is_empty()) +} + +fn rollback_result_label(result: &Result<(), String>) -> &'static str { + match result { + Ok(_) => "rollback_success", + Err(err) if err.contains("peer") => "rollback_partial", + Err(_) => "rollback_failed", + } +} + +fn rebalance_start_rollback_error(start_err: &str, rollback_result: &Result<(), String>) -> String { + match rollback_result { + Ok(_) => format!("failed to propagate rebalance start: {start_err}; rollback result: rollback_success"), + Err(err) => format!( + "failed to propagate rebalance start: {start_err}; rollback result: {}; rollback error: {err}", + rollback_result_label(rollback_result) + ), + } +} + +fn rebalance_rollback_stop_failure_message(rebalance_id: &str, failures: &[String]) -> String { + format!("cluster stop_rebalance rollback for {rebalance_id} partial: {}", failures.join("; ")) +} + +fn rebalance_rollback_terminal_reload_failure_message(rebalance_id: &str, failures: &[String]) -> String { + format!( + "cluster terminal rebalance reload rollback for {rebalance_id} partial: {}", + failures.join("; ") + ) +} + +fn rebalance_rollback_failure_message( + rebalance_id: &str, + stop_failures: &[String], + terminal_reload_failures: &[String], +) -> String { + let mut failures = Vec::new(); + if !stop_failures.is_empty() { + failures.push(rebalance_rollback_stop_failure_message(rebalance_id, stop_failures)); + } + if !terminal_reload_failures.is_empty() { + failures.push(rebalance_rollback_terminal_reload_failure_message(rebalance_id, terminal_reload_failures)); + } + failures.join("; ") +} + +async fn rollback_cluster_rebalance_start( + store: &Arc, + notification_sys: Option<&NotificationSys>, + rebalance_id: &str, +) -> Result<(), String> { + let stop_attempt_at = OffsetDateTime::now_utc(); + if let Some(notification_sys) = notification_sys { + let stop_failures = notification_sys + .stop_rebalance_failures(Some(rebalance_id)) + .await + .map_err(|err| format!("cluster stop_rebalance rollback for {rebalance_id} failed: {err}"))?; + let terminal_reload_attempt_at = OffsetDateTime::now_utc(); + let terminal_reload_failures = match notification_sys.load_rebalance_meta_failures(false).await { + Ok(failures) => failures, + Err(err) => vec![format!("terminal rebalance reload rollback for {rebalance_id} failed: {err}")], + }; + if !stop_failures.is_empty() || !terminal_reload_failures.is_empty() { + let record = RebalanceStopPropagationRecord { + stop_attempt_at: Some(stop_attempt_at), + stop_failures: stop_failures.clone(), + terminal_reload_attempt_at: Some(terminal_reload_attempt_at), + terminal_reload_failures: terminal_reload_failures.clone(), + }; + store.record_rebalance_stop_propagation(record).await.map_err(|err| { + format!( + "cluster rebalance rollback for {rebalance_id} partial; failed to persist stop propagation: {err}; {}", + rebalance_rollback_failure_message(rebalance_id, &stop_failures, &terminal_reload_failures) + ) + })?; + return Err(rebalance_rollback_failure_message( + rebalance_id, + &stop_failures, + &terminal_reload_failures, + )); + } + return Ok(()); + } + + store + .stop_rebalance_for_id(Some(rebalance_id)) + .await + .map_err(|err| format!("local stop_rebalance rollback for {rebalance_id} failed: {err}"))?; + store + .save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt) + .await + .map_err(|err| format!("local rollback stop metadata save for {rebalance_id} failed: {err}"))?; + Ok(()) +} + pub fn register_rebalance_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( Method::POST, @@ -96,6 +226,8 @@ pub struct RebalancePoolStatus { pub id: usize, // Pool index (zero-based) #[serde(rename = "status")] pub status: String, // Active if rebalance is running, empty otherwise + #[serde(rename = "stopping")] + pub stopping: bool, // Stop requested but worker terminal acknowledgement not yet persisted #[serde(rename = "used")] pub used: f64, // Fraction of used space in range 0.0..=1.0 #[serde(rename = "lastError")] @@ -106,6 +238,20 @@ pub struct RebalancePoolStatus { pub progress: Option, // None when rebalance is not running } +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RebalanceStopPropagationStatus { + #[serde(rename = "lastAttemptAt", with = "offsetdatetime_rfc3339")] + pub last_attempt_at: Option, + #[serde(rename = "failedPeers")] + pub failed_peers: Vec, + #[serde(rename = "terminalReloadAttemptAt", with = "offsetdatetime_rfc3339")] + pub terminal_reload_attempt_at: Option, + #[serde(rename = "terminalReloadFailedPeers")] + pub terminal_reload_failed_peers: Vec, + #[serde(rename = "pendingTerminalReload")] + pub pending_terminal_reload: bool, +} + #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct RebalanceAdminStatus { pub id: String, // Identifies the ongoing rebalance operation by a UUID @@ -113,6 +259,8 @@ pub struct RebalanceAdminStatus { pub pools: Vec, // Contains all pools, including inactive #[serde(rename = "stoppedAt", with = "offsetdatetime_rfc3339")] pub stopped_at: Option, // Optional timestamp when rebalance was stopped + #[serde(rename = "stopPropagation")] + pub stop_propagation: RebalanceStopPropagationStatus, } fn calculate_rebalance_progress( @@ -201,6 +349,7 @@ fn build_rebalance_pool_statuses( let mut status = RebalancePoolStatus { id: i, status: ps.info.status.to_string(), + stopping: ps.info.stopping, used: rebalance_pool_used(disk_stats, i), last_error: ps.info.last_error.clone(), cleanup_warnings: ps.cleanup_warnings.clone(), @@ -216,18 +365,58 @@ fn build_rebalance_pool_statuses( .collect() } +fn build_rebalance_stop_propagation_status(meta: &RebalanceMeta) -> RebalanceStopPropagationStatus { + let record = meta + .pool_stats + .iter() + .filter_map(|pool_stat| pool_stat.info.last_error.as_deref()) + .find_map(decode_rebalance_stop_propagation_record); + + if let Some(record) = record { + let last_attempt_at = record.stop_attempt_at.or(meta.stopped_at); + let terminal_reload_attempt_at = record.terminal_reload_attempt_at; + return RebalanceStopPropagationStatus { + pending_terminal_reload: last_attempt_at.is_some() && terminal_reload_attempt_at.is_none(), + last_attempt_at, + failed_peers: record.stop_failures, + terminal_reload_attempt_at, + terminal_reload_failed_peers: record.terminal_reload_failures, + }; + } + + RebalanceStopPropagationStatus { + last_attempt_at: meta.stopped_at, + pending_terminal_reload: false, + ..Default::default() + } +} + +fn build_rebalance_admin_status(now: OffsetDateTime, disk_stats: &[DiskStat], meta: &RebalanceMeta) -> RebalanceAdminStatus { + let stop_time = meta.stopped_at; + RebalanceAdminStatus { + id: meta.id.clone(), + stopped_at: meta.stopped_at, + pools: build_rebalance_pool_statuses(now, stop_time, meta.percent_free_goal, &meta.pool_stats, disk_stats), + stop_propagation: build_rebalance_stop_propagation_status(meta), + } +} + pub struct RebalanceStart {} #[async_trait::async_trait] impl Operation for RebalanceStart { #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string(); + let remote_addr = admin_remote_addr(&req).unwrap_or_default(); info!( event = EVENT_ADMIN_REBALANCE_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_REBALANCE, action = "start", state = "requested", + request_id = %request_id, + remote_addr = %remote_addr, "admin rebalance state" ); @@ -237,6 +426,7 @@ impl Operation for RebalanceStart { let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + let actor = MaskedAccessKey(&input_cred.access_key).to_string(); validate_admin_request( &req.headers, @@ -248,19 +438,27 @@ impl Operation for RebalanceStart { ) .await?; + if rebalance_query_present(&req.uri) { + log_rebalance_request_rejected("start", "invalid_query_parameters", &request_id, &actor, &remote_addr); + return Err(s3_error!(InvalidArgument, "rebalance start does not accept query parameters")); + } + let Some(store) = resolve_object_store_handle() else { return Err(s3_error!(InternalError, "object layer is not initialized")); }; if store.pools.len() == 1 { + log_rebalance_request_rejected("start", "single_pool_not_supported", &request_id, &actor, &remote_addr); return Err(s3_error!(NotImplemented)); } if store.is_decommission_running().await { + log_rebalance_request_rejected("start", "decommission_in_progress", &request_id, &actor, &remote_addr); return Err(s3_error!(InvalidRequest, "cannot start rebalance while decommission is in progress")); } if store.is_rebalance_conflicting_with_decommission().await { + log_rebalance_request_rejected("start", "rebalance_already_in_progress", &request_id, &actor, &remote_addr); return Err(s3_error!(OperationAborted, "rebalance is already in progress")); } @@ -271,24 +469,30 @@ impl Operation for RebalanceStart { let buckets: Vec = bucket_infos.into_iter().map(|bucket| bucket.name).collect(); - let id = match store.init_rebalance_meta(buckets).await { + let id = match store.init_and_start_rebalance(buckets).await { Ok(id) => id, + Err(StorageError::DecommissionAlreadyRunning) => { + log_rebalance_request_rejected("start", "decommission_in_progress", &request_id, &actor, &remote_addr); + return Err(s3_error!(InvalidRequest, "cannot start rebalance while decommission is in progress")); + } + Err(StorageError::RebalanceAlreadyRunning) => { + log_rebalance_request_rejected("start", "rebalance_already_in_progress", &request_id, &actor, &remote_addr); + return Err(s3_error!(OperationAborted, "rebalance is already in progress")); + } Err(e) => { - return Err(s3_error!(InternalError, "failed to initialize rebalance metadata: {}", e)); + return Err(s3_error!(InternalError, "failed to start rebalance: {}", e)); } }; - store - .start_rebalance() - .await - .map_err(|e| s3_error!(InternalError, "failed to start rebalance: {}", e))?; - info!( event = EVENT_ADMIN_REBALANCE_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_REBALANCE, action = "start", state = "started", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, rebalance_id = %id, "admin rebalance state" ); @@ -299,18 +503,65 @@ impl Operation for RebalanceStart { subsystem = LOG_SUBSYSTEM_REBALANCE, action = "start", state = "propagation_started", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + rebalance_id = %id, "admin rebalance state" ); if let Err(err) = notification_sys.load_rebalance_meta(true).await { - info!( + error!( event = EVENT_ADMIN_REBALANCE_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_REBALANCE, action = "start", result = "propagation_failed", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + rebalance_id = %id, error = %err, "admin rebalance state" ); + + let start_err = err.to_string(); + let rollback_result = rollback_cluster_rebalance_start(&store, Some(notification_sys), &id).await; + let rollback_label = rollback_result_label(&rollback_result); + match &rollback_result { + Ok(_) => info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "start", + result = rollback_label, + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + rebalance_id = %id, + propagation_error = %start_err, + "admin rebalance state" + ), + Err(rollback_err) => error!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "start", + result = rollback_label, + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + rebalance_id = %id, + propagation_error = %start_err, + rollback_error = %rollback_err, + "admin rebalance state" + ), + } + + return Err(s3_error!( + InternalError, + "{}", + rebalance_start_rollback_error(&start_err, &rollback_result) + )); } info!( event = EVENT_ADMIN_REBALANCE_STATE, @@ -318,6 +569,11 @@ impl Operation for RebalanceStart { subsystem = LOG_SUBSYSTEM_REBALANCE, action = "start", state = "propagation_completed", + result = "success", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + rebalance_id = %id, "admin rebalance state" ); } @@ -340,12 +596,16 @@ pub struct RebalanceStatus {} impl Operation for RebalanceStatus { #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string(); + let remote_addr = admin_remote_addr(&req).unwrap_or_default(); info!( event = EVENT_ADMIN_REBALANCE_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_REBALANCE, action = "status", state = "requested", + request_id = %request_id, + remote_addr = %remote_addr, "admin rebalance state" ); @@ -355,6 +615,7 @@ impl Operation for RebalanceStatus { let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + let actor = MaskedAccessKey(&input_cred.access_key).to_string(); validate_admin_request( &req.headers, @@ -383,6 +644,7 @@ impl Operation for RebalanceStatus { let mut meta = RebalanceMeta::new(); if let Err(err) = meta.load(first_pool).await { if err == StorageError::ConfigNotFound { + log_rebalance_request_rejected("status", "rebalance_not_started", &request_id, &actor, &remote_addr); return Err(s3_error!(NoSuchResource, "pool rebalance is not started")); } @@ -401,16 +663,25 @@ impl Operation for RebalanceStatus { disk_stats[disk.pool_index as usize].total_space += disk.total_space; } - let stop_time = meta.stopped_at; let now = OffsetDateTime::now_utc(); - let admin_status = RebalanceAdminStatus { - id: meta.id.clone(), - stopped_at: meta.stopped_at, - pools: build_rebalance_pool_statuses(now, stop_time, meta.percent_free_goal, &meta.pool_stats, &disk_stats), - }; + let admin_status = build_rebalance_admin_status(now, &disk_stats, &meta); let data = serde_json::to_string(&admin_status) .map_err(|e| s3_error!(InternalError, "failed to serialize rebalance status response: {}", e))?; + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "status", + result = "success", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + rebalance_id = %admin_status.id, + pool_count = admin_status.pools.len(), + cleanup_warning_count = admin_status.pools.iter().map(|pool| pool.cleanup_warnings.count).sum::(), + "admin rebalance state" + ); let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); @@ -425,12 +696,16 @@ pub struct RebalanceStop {} impl Operation for RebalanceStop { #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string(); + let remote_addr = admin_remote_addr(&req).unwrap_or_default(); info!( event = EVENT_ADMIN_REBALANCE_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_REBALANCE, action = "stop", state = "requested", + request_id = %request_id, + remote_addr = %remote_addr, "admin rebalance state" ); @@ -440,6 +715,7 @@ impl Operation for RebalanceStop { let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + let actor = MaskedAccessKey(&input_cred.access_key).to_string(); validate_admin_request( &req.headers, @@ -451,22 +727,37 @@ impl Operation for RebalanceStop { ) .await?; + if rebalance_query_present(&req.uri) { + log_rebalance_request_rejected("stop", "invalid_query_parameters", &request_id, &actor, &remote_addr); + return Err(s3_error!(InvalidArgument, "rebalance stop does not accept query parameters")); + } + let Some(store) = resolve_object_store_handle() else { return Err(s3_error!(InternalError, "object layer is not initialized")); }; + store + .load_rebalance_meta() + .await + .map_err(|e| s3_error!(InternalError, "failed to load rebalance metadata before stop: {}", e))?; + let expected_rebalance_id = store.current_rebalance_id().await; + if !store.is_rebalance_conflicting_with_decommission().await { + log_rebalance_request_rejected("stop", "rebalance_not_started", &request_id, &actor, &remote_addr); return Err(s3_error!(NoSuchResource, "pool rebalance is not started")); } - if let Some(notification_sys) = get_global_notification_sys() { - notification_sys - .stop_rebalance() + let notification_sys = get_global_notification_sys(); + let stop_attempt_at = OffsetDateTime::now_utc(); + let mut stop_failures = Vec::new(); + if let Some(notification_sys) = notification_sys { + stop_failures = notification_sys + .stop_rebalance_failures(expected_rebalance_id.as_deref()) .await .map_err(|e| s3_error!(InternalError, "failed to stop rebalance via notification system: {}", e))?; } else { store - .stop_rebalance() + .stop_rebalance_for_id(expected_rebalance_id.as_deref()) .await .map_err(|e| s3_error!(InternalError, "failed to stop rebalance: {}", e))?; @@ -482,36 +773,85 @@ impl Operation for RebalanceStop { subsystem = LOG_SUBSYSTEM_REBALANCE, action = "stop", state = "local_stop_persisted", + result = "success", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, "admin rebalance state" ); - if let Some(notification_sys) = get_global_notification_sys() { + + let mut terminal_reload_attempt_at = None; + let mut terminal_reload_failures = Vec::new(); + if let Some(notification_sys) = notification_sys { info!( event = EVENT_ADMIN_REBALANCE_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_REBALANCE, action = "stop", state = "propagation_started", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, "admin rebalance state" ); - if let Err(err) = notification_sys.load_rebalance_meta(false).await { - info!( - event = EVENT_ADMIN_REBALANCE_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_REBALANCE, - action = "stop", - result = "propagation_failed", - error = %err, - "admin rebalance state" - ); + terminal_reload_attempt_at = Some(OffsetDateTime::now_utc()); + match notification_sys.load_rebalance_meta_failures(false).await { + Ok(failures) => { + terminal_reload_failures = failures; + if terminal_reload_failures.is_empty() { + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "stop", + state = "propagation_completed", + result = "success", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + "admin rebalance state" + ); + } + } + Err(err) => { + terminal_reload_failures.push(format!("terminal rebalance reload propagation failed: {err}")); + } } - info!( + } + + if !stop_failures.is_empty() || !terminal_reload_failures.is_empty() { + let record = RebalanceStopPropagationRecord { + stop_attempt_at: Some(stop_attempt_at), + stop_failures: stop_failures.clone(), + terminal_reload_attempt_at, + terminal_reload_failures: terminal_reload_failures.clone(), + }; + store + .record_rebalance_stop_propagation(record) + .await + .map_err(|e| s3_error!(InternalError, "failed to persist rebalance stop propagation metadata: {}", e))?; + + error!( event = EVENT_ADMIN_REBALANCE_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_REBALANCE, action = "stop", - state = "propagation_completed", + result = "propagation_failed", + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + stop_failure_count = stop_failures.len(), + terminal_reload_failure_count = terminal_reload_failures.len(), "admin rebalance state" ); + let mut failures = Vec::new(); + failures.extend(stop_failures); + failures.extend(terminal_reload_failures); + return Err(s3_error!( + InternalError, + "rebalance stop propagation incomplete after local stop was persisted: {}", + failures.join("; ") + )); } let mut header = HeaderMap::new(); @@ -558,11 +898,14 @@ mod rebalance_handler_tests { use super::build_rebalance_pool_progress; use super::calculate_rebalance_progress; use super::{ - RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used, - rebalance_remaining_buckets, rebalance_used_pct, + RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, RebalanceStopPropagationStatus, + build_rebalance_admin_status, build_rebalance_pool_statuses, build_rebalance_stop_propagation_status, + rebalance_pool_used, rebalance_query_present, rebalance_remaining_buckets, rebalance_rollback_failure_message, + rebalance_rollback_stop_failure_message, rebalance_start_rollback_error, rebalance_used_pct, rollback_result_label, }; use crate::admin::handlers::storage_compat::{ - DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats, + DiskStat, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, + RebalanceStats, RebalanceStopPropagationRecord, encode_rebalance_stop_propagation_record, }; use time::OffsetDateTime; @@ -577,6 +920,76 @@ mod rebalance_handler_tests { assert_eq!(eta, 50); } + #[test] + fn test_rebalance_start_rollback_error_reports_successful_rollback() { + let rollback_result = Ok(()); + let message = rebalance_start_rollback_error("peer a failed", &rollback_result); + + assert!(message.contains("failed to propagate rebalance start: peer a failed")); + assert!(message.contains("rollback result: rollback_success")); + } + + #[test] + fn test_rebalance_start_rollback_error_reports_partial_peer_rollback() { + let rollback_result = Err("peer b stop_rebalance failed: timeout".to_string()); + let message = rebalance_start_rollback_error("peer a failed", &rollback_result); + + assert_eq!(rollback_result_label(&rollback_result), "rollback_partial"); + assert!(message.contains("failed to propagate rebalance start: peer a failed")); + assert!(message.contains("rollback result: rollback_partial")); + assert!(message.contains("rollback error: peer b stop_rebalance failed: timeout")); + } + + #[test] + fn test_rebalance_rollback_stop_failure_message_lists_failures() { + let failures = vec![ + "peer a stop_rebalance failed: timeout".to_string(), + "peer b stop_rebalance failed: unavailable".to_string(), + ]; + + let message = rebalance_rollback_stop_failure_message("rebalance-id", &failures); + + assert!(message.contains("cluster stop_rebalance rollback for rebalance-id partial")); + assert!(message.contains("peer a stop_rebalance failed: timeout")); + assert!(message.contains("peer b stop_rebalance failed: unavailable")); + } + + #[test] + fn test_rebalance_rollback_failure_message_lists_stop_and_terminal_reload_failures() { + let stop_failures = vec!["peer a stop_rebalance failed: timeout".to_string()]; + let terminal_reload_failures = vec!["peer b load_rebalance_meta(start=false) failed: unavailable".to_string()]; + + let message = rebalance_rollback_failure_message("rebalance-id", &stop_failures, &terminal_reload_failures); + + assert!(message.contains("cluster stop_rebalance rollback for rebalance-id partial")); + assert!(message.contains("peer a stop_rebalance failed: timeout")); + assert!(message.contains("cluster terminal rebalance reload rollback for rebalance-id partial")); + assert!(message.contains("peer b load_rebalance_meta(start=false) failed: unavailable")); + } + + #[test] + fn test_rebalance_query_present_detects_non_empty_query() { + let uri = "/rustfs/admin/v3/rebalance/start?pool=0".parse().unwrap(); + + assert!(rebalance_query_present(&uri)); + } + + #[test] + fn test_rebalance_query_present_allows_no_or_empty_query() { + let no_query = "/rustfs/admin/v3/rebalance/start".parse().unwrap(); + let empty_query = "/rustfs/admin/v3/rebalance/start?".parse().unwrap(); + + assert!(!rebalance_query_present(&no_query)); + assert!(!rebalance_query_present(&empty_query)); + } + + #[test] + fn test_rollback_result_label_reports_non_peer_failure_as_failed() { + let rollback_result = Err("local stop_rebalance failed: disk error".to_string()); + + assert_eq!(rollback_result_label(&rollback_result), "rollback_failed"); + } + #[test] fn test_calculate_rebalance_progress_stopped_by_end_time() { let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); @@ -861,6 +1274,26 @@ mod rebalance_handler_tests { assert!(statuses[1].progress.is_some()); } + #[test] + fn test_build_rebalance_pool_statuses_reports_stopping() { + let pool_stats = vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + stopping: true, + start_time: Some(OffsetDateTime::from_unix_timestamp(2_000).unwrap()), + ..Default::default() + }, + ..Default::default() + }]; + + let statuses = + build_rebalance_pool_statuses(OffsetDateTime::from_unix_timestamp(2_010).unwrap(), None, 0.3, &pool_stats, &[]); + + assert_eq!(statuses[0].status, "Started"); + assert!(statuses[0].stopping); + } + #[test] fn test_build_rebalance_pool_statuses_empty_inputs() { let statuses = build_rebalance_pool_statuses( @@ -882,9 +1315,11 @@ mod rebalance_handler_tests { let status = RebalanceAdminStatus { id: "id-1".to_string(), stopped_at: None, + stop_propagation: RebalanceStopPropagationStatus::default(), pools: vec![RebalancePoolStatus { id: 0, status: "Started".to_string(), + stopping: true, used: 0.5, last_error: Some("temporary error".to_string()), cleanup_warnings: RebalanceCleanupWarnings { @@ -893,6 +1328,12 @@ mod rebalance_handler_tests { last_bucket: Some("bucket-a".to_string()), last_object: Some("obj".to_string()), last_at: Some(OffsetDateTime::from_unix_timestamp(1_001).unwrap()), + entries: vec![RebalanceCleanupWarningEntry { + bucket: "bucket-a".to_string(), + object: "obj".to_string(), + message: "cleanup warning".to_string(), + timestamp: Some(OffsetDateTime::from_unix_timestamp(1_001).unwrap()), + }], }, progress: Some(RebalPoolProgress { num_objects: 3, @@ -911,8 +1352,100 @@ mod rebalance_handler_tests { assert!(json.contains("\"remainingBuckets\"")); assert!(json.contains("\"lastError\"")); assert!(json.contains("\"cleanupWarnings\"")); + assert!(json.contains("\"stopping\":true")); assert!(json.contains("\"lastMsg\":\"cleanup warning\"")); + assert!(json.contains("\"entries\"")); + assert!(json.contains("\"message\":\"cleanup warning\"")); assert!(json.contains("\"stoppedAt\":null")); + assert!(json.contains("\"stopPropagation\"")); + assert!(json.contains("\"pendingTerminalReload\":false")); + } + + #[test] + fn test_build_rebalance_admin_status_is_stable_for_same_persisted_meta() { + let started = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let disk_stats = vec![ + DiskStat { + total_space: 2_000, + available_space: 1_000, + }, + DiskStat { + total_space: 2_000, + available_space: 1_500, + }, + ]; + let meta = RebalanceMeta { + id: "rebalance-id".to_string(), + percent_free_goal: 0.6, + pool_stats: vec![ + RebalanceStats { + participating: true, + init_capacity: 2_000, + init_free_space: 500, + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + rebalanced_buckets: vec!["bucket-a".to_string()], + bucket: "bucket-b".to_string(), + object: "object.txt".to_string(), + num_objects: 10, + num_versions: 12, + bytes: 300, + info: RebalanceInfo { + status: RebalStatus::Started, + start_time: Some(started), + ..Default::default() + }, + ..Default::default() + }, + RebalanceStats { + participating: false, + info: RebalanceInfo { + status: RebalStatus::Completed, + ..Default::default() + }, + ..Default::default() + }, + ], + ..Default::default() + }; + + let first = build_rebalance_admin_status(OffsetDateTime::from_unix_timestamp(1_030).unwrap(), &disk_stats, &meta); + let second = build_rebalance_admin_status(OffsetDateTime::from_unix_timestamp(1_060).unwrap(), &disk_stats, &meta); + + assert_eq!(first.id, second.id); + assert_eq!(first.stopped_at, second.stopped_at); + assert_eq!(first.stop_propagation.failed_peers, second.stop_propagation.failed_peers); + assert_eq!(first.pools.len(), second.pools.len()); + for (left, right) in first.pools.iter().zip(second.pools.iter()) { + assert_eq!(left.id, right.id); + assert_eq!(left.status, right.status); + assert_eq!(left.stopping, right.stopping); + assert_eq!(left.used, right.used); + assert_eq!(left.last_error, right.last_error); + assert_eq!(left.cleanup_warnings.count, right.cleanup_warnings.count); + assert_eq!( + left.progress.as_ref().map(|progress| ( + progress.num_objects, + progress.num_versions, + progress.bytes, + progress.remaining_buckets, + progress.bucket.as_str(), + progress.object.as_str() + )), + right.progress.as_ref().map(|progress| ( + progress.num_objects, + progress.num_versions, + progress.bytes, + progress.remaining_buckets, + progress.bucket.as_str(), + progress.object.as_str() + )) + ); + } + + assert_ne!( + first.pools[0].progress.as_ref().map(|progress| progress.elapsed), + second.pools[0].progress.as_ref().map(|progress| progress.elapsed) + ); } #[test] @@ -921,9 +1454,14 @@ mod rebalance_handler_tests { let status = RebalanceAdminStatus { id: "id-2".to_string(), stopped_at: Some(stopped), + stop_propagation: RebalanceStopPropagationStatus { + last_attempt_at: Some(stopped), + ..Default::default() + }, pools: vec![RebalancePoolStatus { id: 0, status: "Stopped".to_string(), + stopping: false, used: 0.3, last_error: None, cleanup_warnings: RebalanceCleanupWarnings::default(), @@ -934,5 +1472,45 @@ mod rebalance_handler_tests { let json = serde_json::to_string(&status).unwrap(); assert!(json.contains("\"stoppedAt\"")); assert!(json.contains("1970-01-01T00:16:40Z")); + assert!(json.contains("\"lastAttemptAt\":\"1970-01-01T00:16:40Z\"")); + } + + #[test] + fn test_rebalance_status_exposes_stop_propagation_failures() { + let stop_attempt = OffsetDateTime::from_unix_timestamp(1_000).unwrap(); + let reload_attempt = OffsetDateTime::from_unix_timestamp(1_010).unwrap(); + let encoded_error = encode_rebalance_stop_propagation_record(&RebalanceStopPropagationRecord { + stop_attempt_at: Some(stop_attempt), + stop_failures: vec!["peer node-a stop_rebalance failed: timeout".to_string()], + terminal_reload_attempt_at: Some(reload_attempt), + terminal_reload_failures: vec!["peer node-b load_rebalance_meta(start=false) failed: timeout".to_string()], + }); + let meta = RebalanceMeta { + stopped_at: Some(stop_attempt), + id: "id-3".to_string(), + percent_free_goal: 0.3, + pool_stats: vec![RebalanceStats { + participating: true, + info: RebalanceInfo { + status: RebalStatus::Started, + stopping: true, + last_error: Some(encoded_error), + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + let status = build_rebalance_stop_propagation_status(&meta); + + assert_eq!(status.last_attempt_at, Some(stop_attempt)); + assert_eq!(status.terminal_reload_attempt_at, Some(reload_attempt)); + assert!(!status.pending_terminal_reload); + assert_eq!(status.failed_peers, vec!["peer node-a stop_rebalance failed: timeout"]); + assert_eq!( + status.terminal_reload_failed_peers, + vec!["peer node-b load_rebalance_meta(start=false) failed: timeout"] + ); } } diff --git a/rustfs/src/admin/handlers/storage_compat.rs b/rustfs/src/admin/handlers/storage_compat.rs index bf20f6bd4..55b72cd13 100644 --- a/rustfs/src/admin/handlers/storage_compat.rs +++ b/rustfs/src/admin/handlers/storage_compat.rs @@ -16,15 +16,19 @@ pub(crate) use super::super::storage_compat::{ AdminError, AdminReplicationConfigExt, AdminVersioningConfigExt, CollectMetricsOpts, DailyAllTierStats, DiskStat, ECStore, ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND, - EndpointServerPools, Error, MetricType, PeerRestClient, RUSTFS_META_BUCKET, RebalSaveOpt, RebalanceCleanupWarnings, - RebalanceMeta, RebalanceStats, STORAGE_CLASS_SUB_SYS, StorageError, TierConfig, TierCreds, TierType, collect_local_metrics, - delete_admin_config, get_global_deployment_id, get_global_endpoints_opt, get_global_notification_sys, get_global_region, - global_rustfs_port, init_admin_config_defaults, is_reserved_or_invalid_bucket, load_data_usage_from_backend, - read_admin_config, read_admin_config_without_migrate, save_admin_config, save_admin_server_config, + EndpointServerPools, Error, MetricType, NotificationSys, PeerRestClient, RUSTFS_META_BUCKET, RebalSaveOpt, + RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, STORAGE_CLASS_SUB_SYS, StorageError, + TierConfig, TierCreds, TierType, collect_local_metrics, decode_rebalance_stop_propagation_record, delete_admin_config, + get_global_deployment_id, get_global_endpoints_opt, get_global_notification_sys, get_global_region, global_rustfs_port, + init_admin_config_defaults, is_reserved_or_invalid_bucket, load_data_usage_from_backend, read_admin_config, + read_admin_config_without_migrate, save_admin_config, save_admin_server_config, }; #[cfg(test)] -pub(crate) use super::super::storage_compat::{Endpoint, Endpoints, PoolEndpoints, RebalStatus, RebalanceInfo}; +pub(crate) use super::super::storage_compat::{ + Endpoint, Endpoints, PoolEndpoints, RebalStatus, RebalanceCleanupWarningEntry, RebalanceInfo, + encode_rebalance_stop_propagation_record, +}; pub(crate) mod bucket_target_sys { pub(crate) use super::super::super::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys}; diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index b493d99ee..a94e5aa24 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -43,7 +43,7 @@ use s3s::{ s3_error, }; use serde_urlencoded::from_bytes; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use time::OffsetDateTime; use tracing::{debug, warn}; @@ -811,19 +811,42 @@ pub struct ClearTierQuery { pub force: String, } +fn parse_clear_tier_query(uri: &Uri) -> S3Result { + let mut parsed = ClearTierQuery::default(); + let mut seen = HashSet::with_capacity(2); + let Some(query) = uri.query() else { + return Ok(parsed); + }; + + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "rand" => { + if !seen.insert("rand") { + return Err(s3_error!(InvalidArgument, "duplicate clear-tier query parameter")); + } + parsed.rand = Some(value.into_owned()); + } + "force" => { + if !seen.insert("force") { + return Err(s3_error!(InvalidArgument, "duplicate clear-tier query parameter")); + } + match value.as_ref() { + "true" | "false" => parsed.force = value.into_owned(), + _ => return Err(s3_error!(InvalidArgument, "invalid force flag")), + } + } + _ => return Err(s3_error!(InvalidArgument, "unknown clear-tier query parameter")), + } + } + + Ok(parsed) +} + pub struct ClearTier {} #[async_trait::async_trait] impl Operation for ClearTier { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let query = { - if let Some(query) = req.uri.query() { - let input: ClearTierQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; - input - } else { - ClearTierQuery::default() - } - }; + let query = parse_clear_tier_query(&req.uri)?; let Some(input_cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "authentication required")); @@ -845,7 +868,9 @@ impl Operation for ClearTier { let mut force: bool = false; let force_str = query.force; if !force_str.is_empty() { - force = force_str.parse().unwrap(); + force = force_str + .parse() + .map_err(|_e| s3_error!(InvalidArgument, "invalid force flag"))?; } let t = OffsetDateTime::now_utc(); @@ -1050,6 +1075,30 @@ mod tests { assert_eq!(mapped.message(), Some("tier verification failed. backend unavailable")); } + #[test] + fn parse_clear_tier_query_rejects_unknown_duplicate_and_invalid_force() { + for raw in [ + "/rustfs/admin/v3/tier?rand=token&force=yes", + "/rustfs/admin/v3/tier?rand=token&rand=other", + "/rustfs/admin/v3/tier?rand=token&unexpected=true", + ] { + let uri: Uri = raw.parse().expect("uri should parse"); + let err = parse_clear_tier_query(&uri).expect_err("strict clear-tier query should reject malformed input"); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + } + } + + #[test] + fn parse_clear_tier_query_accepts_valid_force() { + let uri: Uri = "/rustfs/admin/v3/tier?rand=token&force=true" + .parse() + .expect("uri should parse"); + let query = parse_clear_tier_query(&uri).expect("valid clear-tier query should parse"); + + assert_eq!(query.rand.as_deref(), Some("token")); + assert_eq!(query.force, "true"); + } + fn sample_daily_stats() -> DailyAllTierStats { let mut warm = LastDayTierStats::default(); warm.add_stats(TierStats { diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 0c3adadb5..aa81d2e41 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -236,6 +236,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ RouteRiskLevel::High, ), admin(HttpMethod::Post, "/rustfs/admin/v3/pools/cancel", DECOMMISSION, RouteRiskLevel::High), + admin(HttpMethod::Post, "/rustfs/admin/v3/pools/clear", DECOMMISSION, RouteRiskLevel::High), admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/start", REBALANCE, RouteRiskLevel::High), admin(HttpMethod::Get, "/rustfs/admin/v3/rebalance/status", REBALANCE, RouteRiskLevel::Sensitive), admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/stop", REBALANCE, RouteRiskLevel::High), diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index f7bf33599..27e118c5b 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -175,6 +175,7 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::GET, "/v3/pools/status"), admin_route(Method::POST, "/v3/pools/decommission"), admin_route(Method::POST, "/v3/pools/cancel"), + admin_route(Method::POST, "/v3/pools/clear"), admin_route(Method::POST, "/v3/rebalance/start"), admin_route(Method::GET, "/v3/rebalance/status"), admin_route(Method::POST, "/v3/rebalance/stop"), diff --git a/rustfs/src/admin/storage_compat.rs b/rustfs/src/admin/storage_compat.rs index 1f5ea2024..251b0c975 100644 --- a/rustfs/src/admin/storage_compat.rs +++ b/rustfs/src/admin/storage_compat.rs @@ -46,6 +46,7 @@ pub(crate) type RebalSaveOpt = ecstore_rebalance::RebalSaveOpt; pub(crate) type RebalanceCleanupWarnings = ecstore_rebalance::RebalanceCleanupWarnings; pub(crate) type RebalanceMeta = ecstore_rebalance::RebalanceMeta; pub(crate) type RebalanceStats = ecstore_rebalance::RebalanceStats; +pub(crate) type RebalanceStopPropagationRecord = ecstore_rebalance::RebalanceStopPropagationRecord; pub(crate) type StorageError = ecstore_error::StorageError; pub(crate) type Error = StorageError; pub(crate) type Result = core::result::Result; @@ -62,8 +63,19 @@ pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints; #[cfg(test)] pub(crate) type RebalStatus = ecstore_rebalance::RebalStatus; #[cfg(test)] +pub(crate) type RebalanceCleanupWarningEntry = ecstore_rebalance::RebalanceCleanupWarningEntry; +#[cfg(test)] pub(crate) type RebalanceInfo = ecstore_rebalance::RebalanceInfo; +pub(crate) fn decode_rebalance_stop_propagation_record(message: &str) -> Option { + ecstore_rebalance::decode_rebalance_stop_propagation_record(message) +} + +#[cfg(test)] +pub(crate) fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String { + ecstore_rebalance::encode_rebalance_stop_propagation_record(record) +} + pub(crate) trait AdminReplicationConfigExt { fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec; fn has_existing_object_replication(&self, arn: &str) -> (bool, bool); diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index 753433b4b..730de0fa9 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -56,7 +56,47 @@ pub struct QueryPoolStatusRequest { } #[derive(Debug, Clone, serde::Serialize)] -pub struct AdminPoolListItem { +pub struct AdminPoolDecommissionInfo { + #[serde(rename = "startTime", with = "time::serde::rfc3339::option")] + pub start_time: Option, + #[serde(rename = "startSize")] + pub start_size: usize, + #[serde(rename = "totalSize")] + pub total_size: usize, + #[serde(rename = "currentSize")] + pub current_size: usize, + #[serde(rename = "complete")] + pub complete: bool, + #[serde(rename = "failed")] + pub failed: bool, + #[serde(rename = "canceled")] + pub canceled: bool, + #[serde(rename = "queued")] + pub queued: bool, + #[serde(rename = "queuedBuckets")] + pub queued_buckets: Vec, + #[serde(rename = "decommissionedBuckets")] + pub decommissioned_buckets: Vec, + #[serde(rename = "bucket")] + pub bucket: String, + #[serde(rename = "prefix")] + pub prefix: String, + #[serde(rename = "object")] + pub object: String, + #[serde(rename = "objectsDecommissioned")] + pub items_decommissioned: usize, + #[serde(rename = "objectsDecommissionedFailed")] + pub items_decommission_failed: usize, + #[serde(rename = "bytesDecommissioned")] + pub bytes_done: usize, + #[serde(rename = "bytesDecommissionedFailed")] + pub bytes_failed: usize, + #[serde(rename = "waitingReason")] + pub waiting_reason: Option, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct AdminPoolStatus { #[serde(rename = "id")] pub id: usize, #[serde(rename = "cmdline")] @@ -74,9 +114,11 @@ pub struct AdminPoolListItem { #[serde(rename = "status")] pub status: String, #[serde(rename = "decommissionInfo")] - pub decommission: Option, + pub decommission: Option, } +pub type AdminPoolListItem = AdminPoolStatus; + #[derive(Clone, Default)] pub struct DefaultAdminUsecase { context: Option>, @@ -87,6 +129,7 @@ impl DefaultAdminUsecase { const POOL_STATUS_CANCELED: &'static str = "canceled"; const POOL_STATUS_COMPLETE: &'static str = "complete"; const POOL_STATUS_FAILED: &'static str = "failed"; + const POOL_STATUS_QUEUED: &'static str = "queued"; const POOL_STATUS_RUNNING: &'static str = "running"; #[cfg(test)] @@ -240,7 +283,7 @@ impl DefaultAdminUsecase { Ok(pool_statuses.into_iter().map(Self::pool_list_item_from_status).collect()) } - pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult { + pub async fn execute_query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult { let Some(endpoints) = self.endpoints() else { return Err(Self::app_error_default(S3ErrorCode::NotImplemented)); }; @@ -250,8 +293,7 @@ impl DefaultAdminUsecase { } let has_idx = if req.by_id { - let idx = req.pool.parse::().unwrap_or_default(); - if idx < endpoints.as_ref().len() { Some(idx) } else { None } + Self::parse_pool_idx_by_id(&req.pool, endpoints.as_ref().len()) } else { endpoints.get_pool_idx(&req.pool) }; @@ -265,7 +307,11 @@ impl DefaultAdminUsecase { return Err(Self::app_error(S3ErrorCode::InternalError, "Not init")); }; - store.status(idx).await.map_err(ApiError::from) + store + .status(idx) + .await + .map(Self::pool_list_item_from_status) + .map_err(ApiError::from) } fn pool_list_item_from_status(status: PoolStatus) -> AdminPoolListItem { @@ -279,7 +325,7 @@ impl DefaultAdminUsecase { let current_size = decommission.as_ref().map(|info| info.current_size).unwrap_or_default(); let used_size = total_size.saturating_sub(current_size); - AdminPoolListItem { + AdminPoolStatus { id, cmd_line, last_update, @@ -288,7 +334,7 @@ impl DefaultAdminUsecase { used_size, used: Self::used_ratio(total_size, used_size), status: Self::pool_list_status(decommission.as_ref()).to_string(), - decommission, + decommission: decommission.map(Self::admin_decommission_info_from_pool), } } @@ -297,11 +343,46 @@ impl DefaultAdminUsecase { Some(info) if info.complete => Self::POOL_STATUS_COMPLETE, Some(info) if info.failed => Self::POOL_STATUS_FAILED, Some(info) if info.canceled => Self::POOL_STATUS_CANCELED, + Some(info) if info.queued => Self::POOL_STATUS_QUEUED, Some(info) if info.start_time.is_some() => Self::POOL_STATUS_RUNNING, _ => Self::POOL_STATUS_ACTIVE, } } + fn admin_decommission_info_from_pool(info: PoolDecommissionInfo) -> AdminPoolDecommissionInfo { + let waiting_reason = Self::decommission_waiting_reason(&info).map(str::to_string); + AdminPoolDecommissionInfo { + start_time: info.start_time, + start_size: info.start_size, + total_size: info.total_size, + current_size: info.current_size, + complete: info.complete, + failed: info.failed, + canceled: info.canceled, + queued: info.queued, + queued_buckets: info.queued_buckets, + decommissioned_buckets: info.decommissioned_buckets, + bucket: info.bucket, + prefix: info.prefix, + object: info.object, + items_decommissioned: info.items_decommissioned, + items_decommission_failed: info.items_decommission_failed, + bytes_done: info.bytes_done, + bytes_failed: info.bytes_failed, + waiting_reason, + } + } + + fn decommission_waiting_reason(info: &PoolDecommissionInfo) -> Option<&'static str> { + if info.complete || info.failed || info.canceled || info.start_time.is_some() { + return None; + } + if info.queued { + return Some("queued"); + } + Some("waiting_for_worker") + } + fn used_ratio(total_size: usize, used_size: usize) -> f64 { if total_size == 0 { return 0.0; @@ -310,6 +391,11 @@ impl DefaultAdminUsecase { used_size as f64 / total_size as f64 } + fn parse_pool_idx_by_id(pool: &str, endpoint_count: usize) -> Option { + let idx = pool.parse::().ok()?; + (idx < endpoint_count).then_some(idx) + } + pub async fn execute_collect_dependency_readiness(&self) -> DependencyReadiness { collect_runtime_dependency_readiness().await } @@ -346,6 +432,21 @@ mod tests { let _ = readiness.iam_ready; } + #[test] + fn admin_query_pool_status_by_id_rejects_non_numeric_index() { + assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("pool-a", 4), None); + } + + #[test] + fn admin_query_pool_status_by_id_rejects_out_of_range_index() { + assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("4", 4), None); + } + + #[test] + fn admin_query_pool_status_by_id_accepts_valid_index() { + assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("0", 4), Some(0)); + } + #[test] fn admin_pool_list_item_maps_capacity_and_active_status() { let now = OffsetDateTime::UNIX_EPOCH; @@ -437,6 +538,45 @@ mod tests { assert_eq!(item.status, "running"); } + #[test] + fn admin_pool_list_item_exposes_queued_decommission_state() { + let item = DefaultAdminUsecase::pool_list_item_from_status(PoolStatus { + id: 3, + cmd_line: "pool-3".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + queued: true, + queued_buckets: vec!["bucket-a".to_string(), ".rustfs.sys/config".to_string()], + decommissioned_buckets: vec!["bucket-done".to_string()], + bucket: "bucket-a".to_string(), + prefix: "prefix/".to_string(), + object: "object.txt".to_string(), + items_decommissioned: 7, + items_decommission_failed: 1, + bytes_done: 1024, + bytes_failed: 64, + ..Default::default() + }), + }); + + assert_eq!(item.status, "queued"); + let value = serde_json::to_value(item).expect("admin pool status should serialize"); + assert_eq!(value["decommissionInfo"]["queued"], true); + assert_eq!( + value["decommissionInfo"]["queuedBuckets"], + serde_json::json!(["bucket-a", ".rustfs.sys/config"]) + ); + assert_eq!(value["decommissionInfo"]["decommissionedBuckets"], serde_json::json!(["bucket-done"])); + assert_eq!(value["decommissionInfo"]["bucket"], "bucket-a"); + assert_eq!(value["decommissionInfo"]["prefix"], "prefix/"); + assert_eq!(value["decommissionInfo"]["object"], "object.txt"); + assert_eq!(value["decommissionInfo"]["objectsDecommissioned"], 7); + assert_eq!(value["decommissionInfo"]["objectsDecommissionedFailed"], 1); + assert_eq!(value["decommissionInfo"]["bytesDecommissioned"], 1024); + assert_eq!(value["decommissionInfo"]["bytesDecommissionedFailed"], 64); + assert_eq!(value["decommissionInfo"]["waitingReason"], "queued"); + } + #[test] fn admin_pool_list_item_maps_terminal_decommission_statuses() { let complete = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo { @@ -451,11 +591,16 @@ mod tests { canceled: true, ..Default::default() })); + let queued = DefaultAdminUsecase::pool_list_status(Some(&PoolDecommissionInfo { + queued: true, + ..Default::default() + })); let idle = DefaultAdminUsecase::pool_list_status(None); assert_eq!(complete, "complete"); assert_eq!(failed, "failed"); assert_eq!(canceled, "canceled"); + assert_eq!(queued, "queued"); assert_eq!(idle, "active"); } } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 42b499c25..ad9439d06 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -126,6 +126,19 @@ fn background_rebalance_start_error_message(result: crate::storage::rpc::storage result.err().map(|err| format!("start_rebalance failed: {err}")) } +fn stop_rebalance_response(result: crate::storage::rpc::storage_compat::Result<()>) -> StopRebalanceResponse { + match result { + Ok(_) => StopRebalanceResponse { + success: true, + error_info: None, + }, + Err(err) => StopRebalanceResponse { + success: false, + error_info: Some(err.to_string()), + }, + } +} + #[path = "bucket.rs"] mod bucket; #[path = "disk.rs"] @@ -964,10 +977,16 @@ impl Node for NodeService { })); }; match store.reload_pool_meta().await { - Ok(_) => Ok(Response::new(ReloadPoolMetaResponse { - success: true, - error_info: None, - })), + Ok(_) => match store.spawn_missing_local_decommission_routines().await { + Ok(_) => Ok(Response::new(ReloadPoolMetaResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(Response::new(ReloadPoolMetaResponse { + success: false, + error_info: Some(err.to_string()), + })), + }, Err(err) => Ok(Response::new(ReloadPoolMetaResponse { success: false, error_info: Some(err.to_string()), @@ -975,7 +994,7 @@ impl Node for NodeService { } } - async fn stop_rebalance(&self, _request: Request) -> Result, Status> { + async fn stop_rebalance(&self, request: Request) -> Result, Status> { let Some(store) = resolve_object_store_handle() else { return Ok(Response::new(StopRebalanceResponse { success: false, @@ -983,11 +1002,12 @@ impl Node for NodeService { })); }; - let _ = store.stop_rebalance().await; - Ok(Response::new(StopRebalanceResponse { - success: true, - error_info: None, - })) + let expected_rebalance_id = request.into_inner().expected_rebalance_id; + let expected_rebalance_id = (!expected_rebalance_id.is_empty()).then_some(expected_rebalance_id); + + Ok(Response::new(stop_rebalance_response( + store.stop_rebalance_for_id(expected_rebalance_id.as_deref()).await, + ))) } #[tracing::instrument(skip_all, fields(start_rebalance))] @@ -1012,21 +1032,22 @@ impl Node for NodeService { if start_rebalance { log_background_rebalance_task_spawned!(start_rebalance); - let store = store.clone(); - spawn(async move { - if let Some(message) = background_rebalance_start_error_message(store.start_rebalance().await) { - error!( - event = EVENT_RPC_BACKGROUND_TASK_FAILED, - component = LOG_COMPONENT_STORAGE, - subsystem = LOG_SUBSYSTEM_REBALANCE, - operation = "start_rebalance", - state = "failed", - start_rebalance, - error = %message, - "node rpc background task failed" - ); - } - }); + if let Some(message) = background_rebalance_start_error_message(store.start_rebalance().await) { + error!( + event = EVENT_RPC_BACKGROUND_TASK_FAILED, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + operation = "start_rebalance", + state = "failed", + start_rebalance, + error = %message, + "node rpc background task failed" + ); + return Ok(Response::new(LoadRebalanceMetaResponse { + success: false, + error_info: Some(message), + })); + } } Ok(Response::new(LoadRebalanceMetaResponse { @@ -2331,7 +2352,9 @@ mod tests { async fn test_stop_rebalance() { let service = create_test_node_service(); - let request = Request::new(StopRebalanceRequest {}); + let request = Request::new(StopRebalanceRequest { + expected_rebalance_id: String::new(), + }); let response = service.stop_rebalance(request).await; assert!(response.is_ok()); @@ -2373,6 +2396,22 @@ mod tests { assert!(message.contains("boom")); } + #[test] + fn test_stop_rebalance_response_reports_local_stop_error() { + let response = stop_rebalance_response(Err(crate::storage::rpc::storage_compat::Error::other("boom"))); + + assert!(!response.success); + assert!(response.error_info.as_deref().is_some_and(|message| message.contains("boom"))); + } + + #[test] + fn test_stop_rebalance_response_reports_success() { + let response = stop_rebalance_response(Ok(())); + + assert!(response.success); + assert!(response.error_info.is_none()); + } + #[tokio::test] async fn test_load_bucket_metadata_empty_bucket() { let service = create_test_node_service();