From a8aaadb886ccb1a63a159b050679b46440f71e3a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 12:51:44 +0800 Subject: [PATCH 01/20] fix(replication): preserve concurrent remote target changes (#7262) * test(replication): cover concurrent remote target writes * fix(replication): merge remote target writes under transactions * test(replication): preserve concurrent repairs during target updates * fix(replication): retain removal guards after request cancellation * test(replication): enable loopback in ordinary target fixtures * test(replication): isolate remote target mutation scenarios * refactor(admin): keep target writes within existing boundaries * test(replication): assert removed target cache state * test(replication): inspect target cache before client refresh --- rustfs/src/admin/handlers/replication.rs | 786 ++++++++++++++++++++--- rustfs/src/admin/storage_api.rs | 4 +- 2 files changed, 716 insertions(+), 74 deletions(-) diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 40205b703..9c0e485c0 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -19,7 +19,6 @@ use crate::admin::runtime_sources::{ AppContext, app_context_from_req, current_notification_system_for_context, current_replication_pool_handle, current_replication_stats_handle_for_context, current_runtime_port, object_store_from_req, }; -use crate::admin::storage_api::AdminVersioningConfigExt as _; use crate::admin::storage_api::bucket::metadata::BUCKET_TARGETS_FILE; use crate::admin::storage_api::bucket::metadata_sys; use crate::admin::storage_api::bucket::metadata_sys::get_replication_config; @@ -28,9 +27,10 @@ use crate::admin::storage_api::bucket::replication::{BucketStats, ReplicationSta #[cfg(test)] use crate::admin::storage_api::bucket::replication::{REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS}; use crate::admin::storage_api::bucket::target::{ - BucketTarget, BucketTargetType, Credentials as TargetCredentials, LatencyStat, duration_from_secs_or_nanos, + ARN, BucketTarget, BucketTargetType, Credentials as TargetCredentials, LatencyStat, duration_from_secs_or_nanos, }; use crate::admin::storage_api::bucket::target_sys::{BucketTargetError, BucketTargetSys}; +use crate::admin::storage_api::bucket::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions}; use crate::admin::storage_api::contract::list::ListOperations as _; use crate::admin::storage_api::error::StorageError; @@ -50,6 +50,7 @@ use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; +use std::str::FromStr as _; use std::sync::Arc; use std::time::Duration; use time::OffsetDateTime; @@ -103,11 +104,34 @@ fn parse_remote_target_write_modes(uri: &http::Uri) -> S3Result<(bool, bool)> { Ok((update, replace_unreadable)) } -/// Repair decisions use the disk snapshot protected by the metadata transaction, -/// never the stale target cache retained after an unreadable configuration load. -async fn persist_remote_target_repair(bucket: &str, mut target: BucketTarget, incarnation: uuid::Uuid) -> S3Result { +enum RemoteTargetWrite { + Create { replace_unreadable: bool }, + Update { expected: Vec }, +} + +async fn persist_remote_target_repair(bucket: &str, target: BucketTarget, incarnation: uuid::Uuid) -> S3Result { + persist_remote_target_write( + bucket, + target, + incarnation, + RemoteTargetWrite::Create { + replace_unreadable: true, + }, + ) + .await +} + +/// Merge only into the disk snapshot protected by the metadata transaction. +/// An update may reuse remote validation only while that target is unchanged. +async fn persist_remote_target_write( + bucket: &str, + mut target: BucketTarget, + incarnation: uuid::Uuid, + mode: RemoteTargetWrite, +) -> S3Result { let mut discarded_unreadable = false; let mut target_error = None; + let mut conflict = false; let updated = metadata_sys::update_config_with(bucket, BUCKET_TARGETS_FILE, |metadata| { if metadata.bucket_incarnation_id != incarnation { return Err(StorageError::BucketNotFound(bucket.to_string())); @@ -118,12 +142,44 @@ async fn persist_remote_target_repair(bucket: &str, mut target: BucketTarget, in target_error = Some(BucketTargetError::BucketReplicationSourceNotVersioned { bucket: bucket.to_string(), }); - return Err(StorageError::other("source bucket versioning changed before target repair")); + return Err(StorageError::other("source bucket versioning changed before target write")); } discarded_unreadable = metadata.bucket_targets_unreadable(); + if discarded_unreadable + && !matches!( + &mode, + RemoteTargetWrite::Create { + replace_unreadable: true + } + ) + { + target_error = Some(BucketTargetError::BucketRemoteTargetsUnreadable { + bucket: bucket.to_string(), + }); + return Err(StorageError::other("persisted remote targets cannot be decoded")); + } let mut targets = metadata.bucket_target_config.clone().unwrap_or_default(); - let (arn, exists) = BucketTargetSys::remote_arn_for_targets(&targets.targets, &target, &target.deployment_id); - target.arn = arn; + let (update, exists) = match &mode { + RemoteTargetWrite::Create { .. } => { + let (arn, exists) = BucketTargetSys::remote_arn_for_targets(&targets.targets, &target, &target.deployment_id); + target.arn = arn; + (false, exists) + } + RemoteTargetWrite::Update { expected } => { + let current = targets.targets.iter().find(|current| current.arn == target.arn); + if current + .map(serde_json::to_vec) + .transpose() + .map_err(StorageError::other)? + .as_ref() + != Some(expected) + { + conflict = true; + return Err(StorageError::other("remote target changed during validation")); + } + (true, false) + } + }; if target.arn.is_empty() { target_error = Some(BucketTargetError::BucketRemoteArnInvalid { bucket: bucket.to_string(), @@ -131,7 +187,7 @@ async fn persist_remote_target_repair(bucket: &str, mut target: BucketTarget, in return Err(StorageError::other("remote target ARN is empty")); } if !exists { - BucketTargetSys::upsert_target_entry(&mut targets.targets, &target, false).map_err(|error| { + BucketTargetSys::upsert_target_entry(&mut targets.targets, &target, update).map_err(|error| { target_error = Some(error); StorageError::other("remote target merge failed") })?; @@ -139,6 +195,12 @@ async fn persist_remote_target_repair(bucket: &str, mut target: BucketTarget, in serde_json::to_vec(&targets).map_err(StorageError::other) }) .await; + if conflict { + return Err(S3Error::with_message( + S3ErrorCode::OperationAborted, + "remote target changed during validation; retry the request", + )); + } if let Some(error) = target_error { return Err(map_bucket_target_error(error)); } @@ -741,32 +803,60 @@ impl Operation for SetRemoteTargetHandler { return Ok(S3Response::new((StatusCode::OK, Body::from(arn_str)))); } - if !update { - let (arn, exist) = bucket_target_sys - .get_remote_arn(bucket, Some(&remote_target), remote_target.deployment_id.as_str()) - .await; - remote_target.arn = arn.clone(); - if exist && !arn.is_empty() { - let arn_str = serde_json::to_string(&arn).unwrap_or_default(); - - warn!("return exists, arn: {}", arn_str); - // MinIO-compatible clients encrypt the request payload for this endpoint, - // but they parse the success response directly as plain JSON string ARN. - return Ok(S3Response::new((StatusCode::OK, Body::from(arn_str)))); - } - } - - if remote_target.arn.is_empty() { - return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "ARN is empty".to_string())); - } - let _targets_guard = lock_bucket_targets_metadata(bucket).await; - - if update { - let Some(mut target) = bucket_target_sys - .get_remote_bucket_target_by_arn(bucket, &remote_target.arn) + let incarnation = metadata_sys::capture_bucket_metadata_incarnation(bucket) + .await + .map_err(ApiError::from)?; + // Preserve create idempotency even when an existing destination is + // offline, but decide it from disk under the same transaction as writers. + let metadata = { + let transaction_guard = metadata_sys::acquire_bucket_metadata_transaction_lock_for_incarnation(bucket, incarnation) .await + .map_err(ApiError::from)?; + let metadata = metadata_sys::get_config_from_disk(bucket).await.map_err(ApiError::from)?; + transaction_guard.checked_bucket_incarnation().map_err(ApiError::from)?; + if metadata.bucket_incarnation_id != incarnation { + return Err(ApiError::from(StorageError::BucketNotFound(bucket.to_string())).into()); + } + if metadata.bucket_targets_unreadable() { + return Err(map_bucket_target_error(BucketTargetError::BucketRemoteTargetsUnreadable { + bucket: bucket.to_string(), + })); + } + if !update { + let targets = metadata + .bucket_target_config + .as_ref() + .map(|targets| targets.targets.as_slice()) + .unwrap_or_default(); + let (arn, exists) = + BucketTargetSys::remote_arn_for_targets(targets, &remote_target, &remote_target.deployment_id); + if exists && !arn.is_empty() { + let body = serde_json::to_string(&arn) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize target ARN"))?; + return Ok(S3Response::new((StatusCode::OK, Body::from(body)))); + } + } + metadata + }; + let mut mode = RemoteTargetWrite::Create { + replace_unreadable: false, + }; + if update { + if remote_target.arn.is_empty() { + return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "ARN is empty")); + } + let Some(mut target) = metadata + .bucket_target_config + .unwrap_or_default() + .targets + .into_iter() + .find(|target| target.arn == remote_target.arn) else { - return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "Target not found".to_string())); + return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "Target not found")); + }; + mode = RemoteTargetWrite::Update { + expected: serde_json::to_vec(&target) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize target"))?, }; // Overlay only the requested field groups onto the stored target @@ -828,26 +918,16 @@ impl Operation for SetRemoteTargetHandler { remote_target = target; } - let arn = remote_target.arn.clone(); - - let targets = bucket_target_sys - .set_target(bucket, &remote_target, update) + // Neither the process-local target lock nor the cluster metadata + // transaction may be held while the remote endpoint is responding. + bucket_target_sys + .validate_target(bucket, &remote_target) .await .map_err(map_bucket_target_error)?; - let json_targets = serde_json::to_vec(&targets).map_err(|e| { - error!("Serialization error: {}", e); - S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize targets".to_string()) - })?; - - metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets) - .await - .map_err(|e| { - error!("Failed to update bucket targets: {}", e); - S3Error::with_message(S3ErrorCode::InternalError, format!("Failed to update bucket targets: {e}")) - })?; - bucket_target_sys.update_all_targets(bucket, Some(&targets)).await; - - let arn_str = serde_json::to_string(&arn).unwrap_or_default(); + let _targets_guard = lock_bucket_targets_metadata(bucket).await; + let arn = persist_remote_target_write(bucket, remote_target, incarnation, mode).await?; + let arn_str = serde_json::to_string(&arn) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize target ARN"))?; // MinIO-compatible clients encrypt the request payload for this endpoint, // but they parse the success response directly as plain JSON string ARN. @@ -952,25 +1032,74 @@ impl Operation for RemoveRemoteTargetHandler { .await .map_err(ApiError::from)?; - let sys = BucketTargetSys::get(); - let _targets_guard = lock_bucket_targets_metadata(bucket).await; - - let targets = sys.remove_target(bucket, arn_str).await.map_err(map_bucket_target_error)?; - - cancel_active_resync_intent(bucket, arn_str).await?; - - let json_targets = serde_json::to_vec(&targets).map_err(|e| { - error!("Serialization error: {}", e); - S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize targets".to_string()) + let arn = ARN::from_str(arn_str).map_err(|_| { + map_bucket_target_error(BucketTargetError::BucketRemoteArnInvalid { + bucket: bucket.to_string(), + }) })?; - - metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets) + let incarnation = metadata_sys::capture_bucket_metadata_incarnation(bucket) .await - .map_err(|e| { - error!("Failed to update bucket targets: {}", e); - S3Error::with_message(S3ErrorCode::InternalError, format!("Failed to update bucket targets: {e}")) - })?; - sys.update_all_targets(bucket, Some(&targets)).await; + .map_err(ApiError::from)?; + let targets_guard = lock_bucket_targets_metadata(bucket).await; + // Match resync start: local targets -> lifecycle -> metadata transaction + // -> resync admission -> resync status CAS. Keep the transaction through + // cancellation and target persistence so another start cannot interleave. + let transaction_guard = metadata_sys::acquire_bucket_metadata_transaction_lock_for_incarnation(bucket, incarnation) + .await + .map_err(ApiError::from)?; + let metadata = metadata_sys::get_config_from_disk(bucket).await.map_err(ApiError::from)?; + if metadata.bucket_targets_unreadable() { + return Err(map_bucket_target_error(BucketTargetError::BucketRemoteTargetsUnreadable { + bucket: bucket.to_string(), + })); + } + if arn.arn_type == BucketTargetType::ReplicationService { + if !metadata.replication_config_xml.is_empty() && metadata.replication_config.is_none() { + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + "persisted replication rules cannot be decoded", + )); + } + if metadata.replication_config.as_ref().is_some_and(|config| { + config + .filter_all_replication_target_arns() + .iter() + .any(|target| target == arn_str) + }) { + return Err(map_bucket_target_error(BucketTargetError::BucketRemoteRemoveDisallowed { + bucket: bucket.to_string(), + })); + } + } + let mut targets = metadata.bucket_target_config.unwrap_or_default(); + let previous_len = targets.targets.len(); + targets.targets.retain(|target| target.arn != *arn_str); + if targets.targets.len() == previous_len { + return Err(map_bucket_target_error(BucketTargetError::BucketRemoteTargetNotFound { + bucket: bucket.to_string(), + })); + } + let json_targets = serde_json::to_vec(&targets) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize targets"))?; + let bucket = bucket.clone(); + let arn = arn_str.clone(); + // The pool cancellation owns a detached task. Both outer guards must + // outlive it, even when the HTTP future is dropped while awaiting it. + tokio::spawn(async move { + let _targets_guard = targets_guard; + #[cfg(test)] + target_repair_tests::pause_target_removal(&bucket).await; + transaction_guard.checked_bucket_incarnation().map_err(ApiError::from)?; + cancel_active_resync_intent(&bucket, &arn).await?; + metadata_sys::update_bucket_targets_under_transaction_lock(&transaction_guard, &bucket, json_targets) + .await + .map_err(ApiError::from)?; + Ok::<(), S3Error>(()) + }) + .await + .map_err(|error| { + S3Error::with_message(S3ErrorCode::InternalError, format!("remote target removal task failed: {error}")) + })??; Ok(S3Response::new((StatusCode::NO_CONTENT, Body::from("".to_string())))) } @@ -2804,6 +2933,7 @@ mod target_repair_tests { use rustfs_iam::store::{Store as _, object::IAM_CONFIG_PREFIX}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; + use tokio::sync::oneshot; use tracing::instrument::WithSubscriber as _; const ACCESS_KEY: &str = "TARGETREPAIRROOT"; @@ -2819,6 +2949,29 @@ mod target_repair_tests { struct RemoteTargetServer { endpoint: String, task: tokio::task::JoinHandle<()>, + next_request: Arc>>, + } + + struct RequestPause { + reached: oneshot::Sender<()>, + release: oneshot::Receiver<()>, + } + + static REMOVAL_PAUSE: std::sync::Mutex> = std::sync::Mutex::new(None); + + pub(super) async fn pause_target_removal(bucket: &str) { + let pause = { + let mut pending = REMOVAL_PAUSE.lock().expect("removal pause lock"); + if pending.as_ref().is_some_and(|(expected, _)| expected == bucket) { + pending.take().map(|(_, pause)| pause) + } else { + None + } + }; + if let Some(pause) = pause { + pause.reached.send(()).expect("observe removal before cancellation"); + pause.release.await.expect("release removal cancellation"); + } } impl Drop for RemoteTargetServer { @@ -2831,6 +2984,8 @@ mod target_repair_tests { async fn start() -> Self { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind remote target"); let endpoint = listener.local_addr().expect("remote target address").to_string(); + let next_request = Arc::new(std::sync::Mutex::new(None::)); + let request_pause = Arc::clone(&next_request); let task = tokio::spawn(async move { loop { let (mut socket, _) = listener.accept().await.expect("accept remote target request"); @@ -2845,6 +3000,11 @@ mod target_repair_tests { } let head = String::from_utf8_lossy(&request); let first_line = head.lines().next().unwrap_or_default(); + let pause = request_pause.lock().expect("request pause lock").take(); + if let Some(pause) = pause { + pause.reached.send(()).expect("notify request arrival"); + pause.release.await.expect("release remote response"); + } let body = if first_line.starts_with("HEAD /") { "" } else if first_line.starts_with("GET /") && first_line.contains("versioning") { @@ -2862,7 +3022,27 @@ mod target_repair_tests { .expect("reply to remote target request"); } }); - Self { endpoint, task } + Self { + endpoint, + task, + next_request, + } + } + + fn pause_next_request(&self) -> (oneshot::Receiver<()>, oneshot::Sender<()>) { + let (reached, observed) = oneshot::channel(); + let (release, resume) = oneshot::channel(); + assert!( + self.next_request + .lock() + .expect("request pause lock") + .replace(RequestPause { + reached, + release: resume, + }) + .is_none() + ); + (observed, release) } fn target(&self) -> BucketTarget { @@ -2914,10 +3094,10 @@ mod target_repair_tests { } fn request(method: Method, query: &str, body: Vec) -> S3Request { - let operation = if method == Method::GET { - "list-remote-targets" - } else { - "set-remote-target" + let operation = match method { + Method::GET => "list-remote-targets", + Method::DELETE => "remove-remote-target", + _ => "set-remote-target", }; S3Request { input: Body::from(body), @@ -3272,4 +3452,464 @@ mod target_repair_tests { }) .await; } + + async fn remove(arn: &str) -> S3Result<()> { + let query = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("arn", arn) + .finish(); + let response = RemoveRemoteTargetHandler {} + .call(request(Method::DELETE, &query, Vec::new()), Params::new()) + .await?; + assert_eq!(response.output.0, StatusCode::NO_CONTENT); + Ok(()) + } + + async fn persisted_targets() -> BucketTargets { + metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read persisted targets") + .bucket_target_config + .expect("decode persisted targets") + } + + async fn assert_published_targets(targets: &BucketTargets) { + let cached = match BucketTargetSys::get().list_bucket_targets(BUCKET).await { + Ok(cached) => cached, + Err(BucketTargetError::BucketRemoteTargetNotFound { bucket }) if bucket == BUCKET && targets.targets.is_empty() => { + BucketTargets::default() + } + Err(error) => panic!("read published targets: {error}"), + }; + assert_eq!( + serde_json::to_value(cached).expect("encode cache"), + serde_json::to_value(targets).expect("encode disk") + ); + } + + const ORDINARY_TARGET_ENV: [(&str, Option<&str>); 3] = [ + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ]; + + async fn assert_target_write_preserves_uncached_repair(operation: &str) { + temp_env::async_with_vars( + ORDINARY_TARGET_ENV, + Box::pin(async move { + let (_temp, _env) = test_env().await; + let first = RemoteTargetServer::start().await; + let second = RemoteTargetServer::start().await; + let third = RemoteTargetServer::start().await; + let first_arn = repair(&first.target(), "").await.expect("create initial target"); + let stale = persisted_targets().await; + let second_arn = repair(&second.target(), "replace-unreadable=true") + .await + .expect("commit peer repair"); + let repaired = persisted_targets() + .await + .targets + .into_iter() + .find(|target| target.arn == second_arn) + .expect("persisted peer repair"); + // Model a node whose target cache predates the committed repair. + BucketTargetSys::get().update_all_targets(BUCKET, Some(&stale)).await; + let created = match operation { + "create" => Some(repair(&third.target(), "").await.expect("merge target create")), + "update" => { + let mut changed = stale.targets[0].clone(); + changed.replication_sync = true; + assert_eq!(repair(&changed, "update=true&sync=true").await.expect("merge target update"), first_arn); + None + } + "remove" => { + remove(&first_arn).await.expect("merge target removal"); + None + } + _ => unreachable!(), + }; + let targets = persisted_targets().await; + assert_eq!( + targets.targets.len(), + match operation { + "create" => 3, + "update" => 2, + _ => 1, + }, + "{operation}" + ); + let kept = targets + .targets + .iter() + .find(|target| target.arn == second_arn) + .expect("unrelated repair must survive"); + assert_eq!( + serde_json::to_value(kept).expect("encode kept target"), + serde_json::to_value(repaired).expect("encode repair") + ); + if let Some(created) = created { + assert!( + targets + .targets + .iter() + .any(|target| target.arn == created && target.endpoint == third.endpoint) + ); + } + if operation == "update" { + let updated = targets + .targets + .iter() + .find(|target| target.arn == first_arn) + .expect("updated target"); + assert!(updated.replication_sync); + assert_eq!(updated.credentials.as_ref().expect("retained credentials").secret_key, "remote-secret"); + } + assert_published_targets(&targets).await; + }), + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_create_preserves_repairs_missing_from_the_cache() { + assert_target_write_preserves_uncached_repair("create").await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_update_preserves_repairs_missing_from_the_cache() { + assert_target_write_preserves_uncached_repair("update").await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_remove_preserves_repairs_missing_from_the_cache() { + assert_target_write_preserves_uncached_repair("remove").await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_writes_refuse_unreadable_disk_even_with_a_readable_cache() { + temp_env::async_with_vars(ORDINARY_TARGET_ENV, async { + let (_temp, env) = test_env().await; + let server = RemoteTargetServer::start().await; + let arn = repair(&server.target(), "").await.expect("create initial target"); + let stale = persisted_targets().await; + seed_unreadable(&env).await; + BucketTargetSys::get().update_all_targets(BUCKET, Some(&stale)).await; + let file = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read metadata") + .save_file_path(); + let before = crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read original bytes"); + for operation in ["create", "update", "remove"] { + let error = match operation { + "create" => repair(&server.target(), "") + .await + .expect_err("cached idempotency cannot bypass unreadable disk"), + "update" => repair(&stale.targets[0], "update=true&sync=true") + .await + .expect_err("update cannot replace unreadable targets"), + "remove" => remove(&arn).await.expect_err("remove cannot replace unreadable targets"), + _ => unreachable!(), + }; + assert_eq!(error.code(), &S3ErrorCode::InternalError, "{operation}"); + assert_eq!( + crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read rejected write"), + before + ); + assert_published_targets(&stale).await; + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_create_is_idempotent_from_disk_when_the_remote_is_offline() { + temp_env::async_with_vars(ORDINARY_TARGET_ENV, async { + let (_temp, env) = test_env().await; + let mut server = RemoteTargetServer::start().await; + let target = server.target(); + let arn = repair(&target, "").await.expect("create initial target"); + let file = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read metadata") + .save_file_path(); + let before = crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read original bytes"); + BucketTargetSys::get() + .update_all_targets(BUCKET, Some(&BucketTargets::default())) + .await; + server.task.abort(); + assert!((&mut server.task).await.expect_err("remote server must stop").is_cancelled()); + assert_eq!( + tokio::time::timeout(Duration::from_secs(10), repair(&target, "")) + .await + .expect("idempotent create must not wait for the remote") + .expect("recognize persisted target"), + arn + ); + assert_eq!( + crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read idempotent create bytes"), + before + ); + let targets = persisted_targets().await; + assert_eq!(targets.targets.len(), 1); + assert_eq!(targets.targets[0].arn, arn); + }) + .await; + } + + async fn assert_target_update_rejects_concurrent_change(deleted: bool) { + temp_env::async_with_vars( + ORDINARY_TARGET_ENV, + Box::pin(async move { + let (_temp, _env) = test_env().await; + let server = RemoteTargetServer::start().await; + let arn = repair(&server.target(), "").await.expect("create initial target"); + let mut original = persisted_targets().await; + let mut requested = original.targets[0].clone(); + requested.replication_sync = true; + if deleted { + original.targets.clear(); + } else { + original.targets[0].bandwidth_limit = 1024; + } + let (observed, release) = server.pause_next_request(); + let update = repair(&requested, "update=true&sync=true"); + let peer_write = async { + tokio::time::timeout(Duration::from_secs(10), observed) + .await + .expect("validation must reach HTTP source") + .expect("observe validation"); + metadata_sys::update(BUCKET, BUCKET_TARGETS_FILE, serde_json::to_vec(&original).expect("encode peer change")) + .await + .expect("commit peer change during validation"); + release.send(()).expect("release validation response"); + }; + let (result, ()) = tokio::join!(update, peer_write); + assert_eq!( + result.expect_err("stale target update must conflict").code(), + &S3ErrorCode::OperationAborted + ); + let targets = persisted_targets().await; + assert_eq!( + serde_json::to_value(&targets).expect("encode current targets"), + serde_json::to_value(&original).expect("encode peer targets") + ); + assert_published_targets(&targets).await; + if !deleted { + assert_eq!(targets.targets[0].arn, arn); + assert!(!targets.targets[0].replication_sync); + } else { + assert!(BucketTargetSys::get().get_remote_target_client(BUCKET, &arn).await.is_none()); + } + }), + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_update_rejects_same_target_changes_during_remote_validation() { + assert_target_update_rejects_concurrent_change(false).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_update_rejects_target_deletion_during_remote_validation() { + assert_target_update_rejects_concurrent_change(true).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_update_preserves_a_repair_committed_during_validation() { + temp_env::async_with_vars(ORDINARY_TARGET_ENV, async { + let (_temp, _env) = test_env().await; + let first = RemoteTargetServer::start().await; + let second = RemoteTargetServer::start().await; + let first_arn = repair(&first.target(), "").await.expect("create initial target"); + let mut requested = persisted_targets().await.targets.remove(0); + requested.replication_sync = true; + let (observed, release) = first.pause_next_request(); + let update = repair(&requested, "update=true&sync=true"); + let concurrent_repair = async { + tokio::time::timeout(Duration::from_secs(10), observed) + .await + .expect("update validation must reach HTTP source") + .expect("observe update validation"); + let repaired = + tokio::time::timeout(Duration::from_secs(10), repair(&second.target(), "replace-unreadable=true")).await; + let second_arn = repaired + .expect("repair must complete during validation") + .expect("commit concurrent repair"); + let repaired = persisted_targets() + .await + .targets + .into_iter() + .find(|target| target.arn == second_arn) + .expect("read committed repair"); + release.send(()).expect("release update validation response"); + repaired + }; + let (updated_arn, repaired) = tokio::join!(update, concurrent_repair); + assert_eq!(updated_arn.expect("an unrelated target change must not conflict"), first_arn); + let targets = persisted_targets().await; + assert_eq!(targets.targets.len(), 2); + let updated = targets + .targets + .iter() + .find(|target| target.arn == first_arn) + .expect("updated target"); + assert_eq!( + serde_json::to_value(updated).expect("encode updated target"), + serde_json::to_value(requested).expect("encode requested target") + ); + let kept = targets + .targets + .iter() + .find(|target| target.arn == repaired.arn) + .expect("retained repair"); + assert_eq!( + serde_json::to_value(kept).expect("encode retained repair"), + serde_json::to_value(repaired).expect("encode committed repair") + ); + assert_published_targets(&targets).await; + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_validation_does_not_lock_out_a_concurrent_repair() { + temp_env::async_with_vars(ORDINARY_TARGET_ENV, async { + let (_temp, _env) = test_env().await; + let first = RemoteTargetServer::start().await; + let second = RemoteTargetServer::start().await; + let target = first.target(); + BucketTargetSys::get() + .validate_target(BUCKET, &target) + .await + .expect("remote validation must succeed before pausing the create request"); + let (observed, release) = first.pause_next_request(); + let create = repair(&target, ""); + let concurrent_repair = async { + tokio::time::timeout(Duration::from_secs(10), observed) + .await + .expect("validation must reach HTTP source") + .expect("observe validation"); + let repaired = + tokio::time::timeout(Duration::from_secs(10), repair(&second.target(), "replace-unreadable=true")).await; + release.send(()).expect("release validation response"); + repaired + .expect("repair must complete while another remote validation is paused") + .expect("concurrent repair") + }; + let (first_arn, second_arn) = tokio::join!(create, concurrent_repair); + let first_arn = first_arn.expect("merge validated create"); + let targets = persisted_targets().await; + assert_eq!(targets.targets.len(), 2); + assert!(targets.targets.iter().any(|target| target.arn == first_arn)); + assert!(targets.targets.iter().any(|target| target.arn == second_arn)); + assert_published_targets(&targets).await; + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_remove_keeps_both_guards_after_the_http_future_is_dropped() { + temp_env::async_with_vars(ORDINARY_TARGET_ENV, async { + let (_temp, _env) = test_env().await; + let server = RemoteTargetServer::start().await; + let arn = repair(&server.target(), "").await.expect("create initial target"); + let mut replacement = persisted_targets().await.targets.remove(0); + replacement.arn = "arn:rustfs:replication:us-east-1:replacement:remote".to_string(); + let expected = BucketTargets { targets: vec![replacement] }; + let encoded = serde_json::to_vec(&expected).expect("encode next target writer"); + let (reached, observed) = oneshot::channel(); + let (release, resume) = oneshot::channel(); + assert!(REMOVAL_PAUSE.lock().expect("removal pause lock").replace(( + BUCKET.to_string(), RequestPause { reached, release: resume } + )).is_none()); + let removing = tokio::spawn(async move { remove(&arn).await }); + tokio::time::timeout(Duration::from_secs(10), observed).await + .expect("removal must reach cancellation boundary").expect("observe removal"); + removing.abort(); + assert!(removing.await.expect_err("HTTP future must be dropped").is_cancelled()); + + let mut local_writer = Box::pin(lock_bucket_targets_metadata(BUCKET)); + assert!(futures::poll!(local_writer.as_mut()).is_pending(), "HTTP cancellation must not release the local target guard"); + drop(local_writer); + let probe = metadata_sys::ConfigWriteLockProbe::install(BUCKET); + let (entered, mut entered_rx) = oneshot::channel(); + let mut peer_writer = Box::pin(metadata_sys::update_config_with(BUCKET, BUCKET_TARGETS_FILE, move |metadata| { + entered.send(()).expect("observe peer transaction"); + assert!(metadata.bucket_target_config.as_ref().expect("read removed targets").targets.is_empty(), + "a competing writer must observe the completed removal before entering"); + Ok(encoded) + })); + tokio::select! { + result = peer_writer.as_mut() => panic!("HTTP cancellation released the metadata guard before commit: {result:?}"), + () = probe.wait_until_attempted() => {} + } + assert!(matches!(entered_rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)), + "the peer writer must remain outside the metadata transaction"); + release.send(()).expect("allow cancellation and target commit"); + tokio::time::timeout(Duration::from_secs(10), peer_writer).await + .expect("peer writer must proceed after removal commits").expect("persist peer target"); + entered_rx.await.expect("peer transaction entered"); + let targets = persisted_targets().await; + assert_eq!(serde_json::to_value(&targets).expect("encode disk targets"), serde_json::to_value(expected).expect("encode expected targets")); + assert_published_targets(&targets).await; + }).await; + } + + async fn assert_target_remove_checks_persisted_rules(malformed: bool) { + temp_env::async_with_vars(ORDINARY_TARGET_ENV, Box::pin(async move { + let (_temp, env) = test_env().await; + let server = RemoteTargetServer::start().await; + let arn = repair(&server.target(), "").await.expect("create initial target"); + let mut metadata = metadata_sys::get_config_from_disk(BUCKET).await.expect("read initial metadata"); + let file = metadata.save_file_path(); + metadata.replication_config_xml = if malformed { + b"".to_vec() + } else { + format!("{arn}activeEnabled1Disabled{arn}").into_bytes() + }; + // Another node has persisted rules before this node reloads them. + metadata.save_with_store(Arc::clone(&env.ecstore)).await.expect("persist peer rules without refreshing cache"); + assert!(metadata_sys::get_replication_config(BUCKET).await.is_err(), "precondition: cached rules are absent"); + assert_eq!(metadata_sys::get_config_from_disk(BUCKET).await.expect("read peer metadata").replication_config.is_none(), malformed); + let before = crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file).await.expect("read peer bytes"); + let error = remove(&arn).await.expect_err("rules must prevent unsafe target removal"); + assert_eq!(error.code(), if malformed { &S3ErrorCode::InternalError } else { &S3ErrorCode::InvalidRequest }); + assert_eq!(crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file).await.expect("read rejected removal bytes"), before); + let targets = persisted_targets().await; + assert_eq!(targets.targets.len(), 1); + assert_eq!(targets.targets[0].arn, arn); + assert_published_targets(&targets).await; + })) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_remove_checks_current_persisted_replication_rules() { + assert_target_remove_checks_persisted_rules(false).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ordinary_target_remove_rejects_malformed_persisted_replication_rules() { + assert_target_remove_checks_persisted_rules(true).await; + } } diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index ecf18cd6e..56f634c42 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -295,6 +295,8 @@ pub(crate) mod remote_s3_client { } pub(crate) mod metadata_sys { + #[cfg(test)] + pub(crate) use super::ecstore_bucket::metadata_sys::ConfigWriteLockProbe; use std::sync::Arc; use rustfs_policy::policy::BucketPolicy; @@ -667,7 +669,7 @@ pub(crate) mod replication { } pub(crate) mod target { - pub(crate) use super::ecstore_bucket::target::duration_from_secs_or_nanos; + pub(crate) use super::ecstore_bucket::target::{ARN, duration_from_secs_or_nanos}; pub(crate) type BucketTarget = super::ecstore_bucket::target::BucketTarget; pub(crate) type BucketTargetType = super::ecstore_bucket::target::BucketTargetType; pub(crate) type BucketTargets = super::ecstore_bucket::target::BucketTargets; From 5aef1796ccf8efb98aef52cfa5454f211e156c76 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 13:11:35 +0800 Subject: [PATCH 02/20] fix(odm): reject ambiguous native source dot segments (#7263) * fix(odm): reject ambiguous native source dot segments * docs(odm): align native provider limitations with implementation --- docs/operations/on-demand-migration.md | 4 +- rustfs/src/on_demand_migration/azure.rs | 12 +++++ rustfs/src/on_demand_migration/gcs.rs | 11 ++++ rustfs/src/on_demand_migration/native_http.rs | 50 ++++++++++++++++++- 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md index 2932ecb3f..9717559f3 100644 --- a/docs/operations/on-demand-migration.md +++ b/docs/operations/on-demand-migration.md @@ -358,8 +358,8 @@ sum by (bucket, reason) (rate(rustfs_on_demand_migration_pull_failures_total[5m] - **Source updates do not propagate.** Once an object is pulled, the local copy is authoritative; a later change on the source is never noticed. Plan the cutover so the source stops taking writes. - **Unversioned buckets re-pull deleted keys.** An unversioned bucket keeps nothing after a delete, so the key looks like an ordinary miss and is migrated again. Only a versioned bucket can shadow the source with a delete marker (`respect_local_delete_marker`). - **SSE-C source objects are not supported.** They are rejected with 424 `unsupported`; migrate them by another route. -- **Anonymous (credential-less) sources are not supported yet.** `source.credentials: null` parses and passes structural validation, but the client builder has no anonymous mode, so the admin `PUT` refuses it and the runtime would treat such a bucket as unavailable. A public source still needs a key pair. -- **Azure Blob is not a supported source** (rustfs/backlog#2166). GCS is supported only through its XML interoperability API with HMAC keys. +- **Native Azure/GCS keys containing a standalone `.` or `..` path segment are unsupported.** The URL transport would remove that segment and address a different object. These keys fail before any source request; ordinary dotted names, repeated slashes and literal percent escapes keep their identity. +- **Anonymous S3 sources are not supported yet.** `source.credentials: null` parses and passes structural validation, but the S3 client builder has no anonymous mode, so the admin `PUT` refuses it and the runtime would treat such a bucket as unavailable. A public S3 source still needs a key pair; native Azure/GCS credentials belong in their provider blocks. - **LIST merges the source only when asked, and only for v2.** With the default `policy.list_through = false` a client that lists before reading will not see un-migrated keys. Turning it on merges `ListObjectsV2` alone; `ListObjects` (v1) and `ListObjectVersions` stay local. - **A merged listing costs up to two local listings and two source listings per page** (one per side, plus a refill when the previous page consumed most of what that side had buffered). Walking N merged keys at `max-keys=K` therefore costs ceil(N/K) requests and between ceil(N/K) and 2*ceil(N/K) source listings. Source listings are capped at 10 per second per bucket (a compile-time constant); a listing that cannot get a slot inside one second is treated like a source failure and follows `policy.source_error`. - **A degraded merged page loses the source keys in its window.** Under `source_error = not_found` the page is answered locally and the source cursor is left where it was, so the keys the source would have contributed between the previous page's last key and this one are not shown again once pagination moves on. The `x-rustfs-on-demand-migration-list: local_only` header marks every page this happened on. diff --git a/rustfs/src/on_demand_migration/azure.rs b/rustfs/src/on_demand_migration/azure.rs index da904c91e..ff2e90cb0 100644 --- a/rustfs/src/on_demand_migration/azure.rs +++ b/rustfs/src/on_demand_migration/azure.rs @@ -917,6 +917,18 @@ mod tests { assert!(head.sse.is_none()); } + #[tokio::test] + async fn dot_segment_keys_fail_before_any_source_request() { + let (endpoint, recorded) = scripted_server(Vec::new()).await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + for key in [".", "..", "dir/./key", "dir/../key", "\u{fffe}/../key"] { + assert!(matches!(backend.head(key).await, Err(SourceError::Unsupported(_))), "HEAD {key:?}"); + assert!(matches!(backend.get(key, None).await, Err(SourceError::Unsupported(_))), "GET {key:?}"); + assert!(matches!(backend.tagging(key).await, Err(SourceError::Unsupported(_))), "tags {key:?}"); + } + assert!(recorded.lock().expect("recorder lock").is_empty()); + } + #[tokio::test] async fn sas_credentials_travel_in_the_query_and_never_sign() { let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, blob_headers(), String::new())]).await; diff --git a/rustfs/src/on_demand_migration/gcs.rs b/rustfs/src/on_demand_migration/gcs.rs index 5b43ca6ec..e5696748d 100644 --- a/rustfs/src/on_demand_migration/gcs.rs +++ b/rustfs/src/on_demand_migration/gcs.rs @@ -372,6 +372,17 @@ mod tests { ] } + #[tokio::test] + async fn dot_segment_keys_fail_before_any_source_request() { + let (endpoint, recorded) = scripted_server(Vec::new()).await; + let backend = backend(&endpoint); + for key in [".", "..", "dir/./key", "dir/../key", "\u{fffe}/../key"] { + assert!(matches!(backend.head(key).await, Err(SourceError::Unsupported(_))), "HEAD {key:?}"); + assert!(matches!(backend.get(key, None).await, Err(SourceError::Unsupported(_))), "GET {key:?}"); + } + assert!(recorded.lock().expect("recorder lock").is_empty()); + } + #[test] fn objects_list_maps_items_prefixes_and_the_page_token() { let page = parse_objects_list(LIST_PAGE_ONE).expect("page should parse"); diff --git a/rustfs/src/on_demand_migration/native_http.rs b/rustfs/src/on_demand_migration/native_http.rs index 9e1507ce1..519013de1 100644 --- a/rustfs/src/on_demand_migration/native_http.rs +++ b/rustfs/src/on_demand_migration/native_http.rs @@ -116,7 +116,15 @@ impl NativeHttp { .path_segments_mut() .map_err(|_| SourceError::Other("source endpoint cannot carry a path".to_string()))?; path.clear(); - path.extend(segments); + for segment in segments { + // URL normalization drops standalone dot segments. Sending + // that URL could fetch another object and backfill its bytes + // under the originally requested key. + if matches!(segment, "." | "..") { + return Err(SourceError::Unsupported("source path contains an unsupported dot segment".to_string())); + } + path.push(segment); + } } Ok(url) } @@ -431,4 +439,44 @@ mod tests { assert_eq!(url.as_str(), "https://acct.blob.core.windows.net/container/dir/a%20b%3Fc%23d.txt"); assert_eq!(url.query(), None, "a key with '?' must not become a query"); } + + #[test] + fn native_http_refuses_dot_segments_instead_of_addressing_another_object() { + let http = NativeHttp::for_test(Url::parse("https://source.example.com").expect("origin")); + for key in [ + ".", + "..", + "./key", + "../key", + "dir/./key", + "dir/../key", + "dir/.", + "dir/..", + "\u{fffe}/../key", + ] { + let error = http + .url(std::iter::once("bucket").chain(key.split('/'))) + .expect_err("dot segments must not disappear"); + assert!(matches!(error, SourceError::Unsupported(_)), "{key:?}: {error}"); + } + } + + #[test] + fn native_http_preserves_ordinary_dots_empty_segments_and_literal_escapes() { + let http = NativeHttp::for_test(Url::parse("https://source.example.com").expect("origin")); + for (key, path) in [ + ("file.txt", "/bucket/file.txt"), + (".hidden/.../tail.", "/bucket/.hidden/.../tail."), + ("/dir//key/", "/bucket//dir//key/"), + ("%2e/%2E%2E/key", "/bucket/%252e/%252E%252E/key"), + ("a+b &?#", "/bucket/a+b%20&%3F%23"), + ] { + let url = http + .url(std::iter::once("bucket").chain(key.split('/'))) + .expect("representable key"); + assert_eq!(url.path(), path, "{key:?}"); + assert!(url.query().is_none()); + assert!(url.fragment().is_none()); + } + } } From 1a459d650faa9f1e67d788f59a556d1e6ca061d2 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 6 Sep 2026 13:19:50 +0800 Subject: [PATCH 03/20] chore(deps): update flake.lock (#7264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e8be781' (2026-08-29) → 'github:NixOS/nixpkgs/17de0b9' (2026-09-04) • Updated input 'rust-overlay': 'github:oxalica/rust-overlay/996e9b0' (2026-08-29) → 'github:oxalica/rust-overlay/c361047' (2026-09-05) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 6a2416ed1..9fc362d30 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1787964612, - "narHash": "sha256-0N9nghg3nwzX6b6qc77EzjR9cu/Z+UR66FlfsCqiURs=", + "lastModified": 1788549839, + "narHash": "sha256-kOrCcSIA6w9J1hX5DqHy2k9pDTJymExTsbV74U9UtCA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e8be7818e19ada32105a8af937a6a473b38167ca", + "rev": "17de0b976395537756f30a3e78f2f06e5cec89ed", "type": "github" }, "original": { @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1787993548, - "narHash": "sha256-+IAEnmmx5YIhUWo0lp15jLLHchnXo5yKgWsi6C6Cf+0=", + "lastModified": 1788591095, + "narHash": "sha256-Vh+BeLWfbTT9AecazIsQ/Tkg/RzJeX3lEduANf256WA=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "996e9b0b019a4a9eb9e9a5641aefa06d801b5895", + "rev": "c361047d3a538f547f1617bb6b410411929ac9cc", "type": "github" }, "original": { From 38611d2510979cc2d381a4203869d4f71f3e2b4a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 13:28:58 +0800 Subject: [PATCH 04/20] fix(admin): compile metadata test helper only in tests (#7266) fix(admin): compile metadata update facade only for tests --- rustfs/src/admin/storage_api.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 56f634c42..024fd2f59 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -314,6 +314,7 @@ pub(crate) mod metadata_sys { super::ecstore_bucket::metadata_sys::get(bucket).await } + #[cfg(test)] pub(crate) async fn update(bucket: &str, config_file: &str, data: Vec) -> Result { crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await } From 1650f3c2a650b4f0709327d3effd35571115730e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 13:54:59 +0800 Subject: [PATCH 05/20] test(odm): verify global disable across restarts (#7268) * test(odm): cover global disable across restarts * test(odm): accept omitted V1 next marker * test(odm): gate backfill GET until the crash completes * test(odm): remove unused fault action import * test(odm): assert GET gate suspension without network races * test(odm): refresh verified Darwin E2E selection --- .config/e2e-full-selection.txt | 2 +- .config/e2e-smoke-selection.txt | 2 +- crates/e2e_test/src/fake_s3_target/mod.rs | 140 ++++++++++ .../on_demand_migration/interaction_test.rs | 242 +++++++++++++++++- 4 files changed, 381 insertions(+), 5 deletions(-) diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index 2cde94551..6a44dee52 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34 +sha256-darwin=53b05ac745905809d3828c6994bdd8ecf9d20b2b61a8a9d80fe15eb62f932193 sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b diff --git a/.config/e2e-smoke-selection.txt b/.config/e2e-smoke-selection.txt index 0a73fc814..6ebb3fcc0 100644 --- a/.config/e2e-smoke-selection.txt +++ b/.config/e2e-smoke-selection.txt @@ -1 +1 @@ -sha256=a2542dc86bbff56b2177efc621785c56fa7e8d813b209b7d935e1e41a9f0ad15 +sha256=5db88c6fec94d4f269c7d9cfc128bd2adc27b3d7021127e2fa0b1daccc5f900f diff --git a/crates/e2e_test/src/fake_s3_target/mod.rs b/crates/e2e_test/src/fake_s3_target/mod.rs index 65f916fd9..24ca390ba 100644 --- a/crates/e2e_test/src/fake_s3_target/mod.rs +++ b/crates/e2e_test/src/fake_s3_target/mod.rs @@ -451,10 +451,40 @@ impl JournaledHeaders { struct ControlState { scripts: HashMap>, keyed_scripts: HashMap<(Operation, String), VecDeque>, + held_get: Option, requests: VecDeque, next_sequence: u64, } +#[derive(Clone)] +struct HeldGetObject { + bucket: String, + key: String, + entered: watch::Sender, + released: watch::Receiver, +} + +/// Holds every GET of one object, including retries, until this guard is dropped. +#[must_use = "dropping the guard releases the held GET requests"] +pub struct GetObjectGate { + control: Arc>, + entered: watch::Receiver, + released: watch::Sender, +} + +impl GetObjectGate { + pub async fn wait_until_entered(&mut self) -> Result<(), watch::error::RecvError> { + self.entered.wait_for(|count| *count > 0).await.map(|_| ()) + } +} + +impl Drop for GetObjectGate { + fn drop(&mut self) { + lock(&self.control).held_get = None; + self.released.send_replace(true); + } +} + #[derive(Default)] struct StoreState { assign_own_version_ids: bool, @@ -936,6 +966,30 @@ impl FakeS3Target { .extend(std::iter::repeat_n(action, times)); } + /// Hold one exact bucket/key before any GET response can reach the client. + /// The fixture supports one live gate; request and connection deadlines still apply. + pub fn hold_get_object(&self, bucket: &str, key: &str) -> GetObjectGate { + assert!( + bucket.len() <= MAX_RETAINED_IDENTIFIER_BYTES && key.len() <= MAX_RETAINED_IDENTIFIER_BYTES, + "held GET identifiers exceed the fixture limit" + ); + let mut state = lock(&self.control); + assert!(state.held_get.is_none(), "fake target already holds a GET gate"); + let (entered, entered_rx) = watch::channel(0); + let (released, released_rx) = watch::channel(false); + state.held_get = Some(HeldGetObject { + bucket: bucket.to_string(), + key: key.to_string(), + entered, + released: released_rx, + }); + GetObjectGate { + control: Arc::clone(&self.control), + entered: entered_rx, + released, + } + } + pub fn clear_faults(&self) { let mut state = lock(&self.control); state.scripts.clear(); @@ -2243,6 +2297,17 @@ impl S3 for FakeBackend { let fault = request_fault(&req); apply_non_body_fault(fault.as_ref(), &self.control).await?; let input = req.input; + let held_get = lock(&self.control) + .held_get + .as_ref() + .filter(|held| held.bucket == input.bucket && held.key == input.key) + .cloned(); + if let Some(mut held) = held_get { + held.entered.send_modify(|count| *count += 1); + // Keep the gate installed when a request is cancelled or times out: + // a retry must cross the same boundary before returning any bytes. + let _ = held.released.wait_for(|released| *released).await; + } let (version, versioned) = { let state = lock(&self.store); ( @@ -2894,6 +2959,81 @@ mod tests { aws_sdk_s3::primitives::DateTime::from_secs(4_102_444_800) } + #[tokio::test] + async fn get_object_gate_holds_retries_and_releases_on_drop() -> Result<(), BoxError> { + let target = FakeS3Target::start().await?; + let bucket = "gated-target"; + target.create_bucket(bucket); + for key in ["held", "unrelated"] { + target.put_seed_object(bucket, key, Bytes::from_static(b"payload"), &SeedMetadata::default()); + } + { + let gate = target.hold_get_object(bucket, "held"); + let request = || S3Request { + input: GetObjectInput { + bucket: bucket.to_string(), + key: "held".to_string(), + ..Default::default() + }, + method: Method::GET, + uri: Uri::from_static("/gated-target/held"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + // Without a fault, only the gate can suspend this backend method. + let mut first = target.backend.get_object(request()); + assert!(futures::poll!(first.as_mut()).is_pending(), "the first GET must wait at the gate"); + drop(first); + let mut retry = target.backend.get_object(request()); + assert!(futures::poll!(retry.as_mut()).is_pending(), "a cancelled GET must not consume the gate"); + drop(gate); + let std::task::Poll::Ready(response) = futures::poll!(retry.as_mut()) else { + panic!("dropping the gate must release the waiting GET"); + }; + let mut body = response?.output.body.expect("released GET body"); + assert_eq!(body.next().await.transpose()?, Some(Bytes::from_static(b"payload"))); + assert!(body.next().await.is_none(), "released GET body must be complete"); + } + let client = client(&target); + let mut gate = target.hold_get_object(bucket, "held"); + let mut requests = tokio::task::JoinSet::new(); + let first = client.clone(); + requests.spawn(async move { get_bytes(&first, bucket, "held", None).await }); + timeout(Duration::from_secs(2), gate.wait_until_entered()).await??; + requests.abort_all(); + assert!( + requests + .join_next() + .await + .expect("first GET task") + .expect_err("cancel the first GET attempt") + .is_cancelled() + ); + + let retry = client.clone(); + requests.spawn(async move { get_bytes(&retry, bucket, "held", None).await }); + timeout(Duration::from_secs(2), gate.entered.wait_for(|count| *count == 2)).await??; + assert_eq!( + timeout(Duration::from_secs(2), get_bytes(&client, bucket, "unrelated", None)).await??, + Bytes::from_static(b"payload") + ); + assert!(requests.try_join_next().is_none(), "the retry must remain behind the gate"); + drop(gate); + assert_eq!( + timeout(Duration::from_secs(2), requests.join_next()) + .await? + .expect("retried GET task")??, + Bytes::from_static(b"payload") + ); + assert_eq!(get_bytes(&client, bucket, "held", None).await?, Bytes::from_static(b"payload")); + assert_eq!(target.count_requests(Operation::GetObject, "held"), 3); + Ok(()) + } + #[tokio::test] async fn object_lock_target_requires_a_checksum_on_locked_puts() -> Result<(), BoxError> { use aws_sdk_s3::error::ProvideErrorMetadata; diff --git a/crates/e2e_test/src/on_demand_migration/interaction_test.rs b/crates/e2e_test/src/on_demand_migration/interaction_test.rs index 93cafc05c..08a67adf8 100644 --- a/crates/e2e_test/src/on_demand_migration/interaction_test.rs +++ b/crates/e2e_test/src/on_demand_migration/interaction_test.rs @@ -22,8 +22,8 @@ //! local object and what the source was asked for. use super::common::{ - AdminResponse, BoxError, OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, - start_configured_env_with, + ALLOW_LOOPBACK_SOURCE_ENV, AdminResponse, BackfillOp, BackfillRequest, BoxError, ODM_MODULE_SWITCH_ENV, ODM_SERVER_ENV, + OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, start_configured_env_with, }; use crate::common::{RustFSTestEnvironment, replication_fast_env, signed_request}; use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation}; @@ -32,7 +32,7 @@ use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::types::{ BucketVersioningStatus, Event, FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter, ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault, - ServerSideEncryptionConfiguration, ServerSideEncryptionRule, VersioningConfiguration, + ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration, }; use bytes::Bytes; use local_ip_address::local_ip; @@ -733,6 +733,242 @@ async fn test_odm_disable_keeps_pulled_objects_and_stops_source_traffic() -> Tes Ok(()) } +/// The process switch preserves configured buckets and unfinished jobs while +/// restoring local-only S3 behavior, including after an ordinary metadata write. +#[tokio::test] +async fn test_odm_global_disable_preserves_data_config_and_backfill_across_restarts() -> TestResult { + let bucket = "odm-global-disable"; + let mut env = start_configured_env(bucket, SOURCE_BUCKET, |spec| spec.policy.list_through = true).await?; + let pulled_key = "migrated/pulled.bin"; + let remote_key = "remote/untouched.bin"; + let pending_key = "backfill/pending.bin"; + let local_key = "local/kept.bin"; + let source_body = Bytes::from_static(b"source payload"); + let local_body = Bytes::from_static(b"client payload"); + env.seed_source( + SOURCE_BUCKET, + &[ + SeedObject::new(pulled_key, source_body.clone()), + SeedObject::new(remote_key, source_body.clone()), + SeedObject::new(pending_key, source_body.clone()), + ], + ); + env.client + .put_object() + .bucket(bucket) + .key(local_key) + .body(local_body.clone().into()) + .send() + .await?; + let pulled = env.raw_get(bucket, pulled_key).await?; + assert_eq!(pulled.status, 200); + assert_eq!(pulled.header(ODM_RESPONSE_HEADER), Some("source")); + assert_eq!(pulled.body, source_body); + let stored = env.raw_get(bucket, pulled_key).await?; + assert_eq!(stored.status, 200); + assert_eq!(stored.header(ODM_RESPONSE_HEADER), None, "the inline pull has committed locally"); + assert_eq!(stored.body, source_body); + let config = env.get_config(bucket).await?; + assert_eq!(config.status, 200, "{}", config.body); + let config = config.json()?; + + // Hold every attempt until the process has exited, so retries cannot commit + // the only backfill object before the crash. The start checkpoint exists. + let mut pending_get = env.source.hold_get_object(SOURCE_BUCKET, pending_key); + let started = env + .start_backfill( + bucket, + BackfillRequest { + prefix: Some("backfill/".to_string()), + ..BackfillRequest::default() + }, + ) + .await?; + assert_eq!(started.status, 200, "{}", started.body); + let job_id = started.json()?["job"]["job_id"].as_str().ok_or("missing job ID")?.to_string(); + tokio::time::timeout(Duration::from_secs(10), pending_get.wait_until_entered()) + .await + .expect("backfill never reached the held source GET")?; + let process = env.rustfs.process.as_mut().ok_or("missing RustFS process before crash")?; + assert!(process.try_wait()?.is_none(), "RustFS exited before the controlled crash"); + process.kill()?; + let stopped = process.wait()?; + assert!(!stopped.success(), "the interrupted process must exit after being killed"); + drop(env.rustfs.process.take()); + drop(pending_get); + env.source.take_requests(); + env.rustfs + .restart_server_preserving_data(vec![], &[(ODM_MODULE_SWITCH_ENV, "false"), (ALLOW_LOOPBACK_SOURCE_ENV, "true")]) + .await?; + + let off_config = env.get_config(bucket).await?; + assert_eq!(off_config.status, 200, "{}", off_config.body); + assert_eq!(off_config.json()?, config, "the saved configuration and timestamp survive disabling"); + let status = env.status_json(bucket).await?; + assert_eq!(status["configured"], true, "{status}"); + assert_eq!(status["enabled"], true, "the bucket remains configured as enabled: {status}"); + assert_eq!(status["module_enabled"], false, "{status}"); + assert_eq!(status["counters"], Value::Null, "no bucket runtime is installed: {status}"); + let checkpoint = env.backfill_job(bucket).await?.ok_or("disabled module lost the checkpoint")?; + assert_eq!(checkpoint["job_id"], job_id); + assert_eq!(checkpoint["state"], "running", "the interrupted job is retained: {checkpoint}"); + + for (key, body) in [(local_key, &local_body), (pulled_key, &source_body)] { + let get = env.raw_get(bucket, key).await?; + assert_eq!(get.status, 200); + assert_eq!(&get.body, body); + assert_eq!(get.header(ODM_RESPONSE_HEADER), None); + let head = env.client.head_object().bucket(bucket).key(key).send().await?; + assert_eq!(head.content_length(), Some(i64::try_from(body.len())?)); + } + for key in [remote_key, pending_key] { + let get = env.raw_get(bucket, key).await?; + assert_eq!(get.status, 404, "disabled source GET {key}: {}", String::from_utf8_lossy(&get.body)); + let head = env.client.head_object().bucket(bucket).key(key).send().await; + let err = head.expect_err("a source-only object must remain absent locally"); + assert_eq!(err.raw_response().map(|response| response.status().as_u16()), Some(404)); + } + + let replacement = Bytes::from_static(b"written while the module is off"); + for key in [local_key, "local/deleted.bin"] { + env.client + .put_object() + .bucket(bucket) + .key(key) + .body(replacement.clone().into()) + .send() + .await?; + } + env.client + .delete_object() + .bucket(bucket) + .key("local/deleted.bin") + .send() + .await?; + assert_eq!(env.raw_get(bucket, "local/deleted.bin").await?.status, 404); + assert_eq!(env.raw_get(bucket, local_key).await?.body, replacement); + + // Both wire protocols must finish their local pages even though the saved + // configuration still requests list-through. + for use_v2 in [false, true] { + let mut cursor = None; + let mut listed = Vec::new(); + for page_number in 0..2 { + let (keys, truncated, next) = if use_v2 { + let page = env + .client + .list_objects_v2() + .bucket(bucket) + .max_keys(1) + .set_continuation_token(cursor) + .send() + .await?; + ( + page.contents() + .iter() + .map(|object| object.key().expect("listed key").to_string()) + .collect::>(), + page.is_truncated(), + page.next_continuation_token().map(str::to_string), + ) + } else { + let page = env + .client + .list_objects() + .bucket(bucket) + .max_keys(1) + .set_marker(cursor) + .send() + .await?; + // V1 may omit NextMarker without a delimiter; clients then + // continue from the last returned key. + let next = page.next_marker().or_else(|| { + if page.is_truncated() == Some(true) { + page.contents().last().and_then(|object| object.key()) + } else { + None + } + }); + ( + page.contents() + .iter() + .map(|object| object.key().expect("listed key").to_string()) + .collect::>(), + page.is_truncated(), + next.map(str::to_string), + ) + }; + assert_eq!(keys.len(), 1, "one local key per page, V2={use_v2}"); + assert_eq!(truncated, Some(page_number == 0), "local pagination must terminate, V2={use_v2}"); + if page_number == 0 { + assert!(next.as_ref().is_some_and(|value| !value.is_empty()), "missing local cursor, V2={use_v2}"); + } + cursor = next; + listed.extend(keys); + } + assert_eq!(listed, [local_key, pulled_key], "source-only keys must stay absent, V2={use_v2}"); + } + + let spec = env.fake_source_spec(SOURCE_BUCKET); + for response in [ + env.configure_source(bucket, &spec).await?, + env.validate_source(bucket, &spec).await?, + env.backfill(bucket, BackfillOp::Start(BackfillRequest::default())).await?, + ] { + assert_eq!(response.status, 400, "{}", response.body); + assert!(response.body.contains("OnDemandMigrationDisabled"), "{}", response.body); + } + let tagging = Tagging::builder() + .tag_set(Tag::builder().key("module").value("disabled").build()?) + .build()?; + env.client + .put_bucket_tagging() + .bucket(bucket) + .tagging(tagging.clone()) + .send() + .await?; + assert_eq!(env.get_config(bucket).await?.json()?, config, "an unrelated metadata write preserves ODM"); + assert_eq!( + env.backfill_job(bucket).await?, + Some(checkpoint), + "no recovery or checkpoint update while disabled" + ); + assert!( + env.source.requests().is_empty(), + "disabled startup and all requests must leave the source untouched" + ); + + env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?; + env.wait_until_source_consulted(bucket).await?; + assert_eq!( + env.get_config(bucket).await?.json()?, + config, + "reenabling uses the persisted configuration" + ); + let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?; + assert_eq!(tags.tag_set(), tagging.tag_set(), "the ordinary metadata write also persists"); + let resumed = env.raw_get(bucket, remote_key).await?; + assert_eq!(resumed.status, 200); + assert_eq!(resumed.header(ODM_RESPONSE_HEADER), Some("source")); + assert_eq!(resumed.body, source_body, "stored credentials still authenticate without reconfiguration"); + let completed = env + .wait_for_backfill(bucket, SETTLE, |job| job["state"] == "completed") + .await?; + assert_eq!(completed["job_id"], job_id, "the interrupted job resumes without a new start"); + assert_eq!(completed["failed"], 0, "{completed}"); + for (key, body) in [ + (local_key, &replacement), + (pulled_key, &source_body), + (pending_key, &source_body), + ] { + let get = env.raw_get(bucket, key).await?; + assert_eq!(get.status, 200); + assert_eq!(&get.body, body); + assert_eq!(get.header(ODM_RESPONSE_HEADER), None, "{key} remains stored locally"); + } + Ok(()) +} + /// Case 19: the admin surface an operator sees — the configuration read back /// without its secret, and a status document whose counters match the source /// journal exactly. From 6a323c3e91c5c3d85a1bf66d9ea5603824cc4ccb Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 6 Sep 2026 14:11:24 +0800 Subject: [PATCH 06/20] test(heal): cover start retry and deadline outcome contracts (#7242) --- crates/heal/src/heal/manager/tests.rs | 193 ++++++++++++++++++++++++- rustfs/src/admin/handlers/heal.rs | 53 +++++++ rustfs/src/storage/rpc/node_service.rs | 130 +++++++++++++++++ 3 files changed, 373 insertions(+), 3 deletions(-) diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index d8b3e3b16..17feee1b5 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -14,6 +14,9 @@ use super::*; use crate::heal::EcstoreError; +use crate::heal::outcome::{ + HealAbortReason, HealDeferredReason, HealExecutionOutcome, HealObjectDisposition, HealTraversalCoverage, +}; use crate::heal::resume::{CheckpointManager, ReplacementTargetIdentity}; use crate::heal::storage::{HealObjectInfo, HealStorageAPI}; use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType}; @@ -515,10 +518,15 @@ impl HealStorageAPI for MockStorage { async fn heal_object( &self, bucket: &str, - _object: &str, + object: &str, _version_id: Option<&str>, _opts: &HealOpts, ) -> Result<(HealResultItem, Option)> { + if bucket.starts_with("heal-start-retry-deadline-object-") && object == "blocked" { + let hook = COMPLETED_RETENTION_HOOKS.lock().await[bucket].clone(); + hook.started.notify_one(); + std::future::pending::<()>().await; + } if bucket == "completed-retention-failed" { return Err(Error::TaskExecutionFailed { message: "retention fixture failure".to_string(), @@ -580,11 +588,35 @@ impl HealStorageAPI for MockStorage { async fn list_objects_for_heal_page( &self, - _bucket: &str, + bucket: &str, _prefix: &str, - _continuation_token: Option<&str>, + continuation_token: Option<&str>, _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { + if bucket.starts_with("heal-start-retry-deadline-") { + if continuation_token.is_some() { + let hook = COMPLETED_RETENTION_HOOKS.lock().await[bucket].clone(); + hook.started.notify_one(); + std::future::pending::<()>().await; + } + let listing_timeout = bucket.starts_with("heal-start-retry-deadline-listing-"); + let names = if listing_timeout { + vec!["completed"] + } else { + vec!["completed", "blocked"] + }; + let objects = names + .into_iter() + .map(|name| crate::heal::storage::HealListItem { + name: name.to_string(), + version_id: None, + mod_time_unix_nanos: None, + lifecycle_object_info: None, + is_delete_marker: false, + }) + .collect(); + return Ok((objects, listing_timeout.then(|| "next".to_string()), listing_timeout)); + } Ok((Vec::new(), None, false)) } @@ -607,6 +639,161 @@ impl HealStorageAPI for MockStorage { } } +async fn assert_heal_start_retry_control_preserves_real_executor_progress(cancel: bool) { + for phase in ["listing", "object"] { + let bucket = format!("heal-start-retry-deadline-{phase}-{cancel}"); + let manager = HealManager::new(Arc::new(MockStorage), None); + let mut request = HealRequest::new( + HealType::Prefix { + bucket: bucket.clone(), + prefix: String::new(), + }, + HealOptions { + timeout: Some(if cancel { + Duration::from_secs(60) + } else { + Duration::from_millis(200) + }), + ..Default::default() + }, + HealPriority::High, + ); + request.source = HealRequestSource::Admin; + let task_id = request.id.clone(); + let hook = Arc::new(CompletedRetentionHook::default()); + { + let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await; + hooks.insert(bucket.clone(), Arc::clone(&hook)); + hooks.insert(task_id.clone(), Arc::clone(&hook)); + } + manager.submit_heal_request(request).await.expect("admit deadline task"); + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(5), hook.started.notified()) + .await + .expect("executor reaches blocked storage"); + let active = manager.get_task_report(&task_id).await.expect("active report"); + assert_eq!(active.progress.expect("real completed object progress").objects_healed, 1); + if cancel { + manager.active_heals.lock().await[&task_id].cancel_token.cancel(); + } + tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified()) + .await + .expect("deadline archives task"); + let report = manager.get_task_report(&task_id).await.expect("terminal report"); + assert_eq!( + report.status, + if cancel { + HealTaskStatus::Cancelled + } else { + HealTaskStatus::Timeout + }, + "blocked {phase}" + ); + let progress = report.progress.expect("terminal progress retained"); + assert_eq!(progress.objects_healed, 1); + assert_eq!(progress.objects_failed, 0, "interrupted object has no terminal storage result"); + assert_eq!(report.result_items.len(), 1, "completed result retained"); + let outcome = report.outcome.expect("canonical terminal outcome retained"); + assert_eq!( + outcome.execution, + HealExecutionOutcome::Aborted(if cancel { + HealAbortReason::Cancelled + } else { + HealAbortReason::Deadline + }) + ); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!(outcome.counters.healed, 0, "legacy success supplies no authoritative repair proof"); + let completed = outcome + .objects + .iter() + .find(|item| item.identity.object == "completed") + .expect("completed object diagnostic retained"); + assert_eq!(completed.disposition, HealObjectDisposition::Unknown); + if phase == "object" { + let interrupted = outcome + .objects + .iter() + .find(|item| item.identity.object == "blocked") + .expect("interrupted object diagnostic retained"); + assert_eq!( + interrupted.disposition, + if cancel { + HealObjectDisposition::Cancelled + } else { + HealObjectDisposition::Deferred { + reason: HealDeferredReason::Deadline, + retry_not_before: None, + } + } + ); + } else { + assert_eq!(outcome.objects.len(), 1, "an unread page cannot supply object identities"); + } + assert!(!manager.active_heals.lock().await.contains_key(&task_id)); + assert!(!manager.retrying_heals.lock().await.contains_key(&task_id)); + assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id)); + hook.finish.notify_one(); + COMPLETED_RETENTION_HOOKS + .lock() + .await + .retain(|key, _| key != &bucket && key != &task_id); + } +} + +#[tokio::test] +async fn heal_start_retry_deadline_preserves_real_executor_progress() { + assert_heal_start_retry_control_preserves_real_executor_progress(false).await; +} + +#[tokio::test] +async fn heal_start_retry_cancellation_preserves_real_executor_progress() { + assert_heal_start_retry_control_preserves_real_executor_progress(true).await; +} + +#[tokio::test] +async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() { + let manager = HealManager::new( + Arc::new(MockStorage), + Some(HealConfig { + task_timeout: Duration::ZERO, + ..Default::default() + }), + ); + let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None); + request.source = HealRequestSource::Admin; + request.options.timeout = Some(Duration::from_secs(60)); + let task_id = request.id.clone(); + let created_at = request.created_at; + let hook = Arc::new(CompletedRetentionHook::default()); + COMPLETED_RETENTION_HOOKS + .lock() + .await + .insert(task_id.clone(), Arc::clone(&hook)); + manager + .submit_heal_request(request) + .await + .expect("admit explicit-budget task"); + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified()) + .await + .expect("real read-quorum failure prepares retry"); + let retry = manager.retrying_heals.lock().await[&task_id].request.clone(); + assert_eq!(retry.id, task_id); + assert_eq!(retry.created_at, created_at); + assert_eq!(retry.source, HealRequestSource::Admin); + assert_eq!(retry.retry_attempts, 1); + let remaining = retry.options.timeout.expect("retry retains explicit budget"); + assert!(remaining > Duration::ZERO && remaining < Duration::from_secs(60)); + assert!(matches!( + manager.get_task_status(&task_id).await.expect("retry remains queryable"), + HealTaskStatus::Retrying { retry_attempt: 1, .. } + )); + manager.cancel_task(&task_id).await.expect("cancel held retry"); + hook.finish.notify_one(); + COMPLETED_RETENTION_HOOKS.lock().await.remove(&task_id); +} + struct ManagerRecoveryTestHook { replacement_resume_disk: DiskStore, listed: StdMutex, diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 74a2da980..7955a80a7 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -1636,6 +1636,44 @@ mod tests { assert!(executed.load(Ordering::SeqCst)); } + #[tokio::test] + async fn heal_start_retry_preflight_failures_do_not_create_request_identities() { + let hip = HealInitParams { + bucket: "bucket".to_string(), + ..Default::default() + }; + let mut request_ids = Vec::new(); + for attempt in 0..3 { + let executed_ids = &mut request_ids; + let request_params = &hip; + let result = execute_after_heal_control_capability( + || async { + if attempt < 2 { + Err(super::cluster_heal_control_unavailable("test_capability_failure")) + } else { + Ok(()) + } + }, + || async move { + let request = build_heal_channel_request(request_params); + executed_ids.push(request.id); + Ok(()) + }, + ) + .await; + if attempt < 2 { + assert!(result.is_err(), "failed capability checks must not start a heal"); + assert!( + request_ids.is_empty(), + "preflight failure must precede request construction and admission" + ); + } else { + result.expect("restored capabilities allow the first execution"); + assert_eq!(request_ids.len(), 1); + } + } + } + #[test] fn replacement_recovery_status_response_reports_cluster_proof() { let local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); @@ -1743,6 +1781,21 @@ mod tests { assert!(decoded.is_none()); } + #[test] + fn heal_start_retry_conflicts_keep_actionable_public_reasons() { + for (reason, label) in [ + (HealAdmissionDropReason::AlreadyRunning, "already_running"), + (HealAdmissionDropReason::OverlappingPaths, "overlapping_paths"), + ] { + let error = reject_heal_admission(HealAdmissionResult::Dropped(reason)); + assert_eq!(error.code(), &S3ErrorCode::OperationAborted); + assert!( + error.to_string().contains(label), + "the caller must distinguish conflicts from transient coordination failure" + ); + } + } + #[test] fn test_reject_heal_admission_preserves_retry_semantics() { for admission in [ diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 7278991c5..25b200fb2 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -2930,6 +2930,136 @@ mod tests { assert_eq!(err.code(), tonic::Code::InvalidArgument); } + fn heal_start_retry_fixture() -> ( + Arc, + rustfs_heal_contracts::heal_channel::HealChannelRequest, + rustfs_protos::heal_control::RequestMetadata, + ) { + let manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None)); + let mut request = rustfs_heal_contracts::heal_channel::create_heal_request( + "bucket".to_string(), + Some("prefix".to_string()), + true, + None, + ); + request.source = rustfs_heal_contracts::heal_channel::HealRequestSource::Admin; + request.recursive = Some(true); + let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).expect("fixture clock fits in i64"); + let metadata = rustfs_protos::heal_control::RequestMetadata::new(*Uuid::new_v4().as_bytes(), now, now + 30_000, 7); + (manager, request, metadata) + } + + #[tokio::test] + async fn heal_start_retry_exact_forced_envelope_returns_cached_admission() { + let (manager, request, metadata) = heal_start_retry_fixture(); + let request_id = request.id.clone(); + let envelope = rustfs_protos::heal_control::Envelope::start(request, metadata).expect("valid forced start"); + let lost_response = + execute_heal_control_envelope_with_manager(envelope.clone(), metadata.coordinator_epoch, Some(manager.clone())) + .await + .expect("first request is admitted before its response is lost"); + assert_eq!(manager.operations_snapshot().await.queue_length, 1); + + // The caller sees no first response, but retries the original envelope. + let replayed = execute_heal_control_envelope_with_manager(envelope, metadata.coordinator_epoch, Some(manager.clone())) + .await + .expect("an exact envelope replay must recover its receipt"); + assert_eq!(replayed, lost_response); + assert_eq!( + manager.operations_snapshot().await.queue_length, + 1, + "forceStart must not be executed twice" + ); + let outcome = rustfs_protos::heal_control::decode_result(&replayed) + .and_then(|result| result.into_outcome(&request_id, metadata.coordinator_epoch)) + .expect("matching canonical receipt"); + assert!(matches!(outcome, rustfs_protos::heal_control::Outcome::Start { + task_id, admission: rustfs_protos::heal_control::Admission::Accepted, + } if task_id == request_id)); + } + + #[tokio::test] + async fn heal_start_retry_new_forced_request_is_a_distinct_start() { + let (manager, request, metadata) = heal_start_retry_fixture(); + let first_id = request.id.clone(); + let first = rustfs_protos::heal_control::Envelope::start(request.clone(), metadata).expect("first start"); + let _lost_response = execute_heal_control_envelope_with_manager(first, metadata.coordinator_epoch, Some(manager.clone())) + .await + .expect("first admission"); + + // A fresh HTTP forceStart request intentionally requests another start. + let mut next_request = request; + next_request.id = Uuid::new_v4().to_string(); + let next_id = next_request.id.clone(); + let next_metadata = rustfs_protos::heal_control::RequestMetadata { + nonce: *Uuid::new_v4().as_bytes(), + ..metadata + }; + let next = rustfs_protos::heal_control::Envelope::start(next_request, next_metadata).expect("new forced start"); + let response = execute_heal_control_envelope_with_manager(next, metadata.coordinator_epoch, Some(manager.clone())) + .await + .expect("forceStart preserves its explicit admission semantics"); + let outcome = rustfs_protos::heal_control::decode_result(&response) + .and_then(|result| result.into_outcome(&next_id, metadata.coordinator_epoch)) + .expect("new receipt"); + assert!(matches!(outcome, rustfs_protos::heal_control::Outcome::Start { + task_id, admission: rustfs_protos::heal_control::Admission::Accepted, + } if task_id == next_id && task_id != first_id)); + assert_eq!( + manager.operations_snapshot().await.queue_length, + 2, + "a caller must not treat a new forced request as an idempotent transport retry" + ); + } + + #[tokio::test] + async fn heal_start_retry_same_id_with_changed_envelope_conflicts_before_admission() { + let (manager, request, metadata) = heal_start_retry_fixture(); + let original = rustfs_protos::heal_control::Envelope::start(request.clone(), metadata).expect("original start"); + let receipt = + execute_heal_control_envelope_with_manager(original.clone(), metadata.coordinator_epoch, Some(manager.clone())) + .await + .expect("original admission"); + let mut changed_options = request.clone(); + changed_options.remove_corrupted = Some(true); + let changed_metadata = rustfs_protos::heal_control::RequestMetadata { + nonce: *Uuid::new_v4().as_bytes(), + ..metadata + }; + for changed in [ + rustfs_protos::heal_control::Envelope::start(changed_options, metadata).expect("changed options"), + rustfs_protos::heal_control::Envelope::start(request, changed_metadata).expect("changed nonce"), + ] { + let error = execute_heal_control_envelope_with_manager(changed, metadata.coordinator_epoch, Some(manager.clone())) + .await + .expect_err("one request ID cannot identify different envelope bytes"); + assert_eq!(error.code(), tonic::Code::AlreadyExists); + assert_eq!(manager.operations_snapshot().await.queue_length, 1); + } + assert_eq!( + execute_heal_control_envelope_with_manager(original, metadata.coordinator_epoch, Some(manager)) + .await + .expect("conflicts must preserve the original receipt"), + receipt + ); + } + + #[tokio::test] + async fn heal_start_retry_wrong_coordinator_epoch_cannot_admit_locally() { + let (manager, request, metadata) = heal_start_retry_fixture(); + let request_id = request.id.clone(); + let envelope = rustfs_protos::heal_control::Envelope::start(request, metadata).expect("start envelope"); + let error = execute_heal_control_envelope_with_manager(envelope, metadata.coordinator_epoch + 1, Some(manager.clone())) + .await + .expect_err("a different coordinator epoch cannot accept the request"); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert_eq!(manager.operations_snapshot().await.queue_length, 0); + assert!(matches!( + manager.get_task_status(&request_id).await, + Err(rustfs_heal::Error::TaskNotFound { .. }) + )); + } + #[tokio::test] async fn heal_control_executor_preserves_canonical_token_and_drops_query_results() { let manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None)); From 08283d1fdc3ac30a3bcc474602996224dae93da7 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 6 Sep 2026 14:11:45 +0800 Subject: [PATCH 07/20] test(scanner): verify default scoped entry fallback walks (#7241) --- crates/scanner/src/scanner_io/tests.rs | 2 + .../scanner_io/tests/scoped_entry_fallback.rs | 289 ++++++++++++++++++ 2 files changed, 291 insertions(+) create mode 100644 crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 9521fc2fa..bd4add6df 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -39,6 +39,8 @@ use temp_env::with_var; use time::OffsetDateTime; use uuid::Uuid; +mod scoped_entry_fallback; + #[derive(Clone)] struct FixedWorkloadProvider { snapshot: WorkloadAdmissionRegistrySnapshot, diff --git a/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs b/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs new file mode 100644 index 000000000..88a7d91e2 --- /dev/null +++ b/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs @@ -0,0 +1,289 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision}; +use crate::storage_api::owner::EcstoreDiskAPI; + +type DriveIdentities = HashMap; +type WalkCounts = HashMap<(String, String, String), u64>; + +async fn drive_identities(store: &ECStore) -> DriveIdentities { + let mut identities = HashMap::new(); + let mut ids = HashSet::new(); + for set in store.all_set_disks() { + let source = DataUsageCacheSource::new(set.pool_index, set.set_index); + for disk in scanner_set_disk_inventory(set.as_ref()).await { + let id = EcstoreDiskAPI::get_disk_id(disk.as_ref()) + .await + .expect("fixture disk identity should be readable") + .expect("fixture disk must have a durable identity"); + assert!(!id.is_nil()); + assert!(ids.insert(id), "fixture disk identities must be unique"); + let path = crate::ScannerDiskExt::path(disk.as_ref()).to_string_lossy().into_owned(); + assert!(identities.insert(path, (id, source)).is_none()); + } + } + assert_eq!(identities.len(), 8); + identities +} + +fn walk_counts(drives: &DriveIdentities) -> WalkCounts { + rustfs_scanner_metrics::metrics::global_metrics() + .scanner_runtime_details_report() + .bucket_drive_results + .into_iter() + .filter(|result| drives.contains_key(&result.drive)) + .map(|result| ((result.bucket, result.drive, result.result), result.count)) + .collect() +} + +async fn put_and_settle(store: &ECStore, bucket: &str, object: &str) { + let set = &store.pools[0].disk_set[0]; + let mut reader = ScannerPutObjReader::from_vec(b"object".to_vec()); + set.put_object(bucket, object, &mut reader, &ScannerObjectOptions::default()) + .await + .expect("fixture object should persist"); + let lock = set.new_ns_lock(bucket, object).await.expect("fixture namespace lock"); + let _settled = lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("quorum-ACK rename tail must settle before taking the activity baseline"); +} + +async fn create_bucket(store: &ECStore, bucket: &str) { + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("fixture bucket should be created"); + put_and_settle(store, bucket, "initial").await; +} + +async fn persist_baseline(store: &Arc, baseline: &DataUsageInfo) { + let mut baseline = baseline.clone(); + baseline.usage_snapshot_converged = Some(true); + crate::save_config( + store.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + serde_json::to_vec(&baseline).expect("baseline should encode"), + ) + .await + .expect("fixture baseline should persist"); +} + +// Every invocation uses the production default scope. The expected walker set +// comes from storage's per-source inventory, not the resolver's selected names. +async fn run_entry(store: &Arc, cycle: u64, selected: Option<&str>, expect_walks: bool) -> DataUsageInfo { + let drives = drive_identities(store).await; + let inventory = store + .list_bucket_for_scanner(&BucketOptions::default()) + .await + .expect("fixture inventory should be complete"); + assert!(inventory.topology_complete); + let expected_walks = if expect_walks { + inventory + .set_buckets + .into_iter() + .flat_map(|set| { + let source = DataUsageCacheSource::new(set.pool_index, set.set_index); + set.buckets.into_iter().map(move |bucket| ((source, bucket.name), 1_u64)) + }) + .collect::>() + } else { + HashMap::new() + }; + let root_before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("root baseline should be readable"); + let dirty_before = dirty_usage_buckets_for_tests(); + let generation_before = dirty_usage_generation(); + let before = walk_counts(&drives); + let ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); + let (updates, mut receiver) = mpsc::channel(1); + let (observer, observed) = tokio::sync::oneshot::channel(); + let result = tokio::time::timeout( + Duration::from_secs(30), + nsscanner_with_storage_status_scoped( + store.as_ref(), + ScannerCycleRequest { + ctx, + budget, + updates, + want_cycle: cycle, + leader_epoch: 11, + scan_mode: HealScanMode::Normal, + scan_scope: ScannerBucketScanScope::default(), + persisted_usage_baseline: root_before.0.clone().map(Bytes::from), + requires_full_scan: false, + resolved_scope_observer: Some(observer), + }, + ), + ) + .await + .expect("entry cycle should finish within the fixture deadline") + .expect("entry cycle should succeed"); + assert_eq!(result.status, ScannerCycleStatus::Complete); + let scope = observed.await.expect("production resolver should report its decision"); + assert_eq!( + scope.selected_buckets.as_deref(), + selected.map(|name| HashSet::from([name.to_string()])).as_ref() + ); + let usage = receiver.recv().await.expect("one candidate should be delivered"); + assert!(receiver.recv().await.is_none(), "there must be exactly one terminal candidate"); + assert!(usage.usage_snapshot_complete); + assert!(!usage.usage_snapshot_partial); + assert_eq!(usage.scanner_cycle, Some(cycle)); + assert_eq!( + drive_identities(store).await, + drives, + "drive identities must not change during the oracle" + ); + + let after = walk_counts(&drives); + let mut actual = HashMap::new(); + for key in before.keys() { + assert!(after.contains_key(key), "metrics eviction would invalidate this exact-delta oracle"); + } + for ((bucket, drive, outcome), count) in after { + let previous = before + .get(&(bucket.clone(), drive.clone(), outcome.clone())) + .copied() + .unwrap_or(0); + let delta = count.checked_sub(previous).expect("fixture counters must not reset"); + if delta > 0 { + assert_eq!(outcome, "success", "no error or partial walker is expected"); + *actual.entry((drives[&drive].1, bucket)).or_insert(0_u64) += delta; + } + } + assert_eq!( + actual, expected_walks, + "each listed source/bucket must have exactly the expected real walks" + ); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("root after scan"), + root_before, + "producing a candidate must not replace the coordinator-owned root baseline" + ); + assert_eq!(dirty_usage_generation(), generation_before); + assert!( + dirty_usage_buckets_for_tests() == dirty_before, + "candidate delivery must not ACK pending dirty buckets" + ); + usage +} + +#[tokio::test] +#[serial] +async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks() { + let (_dir, store) = setup_two_pool_scanner_store().await; + clear_dirty_usage_buckets_for_tests(); + let hot = format!("hot-{}", Uuid::new_v4().simple()); + let cold = format!("cold-{}", Uuid::new_v4().simple()); + create_bucket(&store, &hot).await; + create_bucket(&store, &cold).await; + record_dirty_usage_bucket(&hot); + let baseline = run_entry(&store, 1, None, true).await; + persist_baseline(&store, &baseline).await; + + // A same-intent, same-cycle Current cache is a retry, not proof that a + // later cycle may reuse unselected buckets without durable incarnation. + run_entry(&store, 1, Some(&hot), false).await; + let usage = run_entry(&store, 2, Some(&hot), true).await; + assert_eq!(usage.buckets_usage[&hot].objects_count, 1); + assert_eq!(usage.buckets_usage[&cold].objects_count, 1); + assert_eq!(usage.objects_total_count, 2); + clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test] +#[serial] +async fn scoped_entry_fallback_rejects_invalid_persisted_baseline_at_the_walker() { + let (_dir, store) = setup_two_pool_scanner_store().await; + clear_dirty_usage_buckets_for_tests(); + let hot = format!("hot-{}", Uuid::new_v4().simple()); + let cold = format!("cold-{}", Uuid::new_v4().simple()); + create_bucket(&store, &hot).await; + create_bucket(&store, &cold).await; + record_dirty_usage_bucket(&hot); + // The first real scan is also the missing persisted-baseline case. + let baseline = run_entry(&store, 1, None, true).await; + for (index, kind) in [ + "malformed", + "unconverged", + "missing-set", + "wrong-source", + "mixed-plan", + "wrong-epoch", + ] + .into_iter() + .enumerate() + { + let mut candidate = baseline.clone(); + candidate.usage_snapshot_converged = Some(true); + match kind { + "unconverged" => candidate.usage_snapshot_converged = Some(false), + "missing-set" => { + candidate.usage_snapshot_set_states.pop(); + } + "wrong-source" => candidate.usage_snapshot_set_states[0].set_index = 99, + "mixed-plan" => candidate.usage_snapshot_set_states[1].scan_plan_digest = Some([0xA5; 32]), + "wrong-epoch" => candidate.usage_snapshot_set_states[0].scanner_epoch = Some(10), + "malformed" => {} + _ => unreachable!(), + } + let bytes = if kind == "malformed" { + b"{broken".to_vec() + } else { + serde_json::to_vec(&candidate).expect("candidate JSON") + }; + crate::save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes) + .await + .expect("negative baseline should persist"); + let usage = run_entry(&store, u64::try_from(index).expect("fixture cycle index should fit") + 2, None, true).await; + assert_eq!(usage.objects_total_count, 2, "{kind}"); + assert_eq!(usage.buckets_usage[&cold].objects_count, 1, "{kind}"); + } + clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test] +#[serial] +async fn scoped_entry_fallback_covers_overflow_and_new_bucket_inventory() { + let (_dir, store) = setup_two_pool_scanner_store().await; + clear_dirty_usage_buckets_for_tests(); + let hot = format!("hot-{}", Uuid::new_v4().simple()); + create_bucket(&store, &hot).await; + record_dirty_usage_bucket(&hot); + let baseline = run_entry(&store, 1, None, true).await; + persist_baseline(&store, &baseline).await; + for index in 0..=crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES { + record_dirty_usage_bucket(&format!("overflow-{index}")); + } + assert!(dirty_usage_buckets_for_tests().len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES); + let usage = run_entry(&store, 2, None, true).await; + assert_eq!(usage.objects_total_count, 1); + + clear_dirty_usage_buckets_for_tests(); + record_dirty_usage_bucket(&hot); + let new_bucket = format!("new-{}", Uuid::new_v4().simple()); + create_bucket(&store, &new_bucket).await; + // Even a previously valid baseline cannot cover the changed inventory. + let usage = run_entry(&store, 3, None, true).await; + assert_eq!(usage.objects_total_count, 2); + assert_eq!(usage.buckets_usage[&new_bucket].objects_count, 1); + clear_dirty_usage_buckets_for_tests(); +} From 760c9d65be2f0f50caf9fb204160e4d40f709c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 6 Sep 2026 14:12:25 +0800 Subject: [PATCH 08/20] fix(replication): close IAM snapshot, marker purge and broadcast gaps (#7195) --- crates/ecstore/src/api/mod.rs | 12 +- crates/ecstore/src/bucket/metadata.rs | 48 +- crates/ecstore/src/bucket/metadata_sys.rs | 242 +++- .../replication_object_decision_boundary.rs | 10 +- .../bucket/replication/replication_pool.rs | 97 ++ .../replication/replication_resyncer.rs | 42 +- crates/iam/src/manager.rs | 229 +++- crates/iam/src/sys.rs | 297 ++++- crates/policy/src/policy/doc.rs | 21 +- crates/replication/src/delete.rs | 166 ++- crates/replication/src/filemeta.rs | 20 + crates/replication/src/lib.rs | 6 +- crates/replication/src/mrf.rs | 169 ++- rustfs/src/admin/handlers/site_replication.rs | 1166 +++++++++++++++-- rustfs/src/admin/handlers/user.rs | 1 + rustfs/src/admin/storage_api.rs | 50 + rustfs/src/app/storage_api.rs | 2 + rustfs/src/site_replication/hooks.rs | 247 +++- rustfs/src/site_replication/mod.rs | 29 +- rustfs/src/site_replication/repair.rs | 6 +- rustfs/src/site_replication/retry.rs | 114 +- rustfs/src/site_replication/state.rs | 125 ++ rustfs/src/site_replication/tests.rs | 494 ++++++- rustfs/src/site_replication/transport.rs | 30 +- 24 files changed, 3379 insertions(+), 244 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index fb7af79ae..e0cc06d26 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -159,15 +159,17 @@ pub mod bucket { BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence, - capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get, - get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, - get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, - get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, + capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at, + delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, + get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config, + get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config, + get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, - update_quota_if_incarnation, update_under_transaction_lock, + update_if_incarnation_at, update_quota_if_incarnation, update_quota_if_incarnation_at, update_under_transaction_lock, + update_under_transaction_lock_at, }; #[cfg(feature = "test-util")] pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support}; diff --git a/crates/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs index 04b97f10f..b959dee60 100644 --- a/crates/ecstore/src/bucket/metadata.rs +++ b/crates/ecstore/src/bucket/metadata.rs @@ -791,9 +791,22 @@ impl BucketMetadata { } } + /// Replace one config payload and stamp its `*_config_updated_at` with the + /// local clock. This is the entry for edits that originate here: the + /// local write time is the edit's source time. pub fn update_config(&mut self, config_file: &str, data: Vec) -> Result { - let updated = OffsetDateTime::now_utc(); + self.update_config_at(config_file, data, OffsetDateTime::now_utc()) + } + /// [`Self::update_config`] with an explicit `updated_at` stamp. + /// + /// For a config replicated from another site the edit's source time is + /// the peer's `updated_at`, not the moment it lands here: staleness of + /// the next incoming item is judged against the stored stamp, so stamping + /// the local apply time would reject a newer source edit that was merely + /// delivered late (backlog#2292). Only replication receivers should pass + /// a foreign time; local edits keep [`Self::update_config`]. + pub fn update_config_at(&mut self, config_file: &str, data: Vec, updated: OffsetDateTime) -> Result { match config_file { BUCKET_POLICY_CONFIG => { self.policy_config_json = data; @@ -1525,6 +1538,39 @@ mod test { assert_eq!(metadata.bucket_incarnation_id, incarnation); } + /// backlog#2292: a replicated config is stamped with the source + /// `updated_at` it was given, not the local clock, while the plain + /// `update_config` entry keeps stamping the local clock. + #[test] + fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() { + let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3); + let mut metadata = BucketMetadata::new("source-stamped"); + + let stamped = metadata + .update_config_at(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(), source_time) + .unwrap(); + assert_eq!(stamped, source_time); + assert_eq!(metadata.policy_config_updated_at, source_time); + + let tagging = b"kv".to_vec(); + let stamped = metadata + .update_config_at(BUCKET_TAGGING_CONFIG, tagging, source_time) + .unwrap(); + assert_eq!(stamped, source_time); + assert_eq!(metadata.tagging_config_updated_at, source_time); + + let before = OffsetDateTime::now_utc(); + let stamped = metadata + .update_config(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec()) + .unwrap(); + assert!(stamped >= before, "a local edit is stamped with the local clock"); + assert_eq!(metadata.policy_config_updated_at, stamped); + assert_eq!( + metadata.tagging_config_updated_at, source_time, + "restamping one config must not move another config's stamp" + ); + } + #[test] fn object_locking_requires_lock_metadata_not_plain_versioning() { use s3s::dto::ObjectLockEnabled; diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 17f185b63..4d0aab0ee 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -567,6 +567,32 @@ pub async fn update_if_incarnation( config_file, data, Some(expected_incarnation_id), + None, + )) + .await +} + +/// [`update_if_incarnation`] stamping the config with `updated_at` instead of +/// the local clock. +/// +/// For a site-replication receiver the edit's source time is the peer's +/// `updated_at`; persisting it keeps the stored `*_config_updated_at` on the +/// source clock so the next item's staleness is judged source-time against +/// source-time (backlog#2292). See [`BucketMetadata::update_config_at`]. +pub async fn update_if_incarnation_at( + bucket: &str, + config_file: &str, + data: Vec, + expected_incarnation_id: Uuid, + updated_at: OffsetDateTime, +) -> Result { + Box::pin(update_with_sys_expected( + get_bucket_metadata_sys()?, + bucket, + config_file, + data, + Some(expected_incarnation_id), + Some(updated_at), )) .await } @@ -577,6 +603,30 @@ pub async fn delete_if_incarnation(bucket: &str, config_file: &str, expected_inc bucket, config_file, Some(expected_incarnation_id), + None, + )) + .await +} + +/// [`delete_if_incarnation`] stamping the cleared config with `updated_at` +/// (a replicated deletion's source time) instead of the local clock. +/// +/// The stamp survives the deletion as the config's `*_config_updated_at`, and +/// that is what the next incoming item is judged against: a local stamp on +/// the delete would reject a newer source re-create that was merely delivered +/// later (backlog#2292). See [`update_if_incarnation_at`]. +pub async fn delete_if_incarnation_at( + bucket: &str, + config_file: &str, + expected_incarnation_id: Uuid, + updated_at: OffsetDateTime, +) -> Result { + Box::pin(delete_with_sys_expected( + get_bucket_metadata_sys()?, + bucket, + config_file, + Some(expected_incarnation_id), + Some(updated_at), )) .await } @@ -598,34 +648,41 @@ async fn update_with_sys( config_file: &str, data: Vec, ) -> Result { - update_with_sys_expected(sys, bucket, config_file, data, None).await + update_with_sys_expected(sys, bucket, config_file, data, None, None).await } +/// `updated_at` is the stamp persisted on the config; `None` uses the local +/// clock (the edit originates here), `Some` carries a replicated edit's +/// source time (backlog#2292). async fn update_with_sys_expected( sys: Arc>, bucket: &str, config_file: &str, data: Vec, expected_incarnation_id: Option, + updated_at: Option, ) -> Result { let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?; - update_under_config_write_guard(sys, &guard, config_file, data).await + update_under_config_write_guard(sys, &guard, config_file, data, updated_at).await } /// [`delete`] against an explicitly supplied metadata system. See /// [`update_with_sys`]. async fn delete_with_sys(sys: Arc>, bucket: &str, config_file: &str) -> Result { - delete_with_sys_expected(sys, bucket, config_file, None).await + delete_with_sys_expected(sys, bucket, config_file, None, None).await } +/// `updated_at`: `None` stamps the local clock; `Some` persists a replicated +/// deletion's source time (backlog#2292). async fn delete_with_sys_expected( sys: Arc>, bucket: &str, config_file: &str, expected_incarnation_id: Option, + updated_at: Option, ) -> Result { let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?; - delete_under_config_write_guard(sys, &guard, config_file).await + delete_under_config_write_guard(sys, &guard, config_file, updated_at).await } /// Owns the complete bucket-config mutation fence. @@ -772,7 +829,21 @@ pub async fn update_under_transaction_lock( data: Vec, ) -> Result { guard.ensure_valid(bucket)?; - update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await + update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, None).await +} + +/// [`update_under_transaction_lock`] stamping the config with `updated_at` +/// (a replicated edit's source time) instead of the local clock; see +/// [`update_if_incarnation_at`] (backlog#2292). +pub async fn update_under_transaction_lock_at( + guard: &BucketMetadataMutationGuard, + bucket: &str, + config_file: &str, + data: Vec, + updated_at: OffsetDateTime, +) -> Result { + guard.ensure_valid(bucket)?; + update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, Some(updated_at)).await } /// Clear one config file while the caller holds this bucket's transaction lock. @@ -782,7 +853,7 @@ pub async fn delete_under_transaction_lock( config_file: &str, ) -> Result { guard.ensure_valid(bucket)?; - delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await + delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, None).await } pub async fn update_quota_if_incarnation( @@ -790,6 +861,29 @@ pub async fn update_quota_if_incarnation( data: Vec, expected_incarnation_id: Uuid, proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken, +) -> Result { + update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, None).await +} + +/// [`update_quota_if_incarnation`] stamping the quota config with +/// `updated_at` (a replicated edit's source time) instead of the local +/// clock; see [`update_if_incarnation_at`] (backlog#2292). +pub async fn update_quota_if_incarnation_at( + bucket: &str, + data: Vec, + expected_incarnation_id: Uuid, + proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken, + updated_at: OffsetDateTime, +) -> Result { + update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, Some(updated_at)).await +} + +async fn update_quota_if_incarnation_stamped( + bucket: &str, + data: Vec, + expected_incarnation_id: Uuid, + proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken, + updated_at: Option, ) -> Result { let sys = get_bucket_metadata_sys()?; let guard = Box::pin(acquire_config_write_guard_for_incarnation( @@ -807,7 +901,7 @@ pub async fn update_quota_if_incarnation( achieved: 0, }); } - update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await + update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data, updated_at).await } pub async fn update_bucket_targets_under_transaction_lock( @@ -823,6 +917,7 @@ async fn update_under_config_write_guard( guard: &BucketMetadataMutationGuard, config_file: &str, data: Vec, + updated_at: Option, ) -> Result { guard.ensure_valid(&guard.bucket)?; let metadata_sys = sys.read().await.clone(); @@ -834,7 +929,7 @@ async fn update_under_config_write_guard( Some(&guard.transaction_guard), &guard.bucket, "bucket config transaction", - metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id), + metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id, updated_at), ), ) .await?; @@ -846,6 +941,7 @@ async fn delete_under_config_write_guard( sys: Arc>, guard: &BucketMetadataMutationGuard, config_file: &str, + updated_at: Option, ) -> Result { guard.ensure_valid(&guard.bucket)?; let metadata_sys = sys.read().await.clone(); @@ -857,7 +953,7 @@ async fn delete_under_config_write_guard( Some(&guard.transaction_guard), &guard.bucket, "bucket config deletion transaction", - metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id), + metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id, updated_at), ), ) .await?; @@ -1762,15 +1858,17 @@ impl BucketMetadataSys { /// `update` and the config read alone). Keep these boxed. pub async fn update(&self, bucket: &str, config_file: &str, data: Vec) -> Result { let incarnation_id = Box::pin(self.get_bucket_incarnation_id(bucket)).await?; - Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id)).await + Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id, None)).await } pub async fn delete(&self, bucket: &str, config_file: &str) -> Result { let incarnation_id = self.get_bucket_incarnation_id(bucket).await?; - self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id) + self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id, None) .await } + /// `updated_at`: `None` stamps the local clock; `Some` persists a + /// replicated edit's source time (backlog#2292). async fn update_checked( &self, bucket: &str, @@ -1778,6 +1876,7 @@ impl BucketMetadataSys { data: Vec, parse: bool, expected_incarnation_id: Uuid, + updated_at: Option, ) -> Result { // Load through this system's own store, the one `save` persists to // (backlog#1052 S7). Reading from the ambient handle instead made the @@ -1788,7 +1887,10 @@ impl BucketMetadataSys { return Err(Error::BucketNotFound(bucket.to_string())); } - let updated = bm.update_config(config_file, data)?; + let updated = match updated_at { + Some(updated_at) => bm.update_config_at(config_file, data, updated_at)?, + None => bm.update_config(config_file, data)?, + }; Box::pin(self.save(bm)).await?; @@ -3755,6 +3857,106 @@ mod tests { ); } + /// backlog#2292: the explicit-stamp write path persists the given source + /// time as the config's `*_config_updated_at` — through the incarnation + /// path and through an already-held transaction guard — and survives a + /// reload from disk, while the plain path keeps stamping the local clock. + #[tokio::test] + async fn explicit_updated_at_is_persisted_as_the_config_stamp() { + let (dirs, ecstore) = isolated_store_over_temp_disks().await; + let bucket = "source-stamped-config"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created"); + } + let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore))); + let source_time = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600); + let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(); + let tagging = b"kv".to_vec(); + + // Incarnation path (`update_if_incarnation_at` minus the ambient lookup). + let stamped = + update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(source_time)) + .await + .expect("source-stamped policy write should persist"); + assert_eq!(stamped, source_time); + + // Held-guard path (`update_under_transaction_lock_at` minus the ambient lookup). + let guard = acquire_config_write_guard(sys.clone(), bucket).await.expect("write guard"); + let stamped = update_under_config_write_guard(sys.clone(), &guard, BUCKET_TAGGING_CONFIG, tagging, Some(source_time)) + .await + .expect("source-stamped tagging write should persist"); + drop(guard); + assert_eq!(stamped, source_time); + + let metadata_sys = sys.read().await.clone(); + metadata_sys.metadata_map.write().await.clear(); + let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk"); + assert_eq!(reloaded.policy_config_updated_at, source_time); + assert_eq!(reloaded.tagging_config_updated_at, source_time); + + // The plain path is unchanged: a local edit is stamped with the local clock. + let before = OffsetDateTime::now_utc(); + let stamped = update_with_sys(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy) + .await + .expect("locally stamped policy write should persist"); + assert!(stamped >= before, "the plain write path must keep stamping the local clock"); + let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk"); + assert_eq!(reloaded.policy_config_updated_at, stamped); + assert_eq!( + reloaded.tagging_config_updated_at, source_time, + "an unrelated config keeps its source stamp" + ); + } + + /// backlog#2292: a replicated delete persists the source time as the + /// cleared config's `*_config_updated_at`, so the receive-side gate + /// (source time against stored stamp) lets a newer source re-create land + /// even when the delete was applied later than the re-create's source + /// time; the plain delete keeps stamping the local clock. + #[tokio::test] + async fn explicit_updated_at_is_persisted_by_a_delete() { + let (dirs, ecstore) = isolated_store_over_temp_disks().await; + let bucket = "source-stamped-delete"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created"); + } + let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore))); + let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(); + let created_at = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600); + let deleted_at = created_at + Duration::from_secs(60); + let recreated_at = deleted_at + Duration::from_secs(60); + + update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(created_at)) + .await + .expect("source-stamped policy write should persist"); + let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, Some(deleted_at)) + .await + .expect("source-stamped policy delete should persist"); + assert_eq!(stamped, deleted_at); + + let metadata_sys = sys.read().await.clone(); + metadata_sys.metadata_map.write().await.clear(); + let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk"); + assert!(reloaded.policy_config_json.is_empty(), "the delete cleared the payload"); + assert_eq!(reloaded.policy_config_updated_at, deleted_at, "the delete kept the source stamp"); + assert!( + recreated_at >= reloaded.policy_config_updated_at, + "a re-create newer than the delete's source time is not stale against the stored stamp" + ); + + // The plain delete path is unchanged: stamped with the local clock. + update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy, None, Some(recreated_at)) + .await + .expect("re-create should persist"); + let before = OffsetDateTime::now_utc(); + let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, None) + .await + .expect("locally stamped delete should persist"); + assert!(stamped >= before, "the plain delete path must keep stamping the local clock"); + let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk"); + assert_eq!(reloaded.policy_config_updated_at, stamped); + } + /// The load and the persisted write share one write guard, so concurrent /// rewrites of the same config compose instead of clobbering each other. /// Moving the load outside that guard loses all but the last tag. @@ -3971,10 +4173,16 @@ mod tests { let new_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); assert_ne!(old_incarnation, new_incarnation); - let err = - update_with_sys_expected(sys.clone(), bucket, BUCKET_TAGGING_CONFIG, b"".to_vec(), Some(old_incarnation)) - .await - .expect_err("a request authorized for the deleted incarnation must fail closed"); + let err = update_with_sys_expected( + sys.clone(), + bucket, + BUCKET_TAGGING_CONFIG, + b"".to_vec(), + Some(old_incarnation), + None, + ) + .await + .expect_err("a request authorized for the deleted incarnation must fail closed"); assert!(matches!(err, Error::BucketNotFound(name) if name == bucket)); let persisted = sys.read().await.get_config_from_disk(bucket).await.unwrap(); @@ -4009,7 +4217,7 @@ mod tests { }], }) .unwrap(); - update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging) + update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging, None) .await .unwrap(); assert!(!delete.is_finished()); diff --git a/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs b/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs index 3544ac27b..553a57e1b 100644 --- a/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs @@ -20,9 +20,9 @@ pub use rustfs_replication::{ pub(crate) use rustfs_replication::{ ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision, - delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete, - is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match, - replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error, - resync_existing_delete_replication_info, resync_target_for_object, should_retry_delete_marker_purge, - single_part_replica_etag_mismatch, target_delete_version_id, + delete_replication_object_opts, delete_replication_target_version_id, heal_uses_delete_replication_path, + is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication, + replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size, + replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info, + resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch, }; diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 9efa9864e..e20ad9dbe 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -882,6 +882,20 @@ fn reconstructed_heal_delete_info( ) -> DeletedObjectReplicationInfo { let mut rstate = oi.replication_state(); rstate.replicate_decision_str = dsc.to_string(); + // The caller hands us a blank ObjectInfo (the source marker may already be + // gone), so the state above carries no target-assigned marker version ids. + // Restore them from the journal: `delete_marker_purge_version_id` must hit + // the id the target reported, not fall back to the source marker id, which + // a target that mints its own ids answers with an idempotent 204 that would + // acknowledge the intent while the real marker stays behind (backlog#2290). + // The corrupt flag rides along so a refusal stays a refusal after restart. + for (arn, version_id) in &entry.target_delete_marker_version_ids { + rstate + .target_delete_marker_version_ids + .entry(arn.clone()) + .or_insert_with(|| version_id.clone()); + } + rstate.target_delete_marker_version_ids_corrupt |= entry.target_delete_marker_version_ids_corrupt; let delete_marker_mtime = entry .delete_marker_mtime @@ -6601,4 +6615,87 @@ mod tests { replacement_data ); } + + /// backlog#2290: a delete-marker purge intent that survives a restart + /// through the MRF journal addresses the marker version the TARGET + /// assigned, exactly as the live watcher does (see the + /// `requires_delayed_purge` spawn). The journal carries the per-ARN ids + /// (`targetDeleteMarkerVersionIDs`) and replay restores them into the + /// reconstructed replication state; without that the replay would fall + /// back to the source marker id, which a target that mints its own ids + /// answers with an idempotent 204 — the entry would be acknowledged while + /// the real marker stayed behind. + #[test] + fn mrf_delete_marker_purge_replay_preserves_target_assigned_marker_version() { + use super::super::replication_object_decision_boundary::{delete_marker_purge_mrf_entry, delete_marker_purge_version_id}; + + let arn = "arn:minio:replication::generic-target:photos".to_string(); + let source_marker = uuid::Uuid::new_v4(); + let remote_marker = "remote-assigned-marker-version".to_string(); + + let live_oi = ObjectInfo { + bucket: "photos".to_string(), + name: "obj".to_string(), + version_id: Some(source_marker), + delete_marker: true, + ..Default::default() + }; + let mut live_state = live_oi.replication_state(); + live_state.replicate_decision_str = replicate_decision_for_admitted_targets(std::slice::from_ref(&arn)).to_string(); + live_state + .target_delete_marker_version_ids + .insert(arn.clone(), remote_marker.clone()); + let live = DeletedObjectReplicationInfo { + delete_object: ReplicationDeletedObject { + object_name: "obj".to_string(), + delete_marker: true, + delete_marker_version_id: Some(source_marker), + replication_state: Some(live_state), + ..Default::default() + }, + bucket: "photos".to_string(), + ..Default::default() + }; + assert_eq!( + delete_marker_purge_version_id(live.delete_object.replication_state.as_ref(), &arn, source_marker), + Some(Some(remote_marker.clone())), + "the live purge addresses the recorded target version" + ); + + // Watch window exhausted: persist the intent, restart, replay it. + let entry = delete_marker_purge_mrf_entry(&live, vec![arn.clone()]); + let replay_oi = ObjectInfo { + bucket: entry.bucket.clone(), + name: entry.object.clone(), + version_id: entry.version_id, + delete_marker: entry.delete_marker, + ..Default::default() + }; + let dsc = replicate_decision_for_admitted_targets(&entry.target_arns); + let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc); + + assert_eq!( + delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker), + Some(Some(remote_marker)), + "the MRF replay must address the target-assigned marker version, not source marker {source_marker}" + ); + + // A refusal (inconsistent recorded ids) must stay a refusal across the + // journal round trip instead of degrading into the source-id fallback. + let mut refused = live; + refused + .delete_object + .replication_state + .as_mut() + .expect("state was set above") + .target_delete_marker_version_ids_corrupt = true; + let entry = delete_marker_purge_mrf_entry(&refused, vec![arn.clone()]); + assert!(entry.target_delete_marker_version_ids_corrupt); + let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc); + assert_eq!( + delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker), + None, + "the MRF replay must keep refusing to guess when the recorded ids were inconsistent" + ); + } } diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index cf78dee70..64cbebeaa 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -32,11 +32,11 @@ use super::replication_msgp_boundary::ReplicationMsgpCodec; use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate}; use super::replication_object_decision_boundary::{ MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, - delete_replication_creates_marker, heal_uses_delete_replication_path, is_object_lock_denied_delete, - is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match, - replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error, - resync_existing_delete_replication_info, should_retry_delete_marker_purge, single_part_replica_etag_mismatch, - target_delete_version_id, + delete_replication_creates_marker, delete_replication_target_version_id, heal_uses_delete_replication_path, + is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication, + replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size, + replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info, + should_retry_delete_marker_purge, single_part_replica_etag_mismatch, }; use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission}; use super::replication_resync_boundary::ResyncStatusType; @@ -2051,7 +2051,11 @@ pub(crate) async fn replicate_delete_with_outcome( let is_version_purge = is_version_delete_replication(&dobj.delete_object); - let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object); + // The watcher exists to purge a replicated marker once the SOURCE marker + // vanishes. A version purge is that purge already (its failures reach the + // journal as a purge entry), so it must not spawn a second watcher that + // journals a duplicate intent (backlog#2290). + let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object) && !is_version_purge; let (replication_status, prev_status) = if !is_version_purge { ( @@ -2761,12 +2765,6 @@ fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str } async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc) -> ReplicatedTargetInfo { - let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id { - version_id.to_owned() - } else { - dobj.delete_object.version_id.unwrap_or_default() - }; - let mut rinfo = dobj .delete_object .replication_state @@ -2799,7 +2797,25 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli return rinfo; } - let version_id = target_delete_version_id(version_id, is_version_purge); + // Purging a replicated delete marker addresses the version the target + // assigned (recorded when the marker was created there); see + // `delete_replication_target_version_id`. A corrupt record is a failure, + // not a guess: the entry stays visible until the metadata is repaired. + let Some(version_id) = delete_replication_target_version_id(&dobj.delete_object, &tgt_client.arn) else { + warn!( + event = EVENT_DELETE_MARKER_PURGE_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = tgt_client.bucket, + object = dobj.delete_object.object_name, + arn = %tgt_client.arn, + reason = "recorded_target_version_inconsistent", + "Replicated version purge refused: recorded target delete-marker version metadata is inconsistent" + ); + rinfo.version_purge_status = VersionPurgeStatusType::Failed; + rinfo.error = Some("recorded target delete-marker version metadata is inconsistent".to_string()); + return rinfo; + }; if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() { match head_object_for_worker( diff --git a/crates/iam/src/manager.rs b/crates/iam/src/manager.rs index 384ed9a0a..a859fe59a 100644 --- a/crates/iam/src/manager.rs +++ b/crates/iam/src/manager.rs @@ -429,6 +429,27 @@ where } } + /// The cached mapping record for one user or group, looked up in the same + /// cache partition `policy_db_set` writes it to (group / STS / regular+service + /// user). `None` when no mapping is stored. + pub async fn get_mapped_policy_record(&self, name: &str, user_type: UserType, is_group: bool) -> Option { + let cache = self.cache.snapshot(); + if is_group { + cache.group_policies.get(name).cloned() + } else if user_type == UserType::Sts { + cache.sts_policies.get(name).cloned() + } else { + cache.user_policies.get(name).cloned() + } + } + + /// The cached group record (members, status, own timestamp) without the + /// mapped-policy overlay `get_group_description` applies. `None` when the + /// group does not exist. + pub async fn get_group_info(&self, name: &str) -> Option { + self.cache.snapshot().groups.get(name).cloned() + } + pub async fn get_policy(&self, name: &str) -> Result { if name.is_empty() { return Err(Error::InvalidArgument); @@ -534,6 +555,17 @@ where } pub async fn set_policy(&self, name: &str, policy: Policy) -> Result { + self.set_policy_at(name, policy, OffsetDateTime::now_utc()).await + } + + /// [`Self::set_policy`] stamping the document with `updated_at` instead + /// of the local clock. + /// + /// A site-replication receiver passes the edit's source time: the next + /// incoming revision is judged against the stored `UpdateDate`, so a + /// local stamp would reject a newer source edit that was merely delivered + /// later (backlog#2291). The returned stamp is the one persisted. + pub async fn set_policy_at(&self, name: &str, policy: Policy, updated_at: OffsetDateTime) -> Result { if name.is_empty() || policy.is_empty() { return Err(Error::InvalidArgument); } @@ -544,18 +576,17 @@ where .get(name) .map(|v| { let mut p = v.clone(); - p.update(policy.clone()); + p.update_at(policy.clone(), updated_at); p }) - .unwrap_or_else(|| PolicyDoc::new(policy)); + .unwrap_or_else(|| PolicyDoc::new_at(policy, updated_at)); self.api.save_policy_doc(name, policy_doc.clone()).await?; - let now = OffsetDateTime::now_utc(); + self.cache + .add_or_update_policy_doc(name, &policy_doc, OffsetDateTime::now_utc()); - self.cache.add_or_update_policy_doc(name, &policy_doc, now); - - Ok(now) + Ok(updated_at) } pub async fn list_policies(&self, bucket_name: &str) -> Result> { @@ -789,6 +820,12 @@ where /// create a service account and update cache pub async fn add_service_account(&self, cred: Credentials) -> Result { + self.add_service_account_at(cred, OffsetDateTime::now_utc()).await + } + + /// [`Self::add_service_account`] stamping the identity with `updated_at` + /// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291). + pub async fn add_service_account_at(&self, cred: Credentials, updated_at: OffsetDateTime) -> Result { if cred.access_key.is_empty() || cred.parent_user.is_empty() { return Err(Error::InvalidArgument); } @@ -800,7 +837,8 @@ where } drop(cache); - let u = UserIdentity::new(cred); + let mut u = UserIdentity::new(cred); + u.update_at = Some(updated_at); self.api .save_user_identity(&u.credentials.access_key, UserType::Svc, u.clone(), None) @@ -808,10 +846,22 @@ where self.update_user_with_claims(&u.credentials.access_key, u.clone())?; - Ok(OffsetDateTime::now_utc()) + Ok(updated_at) } pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result { + self.update_service_account_at(name, opts, OffsetDateTime::now_utc()).await + } + + /// [`Self::update_service_account`] stamping the identity with + /// `updated_at` instead of the local clock; see [`Self::set_policy_at`] + /// (backlog#2291). + pub async fn update_service_account_at( + &self, + name: &str, + opts: UpdateServiceAccountOpts, + updated_at: OffsetDateTime, + ) -> Result { let _mutation_guard = self.cache.service_account_mutation_lock().lock().await; let cache = self.cache.snapshot(); let Some(ui) = cache.users.get(name).cloned() else { @@ -858,13 +908,7 @@ where } if let Some(status) = opts.status { - match status.as_str() { - val if val == AccountStatus::Enabled.as_ref() => cr.status = auth::ACCOUNT_ON.to_owned(), - val if val == AccountStatus::Disabled.as_ref() => cr.status = auth::ACCOUNT_OFF.to_owned(), - auth::ACCOUNT_ON => cr.status = auth::ACCOUNT_ON.to_owned(), - auth::ACCOUNT_OFF => cr.status = auth::ACCOUNT_OFF.to_owned(), - _ => cr.status = auth::ACCOUNT_OFF.to_owned(), - } + cr.status = account_status_flag(&status).to_owned(); } let mut m: HashMap = if token_without_expiration { @@ -916,8 +960,8 @@ where cr.session_token = jwt_sign(&m, &cr.secret_key)?; - let u = UserIdentity::new(cr); - let updated_at = u.update_at.unwrap_or_else(OffsetDateTime::now_utc); + let mut u = UserIdentity::new(cr); + u.update_at = Some(updated_at); self.api .save_user_identity(&u.credentials.access_key, UserType::Svc, u.clone(), None) .await?; @@ -1149,6 +1193,20 @@ where Ok((policies.into_iter().collect(), update_at)) } pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result { + self.policy_db_set_at(name, user_type, is_group, policy, OffsetDateTime::now_utc()) + .await + } + + /// [`Self::policy_db_set`] stamping the mapping with `updated_at` instead + /// of the local clock; see [`Self::set_policy_at`] (backlog#2291). + pub async fn policy_db_set_at( + &self, + name: &str, + user_type: UserType, + is_group: bool, + policy: &str, + updated_at: OffsetDateTime, + ) -> Result { if name.is_empty() { return Err(Error::InvalidArgument); } @@ -1168,10 +1226,11 @@ where self.cache.delete_user_policy(name, OffsetDateTime::now_utc()); } - return Ok(OffsetDateTime::now_utc()); + return Ok(updated_at); } - let mp = MappedPolicy::new(policy); + let mut mp = MappedPolicy::new(policy); + mp.update_at = updated_at; let cache = self.cache.snapshot(); let policy_docs_cache = Arc::clone(&cache.policy_docs); @@ -1194,7 +1253,7 @@ where self.cache.add_or_update_user_policy(name, &mp, OffsetDateTime::now_utc()); } - Ok(OffsetDateTime::now_utc()) + Ok(updated_at) } pub async fn set_temp_user(&self, access_key: &str, cred: &Credentials, policy_name: Option<&str>) -> Result { @@ -1391,6 +1450,17 @@ where } pub async fn add_user(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result { + self.add_user_at(access_key, args, OffsetDateTime::now_utc()).await + } + + /// [`Self::add_user`] stamping the identity with `updated_at` instead of + /// the local clock; see [`Self::set_policy_at`] (backlog#2291). + pub async fn add_user_at( + &self, + access_key: &str, + args: &AddOrUpdateUserReq, + updated_at: OffsetDateTime, + ) -> Result { let cache = self.cache.snapshot(); let users = Arc::clone(&cache.users); if let Some(x) = users.get(access_key) { @@ -1408,12 +1478,13 @@ where _ => auth::ACCOUNT_OFF, } }; - let user_entry = UserIdentity::from(Credentials { + let mut user_entry = UserIdentity::from(Credentials { access_key: access_key.to_string(), secret_key: args.secret_key.to_string(), status: status.to_owned(), ..Default::default() }); + user_entry.update_at = Some(updated_at); self.api .save_user_identity(access_key, UserType::Reg, user_entry.clone(), None) @@ -1421,7 +1492,7 @@ where self.update_user_with_claims(access_key, user_entry)?; - Ok(OffsetDateTime::now_utc()) + Ok(updated_at) } pub async fn delete_user(&self, access_key: &str, utype: UserType) -> Result<()> { @@ -1599,6 +1670,17 @@ where } pub async fn set_user_status(&self, access_key: &str, status: AccountStatus) -> Result { + self.set_user_status_at(access_key, status, OffsetDateTime::now_utc()).await + } + + /// [`Self::set_user_status`] stamping the identity with `updated_at` + /// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291). + pub async fn set_user_status_at( + &self, + access_key: &str, + status: AccountStatus, + updated_at: OffsetDateTime, + ) -> Result { if access_key.is_empty() { return Err(Error::InvalidArgument); } @@ -1625,12 +1707,13 @@ where } }; - let user_entry = UserIdentity::from(Credentials { + let mut user_entry = UserIdentity::from(Credentials { access_key: access_key.to_string(), secret_key: u.credentials.secret_key.clone(), status: status.to_owned(), ..Default::default() }); + user_entry.update_at = Some(updated_at); drop(cache); drop(users); @@ -1640,7 +1723,7 @@ where self.update_user_with_claims(access_key, user_entry)?; - Ok(OffsetDateTime::now_utc()) + Ok(updated_at) } fn update_user_with_claims(&self, k: &str, u: UserIdentity) -> Result<()> { @@ -1676,6 +1759,17 @@ where } pub async fn add_users_to_group(&self, group: &str, members: Vec) -> Result { + self.add_users_to_group_at(group, members, OffsetDateTime::now_utc()).await + } + + /// [`Self::add_users_to_group`] stamping the group with `updated_at` + /// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291). + pub async fn add_users_to_group_at( + &self, + group: &str, + members: Vec, + updated_at: OffsetDateTime, + ) -> Result { if group.is_empty() { return Err(Error::InvalidArgument); } @@ -1693,6 +1787,14 @@ where } } + // The group's own timestamp moves with every membership or status + // change: site replication judges an incoming group item against it + // (backlog#2291), so it must reflect the last change, not creation. + // `updated_at` is the record's stamp only; the cache is published + // with the local clock, because `LockedCache::exec` drops a write + // whose time predates the entity's load time — a replicated edit + // whose source time is older than this node's startup would + // otherwise never reach the cache. let gi = match cache.groups.get(group) { Some(res) => { let mut gi = res.clone(); @@ -1701,15 +1803,20 @@ where uniq_set.extend(members.iter().cloned()); gi.members = uniq_set.into_iter().collect(); + gi.update_at = Some(updated_at); + gi + } + None => { + let mut gi = GroupInfo::new(members.clone()); + gi.update_at = Some(updated_at); gi } - None => GroupInfo::new(members.clone()), }; drop(cache); self.api.save_group_info(group, gi.clone()).await?; - let now = self.cache.with_write_lock(|cache| { + self.cache.with_write_lock(|cache| { let now = OffsetDateTime::now_utc(); cache.add_or_update_group(group, &gi, now); @@ -1719,13 +1826,18 @@ where m.insert(group.to_string()); cache.add_or_update_user_group_membership(member, &m, now); }); - now }); - Ok(now) + Ok(updated_at) } pub async fn set_group_status(&self, name: &str, enable: bool) -> Result { + self.set_group_status_at(name, enable, OffsetDateTime::now_utc()).await + } + + /// [`Self::set_group_status`] stamping the group with `updated_at` instead + /// of the local clock; see [`Self::set_policy_at`] (backlog#2291). + pub async fn set_group_status_at(&self, name: &str, enable: bool, updated_at: OffsetDateTime) -> Result { if name.is_empty() { return Err(Error::InvalidArgument); } @@ -1743,12 +1855,15 @@ where } else { gi.status = STATUS_DISABLED.to_owned(); } + gi.update_at = Some(updated_at); self.api.save_group_info(name, gi.clone()).await?; + // Cache publication time is the local clock, not the record stamp + // (see `add_users_to_group_at`). self.cache.add_or_update_group(name, &gi, OffsetDateTime::now_utc()); - Ok(OffsetDateTime::now_utc()) + Ok(updated_at) } pub async fn get_group_description(&self, name: &str) -> Result { @@ -1818,6 +1933,20 @@ where name: &str, members: Vec, update_cache_only: bool, + ) -> Result { + self.remove_members_from_group_at(name, members, update_cache_only, OffsetDateTime::now_utc()) + .await + } + + /// [`Self::remove_members_from_group`] stamping the group with + /// `updated_at` instead of the local clock; see [`Self::set_policy_at`] + /// (backlog#2291). + pub async fn remove_members_from_group_at( + &self, + name: &str, + members: Vec, + update_cache_only: bool, + updated_at: OffsetDateTime, ) -> Result { let cache = self.cache.snapshot(); let mut gi = cache @@ -1830,12 +1959,14 @@ where let s: HashSet<&String> = HashSet::from_iter(gi.members.iter()); let d: HashSet<&String> = HashSet::from_iter(members.iter()); gi.members = s.difference(&d).map(|v| v.to_string()).collect::>(); - + gi.update_at = Some(updated_at); if !update_cache_only { self.api.save_group_info(name, gi.clone()).await?; } - let now = self.cache.with_write_lock(|cache| { + self.cache.with_write_lock(|cache| { + // Sample after storage completes so a concurrent reload cannot + // make this publication older than the cache it must update. let now = OffsetDateTime::now_utc(); cache.add_or_update_group(name, &gi, now); @@ -1847,13 +1978,25 @@ where cache.add_or_update_user_group_membership(member, &m, now); } }); - now }); - Ok(now) + Ok(updated_at) } pub async fn remove_users_from_group(&self, group: &str, members: Vec) -> Result { + self.remove_users_from_group_at(group, members, OffsetDateTime::now_utc()) + .await + } + + /// [`Self::remove_users_from_group`] stamping the group with `updated_at` + /// instead of the local clock; a group delete (no members) leaves no + /// record and returns the stamp unchanged (backlog#2291). + pub async fn remove_users_from_group_at( + &self, + group: &str, + members: Vec, + updated_at: OffsetDateTime, + ) -> Result { if group.is_empty() { return Err(Error::InvalidArgument); } @@ -1902,18 +2045,17 @@ where return Err(err); } - let now = self.cache.with_write_lock(|cache| { + self.cache.with_write_lock(|cache| { let now = OffsetDateTime::now_utc(); self.remove_group_from_memberships_map_unlocked(cache, group, now); cache.delete_group(group, now); cache.delete_group_policy(group, now); - now }); - return Ok(now); + return Ok(updated_at); } - self.remove_members_from_group(group, members, false).await + self.remove_members_from_group_at(group, members, false, updated_at).await } fn remove_group_from_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, now: OffsetDateTime) { @@ -2235,6 +2377,19 @@ where } } +/// The stored `status` flag for a service-account status given on the admin +/// or replication wire: the madmin `enabled` / `disabled` words and the stored +/// `on` / `off` flags are both accepted; anything else disables the account. +pub(crate) fn account_status_flag(status: &str) -> &'static str { + match status { + val if val == AccountStatus::Enabled.as_ref() => auth::ACCOUNT_ON, + val if val == AccountStatus::Disabled.as_ref() => auth::ACCOUNT_OFF, + auth::ACCOUNT_ON => auth::ACCOUNT_ON, + auth::ACCOUNT_OFF => auth::ACCOUNT_OFF, + _ => auth::ACCOUNT_OFF, + } +} + pub fn get_default_policies() -> HashMap { let default_policies = &DEFAULT_POLICIES; default_policies diff --git a/crates/iam/src/sys.rs b/crates/iam/src/sys.rs index cd38d5f4e..5b0a58c92 100644 --- a/crates/iam/src/sys.rs +++ b/crates/iam/src/sys.rs @@ -385,7 +385,14 @@ impl IamSys { } pub async fn set_policy(&self, name: &str, policy: Policy) -> Result { - let updated_at = self.store.set_policy(name, policy).await?; + self.set_policy_at(name, policy, OffsetDateTime::now_utc()).await + } + + /// [`Self::set_policy`] stamping the document with `updated_at` (a + /// replicated edit's source time) instead of the local clock; see + /// `IamCache::set_policy_at` (backlog#2291). + pub async fn set_policy_at(&self, name: &str, policy: Policy, updated_at: OffsetDateTime) -> Result { + let updated_at = self.store.set_policy_at(name, policy, updated_at).await?; if !self.has_watcher() { for r in notify_iam_load_policy(name).await { @@ -643,7 +650,18 @@ impl IamSys { } pub async fn set_user_status(&self, name: &str, status: rustfs_madmin::AccountStatus) -> Result { - let updated_at = self.store.set_user_status(name, status).await?; + self.set_user_status_at(name, status, OffsetDateTime::now_utc()).await + } + + /// [`Self::set_user_status`] stamping the identity with `updated_at` (a + /// replicated edit's source time) instead of the local clock (backlog#2291). + pub async fn set_user_status_at( + &self, + name: &str, + status: rustfs_madmin::AccountStatus, + updated_at: OffsetDateTime, + ) -> Result { + let updated_at = self.store.set_user_status_at(name, status, updated_at).await?; self.notify_for_user(name, false).await; @@ -655,6 +673,20 @@ impl IamSys { parent_user: &str, groups: Option>, opts: NewServiceAccountOpts, + ) -> Result<(Credentials, OffsetDateTime)> { + self.new_service_account_at(parent_user, groups, opts, OffsetDateTime::now_utc()) + .await + } + + /// [`Self::new_service_account`] stamping the identity with `updated_at` + /// (a replicated edit's source time) instead of the local clock + /// (backlog#2291). + pub async fn new_service_account_at( + &self, + parent_user: &str, + groups: Option>, + opts: NewServiceAccountOpts, + updated_at: OffsetDateTime, ) -> Result<(Credentials, OffsetDateTime)> { if parent_user.is_empty() { return Err(IamError::InvalidArgument); @@ -724,11 +756,18 @@ impl IamSys { let mut cred = create_new_credentials_with_metadata(&access_key, &secret_key, &m, &secret_key)?; cred.parent_user = parent_user.to_owned(); cred.groups = groups; - cred.status = ACCOUNT_ON.to_owned(); + // The status is part of the created identity: a replicated disabled + // account must never exist enabled, not even between a create and a + // follow-up status write (backlog#2289). + cred.status = opts + .status + .as_deref() + .map_or(ACCOUNT_ON, crate::manager::account_status_flag) + .to_owned(); cred.name = opts.name; cred.description = opts.description; - let create_at = self.store.add_service_account(cred.clone()).await?; + let create_at = self.store.add_service_account_at(cred.clone(), updated_at).await?; self.notify_for_service_account(&cred.access_key).await; @@ -736,11 +775,23 @@ impl IamSys { } pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result { + self.update_service_account_at(name, opts, OffsetDateTime::now_utc()).await + } + + /// [`Self::update_service_account`] stamping the identity with + /// `updated_at` (a replicated edit's source time) instead of the local + /// clock (backlog#2291). + pub async fn update_service_account_at( + &self, + name: &str, + opts: UpdateServiceAccountOpts, + updated_at: OffsetDateTime, + ) -> Result { if name == SITE_REPLICATOR_SERVICE_ACCOUNT && !opts.allow_site_replicator_account { return Err(IamError::IAMActionNotAllowed); } - let updated_at = self.store.update_service_account(name, opts).await?; + let updated_at = self.store.update_service_account_at(name, opts, updated_at).await?; self.notify_for_service_account(name).await; @@ -940,6 +991,17 @@ impl IamSys { } pub async fn create_user(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result { + self.create_user_at(access_key, args, OffsetDateTime::now_utc()).await + } + + /// [`Self::create_user`] stamping the identity with `updated_at` (a + /// replicated edit's source time) instead of the local clock (backlog#2291). + pub async fn create_user_at( + &self, + access_key: &str, + args: &AddOrUpdateUserReq, + updated_at: OffsetDateTime, + ) -> Result { if !is_access_key_valid(access_key) { return Err(IamError::InvalidAccessKeyLength); } @@ -952,7 +1014,7 @@ impl IamSys { return Err(IamError::InvalidSecretKeyLength); } - let updated_at = self.store.add_user(access_key, args).await?; + let updated_at = self.store.add_user_at(access_key, args, updated_at).await?; self.load_user(access_key, UserType::Reg).await?; self.notify_for_user(access_key, false).await; @@ -1026,10 +1088,21 @@ impl IamSys { } pub async fn add_users_to_group(&self, group: &str, users: Vec) -> Result { + self.add_users_to_group_at(group, users, OffsetDateTime::now_utc()).await + } + + /// [`Self::add_users_to_group`] stamping the group with `updated_at` (a + /// replicated edit's source time) instead of the local clock (backlog#2291). + pub async fn add_users_to_group_at( + &self, + group: &str, + users: Vec, + updated_at: OffsetDateTime, + ) -> Result { if contains_reserved_chars(group) { return Err(IamError::GroupNameContainsReservedChars); } - let updated_at = self.store.add_users_to_group(group, users).await?; + let updated_at = self.store.add_users_to_group_at(group, users, updated_at).await?; self.notify_for_group(group).await; @@ -1037,7 +1110,19 @@ impl IamSys { } pub async fn remove_users_from_group(&self, group: &str, users: Vec) -> Result { - let updated_at = self.store.remove_users_from_group(group, users).await?; + self.remove_users_from_group_at(group, users, OffsetDateTime::now_utc()).await + } + + /// [`Self::remove_users_from_group`] stamping the group with `updated_at` + /// (a replicated edit's source time) instead of the local clock + /// (backlog#2291). + pub async fn remove_users_from_group_at( + &self, + group: &str, + users: Vec, + updated_at: OffsetDateTime, + ) -> Result { + let updated_at = self.store.remove_users_from_group_at(group, users, updated_at).await?; self.notify_for_group(group).await; @@ -1045,7 +1130,13 @@ impl IamSys { } pub async fn set_group_status(&self, group: &str, enable: bool) -> Result { - let updated_at = self.store.set_group_status(group, enable).await?; + self.set_group_status_at(group, enable, OffsetDateTime::now_utc()).await + } + + /// [`Self::set_group_status`] stamping the group with `updated_at` (a + /// replicated edit's source time) instead of the local clock (backlog#2291). + pub async fn set_group_status_at(&self, group: &str, enable: bool, updated_at: OffsetDateTime) -> Result { + let updated_at = self.store.set_group_status_at(group, enable, updated_at).await?; self.notify_for_group(group).await; @@ -1055,6 +1146,22 @@ impl IamSys { self.store.get_group_description(group).await } + /// The stored group record itself (see `IamCache::get_group_info`). + pub async fn get_group_info(&self, group: &str) -> Option { + self.store.get_group_info(group).await + } + + /// The stored policy document, `Error::NoSuchPolicy` when absent. + pub async fn get_policy_doc(&self, name: &str) -> Result { + self.store.get_policy_doc(name).await + } + + /// The stored mapping record for one user or group (see + /// `IamCache::get_mapped_policy_record`). + pub async fn get_mapped_policy_record(&self, name: &str, user_type: UserType, is_group: bool) -> Option { + self.store.get_mapped_policy_record(name, user_type, is_group).await + } + pub async fn list_groups_load(&self) -> Result> { self.store.update_groups().await } @@ -1064,7 +1171,24 @@ impl IamSys { } pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result { - let updated_at = self.store.policy_db_set(name, user_type, is_group, policy).await?; + self.policy_db_set_at(name, user_type, is_group, policy, OffsetDateTime::now_utc()) + .await + } + + /// [`Self::policy_db_set`] stamping the mapping with `updated_at` (a + /// replicated edit's source time) instead of the local clock (backlog#2291). + pub async fn policy_db_set_at( + &self, + name: &str, + user_type: UserType, + is_group: bool, + policy: &str, + updated_at: OffsetDateTime, + ) -> Result { + let updated_at = self + .store + .policy_db_set_at(name, user_type, is_group, policy, updated_at) + .await?; if !self.has_watcher() { for r in notify_iam_load_policy_mapping(name, user_type.to_u64(), is_group).await { @@ -1846,6 +1970,11 @@ pub struct NewServiceAccountOpts { pub expiration: Option, pub allow_site_replicator_account: bool, pub claims: Option>, + /// Status the account is created with (`enabled` / `disabled` or the + /// stored `on` / `off` flags); `None` creates it enabled. Site + /// replication passes the source account's status so a disabled account + /// is never enabled on the peer, not even transiently (backlog#2289). + pub status: Option, } pub struct UpdateServiceAccountOpts { @@ -2081,6 +2210,9 @@ mod tests { block_delete: Arc, delete_started: Arc, release_delete: Arc, + block_group_save: Arc, + group_save_started: Arc, + group_save_release: Arc, } impl StsTestMockStore { @@ -2094,6 +2226,9 @@ mod tests { block_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)), delete_started: Arc::new(tokio::sync::Notify::new()), release_delete: Arc::new(tokio::sync::Notify::new()), + block_group_save: Arc::new(std::sync::atomic::AtomicBool::new(false)), + group_save_started: Arc::new(tokio::sync::Notify::new()), + group_save_release: Arc::new(tokio::sync::Notify::new()), } } @@ -2197,11 +2332,15 @@ mod tests { } async fn save_group_info(&self, _name: &str, _item: GroupInfo) -> Result<()> { - Err(Error::InvalidArgument) + if self.block_group_save.load(std::sync::atomic::Ordering::SeqCst) { + self.group_save_started.notify_one(); + self.group_save_release.notified().await; + } + Ok(()) } async fn delete_group_info(&self, _name: &str) -> Result<()> { - Err(Error::InvalidArgument) + Ok(()) } async fn load_group(&self, name: &str, m: &mut HashMap) -> Result<()> { @@ -2378,6 +2517,140 @@ mod tests { IamSys::new(cache) } + async fn assert_group_write_during_reload_is_published(remove: bool) { + let iam_sys = Arc::new(temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], test_iam_sys()).await); + let member = "sts-fallback-test-parent"; + let group = if remove { "testgroup" } else { "new-published-group" }; + let source_time = OffsetDateTime::now_utc() - time::Duration::hours(1); + iam_sys + .store + .api + .block_group_save + .store(true, std::sync::atomic::Ordering::SeqCst); + let before = iam_sys.store.cache.snapshot(); + let writer_iam = iam_sys.clone(); + let writer = tokio::spawn(async move { + if remove { + writer_iam + .remove_users_from_group_at(group, vec![member.to_string()], source_time) + .await + } else { + writer_iam + .add_users_to_group_at(group, vec![member.to_string()], source_time) + .await + } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), iam_sys.store.api.group_save_started.notified()) + .await + .expect("group save should reach the barrier"); + // The pending store write has not changed the cache, so the production + // full-reload snapshot guard permits this replacement. + assert!(iam_sys.store.cache.with_write_lock(|cache| cache.matches_snapshot(&before))); + iam_sys + .store + .api + .load_all(&iam_sys.store.cache) + .await + .expect("reload while group save is pending"); + iam_sys.store.api.group_save_release.notify_one(); + assert_eq!(writer.await.expect("join group writer").expect("group write should succeed"), source_time); + let info = iam_sys + .get_group_info(group) + .await + .expect("successful group write must remain readable after reload"); + assert_eq!(info.update_at, Some(source_time), "source timestamp must remain on the record"); + assert_eq!(info.members, if remove { Vec::new() } else { vec![member.to_string()] }); + let groups = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned(); + assert_eq!( + groups.is_some_and(|groups| groups.contains(group)), + !remove, + "membership index must reflect the write" + ); + } + + #[tokio::test] + #[serial] + async fn add_group_write_during_reload_publishes_after_store_save() { + assert_group_write_during_reload_is_published(false).await; + } + + #[tokio::test] + #[serial] + async fn remove_group_write_during_reload_publishes_after_store_save() { + assert_group_write_during_reload_is_published(true).await; + } + + /// Review finding on rustfs#7195: a replicated group edit carries a source + /// stamp that may predate this node's cache load time. The stamp belongs on + /// the record only; publishing the cache with it makes `LockedCache::exec` + /// drop the write, so the group is written to the store but unreadable + /// here and the receiver's next `set_group_status_at` fails with + /// `NoSuchGroup`. Add, status and removal must all publish with the local + /// clock while keeping the source stamp on `GroupInfo::update_at`. + #[tokio::test] + async fn group_writes_stamped_before_the_cache_load_time_still_publish() { + let iam_sys = test_iam_sys().await; + let member = "group-stamp-member"; + let identity = UserIdentity { + version: 1, + credentials: Credentials { + access_key: member.to_string(), + secret_key: "longenoughsecret".to_string(), + status: "on".to_string(), + ..Default::default() + }, + update_at: Some(OffsetDateTime::now_utc()), + }; + iam_sys.store.cache.with_write_lock(|cache| { + cache.add_or_update_user(member, &identity, OffsetDateTime::now_utc()); + // The startup load publishes every entity with the load time. + cache.replace_groups(CacheEntity::new(HashMap::new())); + cache.replace_user_group_memberships(CacheEntity::new(HashMap::new())); + }); + + let group = "group-stamp"; + let source_time = OffsetDateTime::now_utc() - time::Duration::hours(1); + let stamped = iam_sys + .add_users_to_group_at(group, vec![member.to_string()], source_time) + .await + .expect("add members with a source stamp older than the cache load"); + assert_eq!(stamped, source_time, "the returned stamp is the source time"); + let info = iam_sys + .get_group_info(group) + .await + .expect("the group must be readable right after the add"); + assert_eq!(info.members, vec![member.to_string()]); + assert_eq!(info.update_at, Some(source_time), "the record keeps the source stamp"); + let memberships = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned(); + assert!( + memberships.is_some_and(|groups| groups.contains(group)), + "the membership index is published too" + ); + + let disabled_at = source_time + time::Duration::seconds(1); + iam_sys + .set_group_status_at(group, false, disabled_at) + .await + .expect("status change with a source stamp older than the cache load"); + let info = iam_sys.get_group_info(group).await.expect("group after status change"); + assert_eq!(info.status, "disabled"); + assert_eq!(info.update_at, Some(disabled_at)); + + let removed_at = source_time + time::Duration::seconds(2); + iam_sys + .remove_users_from_group_at(group, vec![member.to_string()], removed_at) + .await + .expect("removal with a source stamp older than the cache load"); + let info = iam_sys.get_group_info(group).await.expect("group after removal"); + assert!(info.members.is_empty(), "the removal must be visible in the cache"); + assert_eq!(info.update_at, Some(removed_at)); + let memberships = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned(); + assert!( + !memberships.is_some_and(|groups| groups.contains(group)), + "the membership index follows the removal" + ); + } + fn service_account_opts(access_key: &str, secret_key: &str) -> NewServiceAccountOpts { NewServiceAccountOpts { access_key: access_key.to_string(), diff --git a/crates/policy/src/policy/doc.rs b/crates/policy/src/policy/doc.rs index dc2b83fa9..30ef4ca03 100644 --- a/crates/policy/src/policy/doc.rs +++ b/crates/policy/src/policy/doc.rs @@ -45,18 +45,33 @@ pub struct PolicyDoc { impl PolicyDoc { pub fn new(policy: Policy) -> Self { + Self::new_at(policy, OffsetDateTime::now_utc()) + } + + /// [`Self::new`] with an explicit `UpdateDate` (and `CreateDate`). + /// + /// A replicated document keeps the edit's source time: the receiver + /// judges the next incoming revision against the stored stamp, so a + /// local stamp would reject a newer source edit that was merely + /// delivered later. + pub fn new_at(policy: Policy, at: OffsetDateTime) -> Self { Self { version: 1, policy, - create_date: Some(OffsetDateTime::now_utc()), - update_date: Some(OffsetDateTime::now_utc()), + create_date: Some(at), + update_date: Some(at), } } pub fn update(&mut self, policy: Policy) { + self.update_at(policy, OffsetDateTime::now_utc()); + } + + /// [`Self::update`] with an explicit `UpdateDate`; see [`Self::new_at`]. + pub fn update_at(&mut self, policy: Policy, at: OffsetDateTime) { self.version += 1; self.policy = policy; - self.update_date = Some(OffsetDateTime::now_utc()); + self.update_date = Some(at); if self.create_date.is_none() { self.create_date = self.update_date; diff --git a/crates/replication/src/delete.rs b/crates/replication/src/delete.rs index ffd014796..88a492533 100644 --- a/crates/replication/src/delete.rs +++ b/crates/replication/src/delete.rs @@ -76,6 +76,21 @@ impl ReplicationWorkerOperation for DeletedObjectReplicationInfo { .delete_object .delete_marker_mtime .and_then(|t| i64::try_from(t.unix_timestamp_nanos()).ok()), + // Carry the target-assigned marker version ids (and the fail-closed corrupt + // flag) into the journal so a purge intent replayed after a restart addresses + // the same version the live path did (backlog#2290). Only delete-marker state + // ever records these; other deletes serialize an empty map. + target_delete_marker_version_ids: self + .delete_object + .replication_state + .as_ref() + .map(|state| state.target_delete_marker_version_ids.clone()) + .unwrap_or_default(), + target_delete_marker_version_ids_corrupt: self + .delete_object + .replication_state + .as_ref() + .is_some_and(|state| state.target_delete_marker_version_ids_corrupt), target_arns: self.admitted_target_arns(), force_delete_id: self.delete_object.force_delete_id, force_delete_generation: self.delete_object.force_delete_generation, @@ -238,6 +253,28 @@ pub fn delete_marker_purge_version_id( }) } +/// The version a delete replication addresses on `arn`, or `None` to refuse. +/// +/// A version purge whose purged version is a delete marker must address the +/// marker version the TARGET assigned — the recorded mapping, exactly as the +/// delayed-purge watcher does. The source-side `DELETE ?versionId=` +/// replicates as such a purge, and a generic S3 target answers a DELETE of an +/// unknown versionId with 204 while keeping its marker, so addressing it by +/// the source id reported success and left the marker behind (backlog#2290, +/// R6.1 on the VMs). Nothing recorded falls back to the source-derived id +/// (id-mirroring peers); a corrupt record refuses, as the watcher does. +pub fn delete_replication_target_version_id(dobj: &DeletedObject, arn: &str) -> Option> { + let is_version_purge = is_version_delete_replication(dobj); + if is_version_purge + && !dobj.delete_marker + && let Some(marker) = dobj.delete_marker_version_id + { + return delete_marker_purge_version_id(dobj.replication_state.as_ref(), arn, marker); + } + let source_version = dobj.delete_marker_version_id.or(dobj.version_id).unwrap_or_default(); + Some(target_delete_version_id(source_version, is_version_purge)) +} + /// Shape an exhausted purge intent as a marker-creation delete entry. Replay /// reconstructs it with `delete_marker: true`, finds the source marker gone, /// and funnels into the stale-marker branch of `replicate_delete_with_outcome` @@ -258,9 +295,9 @@ mod tests { use super::{ DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, - delete_replication_creates_marker, is_object_lock_denied_delete, is_retryable_delete_replication_head_error, - is_version_delete_replication, replicate_delete_outcome, resync_existing_delete_replication_info, - should_retry_delete_marker_purge, target_delete_version_id, + delete_replication_creates_marker, delete_replication_target_version_id, is_object_lock_denied_delete, + is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, + resync_existing_delete_replication_info, should_retry_delete_marker_purge, target_delete_version_id, }; use crate::storage_api::DeletedObject; use crate::{ @@ -595,6 +632,76 @@ mod tests { assert_eq!(entry.retry_count, 0); assert_eq!(entry.bucket, "bucket-a"); assert_eq!(entry.object, "doc.txt"); + assert!( + entry.target_delete_marker_version_ids.is_empty(), + "no recorded target marker ids means the journal carries none" + ); + assert!(!entry.target_delete_marker_version_ids_corrupt); + } + + /// backlog#2290: a purge intent journaled to MRF must carry the marker + /// version ids the targets assigned, plus the fail-closed corrupt flag, + /// so a replay after restart addresses the same version the live path did. + #[test] + fn delete_marker_purge_mrf_entry_carries_target_assigned_marker_versions() { + let delete_marker_version_id = Uuid::new_v4(); + let mut state = ReplicationState::default(); + state + .target_delete_marker_version_ids + .insert("arn:a".to_string(), "remote-marker-a".to_string()); + state + .target_delete_marker_version_ids + .insert("arn:b".to_string(), "remote-marker-b".to_string()); + let mut dobj = DeletedObjectReplicationInfo { + delete_object: DeletedObject { + object_name: "doc.txt".to_string(), + delete_marker: false, + version_id: Some(Uuid::new_v4()), + delete_marker_version_id: Some(delete_marker_version_id), + replication_state: Some(state), + ..Default::default() + }, + bucket: "bucket-a".to_string(), + ..Default::default() + }; + + let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]); + assert_eq!( + entry.target_delete_marker_version_ids, + HashMap::from([ + ("arn:a".to_string(), "remote-marker-a".to_string()), + ("arn:b".to_string(), "remote-marker-b".to_string()), + ]), + "every recorded target marker id survives the journal, regardless of the retried ARN subset" + ); + assert!(!entry.target_delete_marker_version_ids_corrupt); + assert_eq!( + delete_marker_purge_version_id( + Some(&ReplicationState { + target_delete_marker_version_ids: entry.target_delete_marker_version_ids, + ..Default::default() + }), + "arn:a", + delete_marker_version_id + ), + Some(Some("remote-marker-a".to_string())) + ); + + // The live path refuses to purge on inconsistent metadata and reports the target + // as failed; the journaled intent must keep refusing after a restart. + dobj.delete_object + .replication_state + .as_mut() + .expect("state was set above") + .target_delete_marker_version_ids_corrupt = true; + let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]); + assert!(entry.target_delete_marker_version_ids_corrupt); + + // A delete without replication state journals an empty map. + dobj.delete_object.replication_state = None; + let entry = dobj.to_mrf_entry(); + assert!(entry.target_delete_marker_version_ids.is_empty()); + assert!(!entry.target_delete_marker_version_ids_corrupt); } #[test] @@ -656,4 +763,57 @@ mod tests { assert!(!is_object_lock_denied_delete(Some("InternalError"), Some("retention lookup failed"))); assert!(!is_object_lock_denied_delete(None, Some("legal hold"))); } + + fn purge_of_marker(marker: Uuid, state: Option) -> DeletedObject { + DeletedObject { + object_name: "obj".to_string(), + delete_marker: false, + delete_marker_version_id: Some(marker), + version_id: None, + replication_state: state, + ..Default::default() + } + } + + #[test] + fn delete_replication_target_version_id_addresses_recorded_marker_for_purges() { + let arn = "arn:minio:replication::generic:photos"; + let marker = Uuid::new_v4(); + let mut state = ReplicationState::default(); + state + .target_delete_marker_version_ids + .insert(arn.to_string(), "remote-marker".to_string()); + + // purge of a replicated marker: the target's own version + assert_eq!( + delete_replication_target_version_id(&purge_of_marker(marker, Some(state.clone())), arn), + Some(Some("remote-marker".to_string())) + ); + // nothing recorded for this arn: the source-derived id (id-mirroring peers) + assert_eq!( + delete_replication_target_version_id(&purge_of_marker(marker, None), arn), + Some(Some(marker.to_string())) + ); + // corrupt record: refuse instead of guessing + state.target_delete_marker_version_ids_corrupt = true; + assert_eq!(delete_replication_target_version_id(&purge_of_marker(marker, Some(state)), arn), None); + + // marker creation keeps the source id (the target mints its own on a + // versionless DELETE; the id only travels in the source header) + let creation = DeletedObject { + object_name: "obj".to_string(), + delete_marker: true, + delete_marker_version_id: Some(marker), + ..Default::default() + }; + assert_eq!(delete_replication_target_version_id(&creation, arn), Some(Some(marker.to_string()))); + // plain version purge: the source version id + let version = Uuid::new_v4(); + let purge = DeletedObject { + object_name: "obj".to_string(), + version_id: Some(version), + ..Default::default() + }; + assert_eq!(delete_replication_target_version_id(&purge, arn), Some(Some(version.to_string()))); + } } diff --git a/crates/replication/src/filemeta.rs b/crates/replication/src/filemeta.rs index 7af1b3141..555b58c57 100644 --- a/crates/replication/src/filemeta.rs +++ b/crates/replication/src/filemeta.rs @@ -641,6 +641,26 @@ pub struct MrfReplicateEntry { #[serde(rename = "deleteMarkerMtime", skip_serializing_if = "Option::is_none", default)] pub delete_marker_mtime: Option, + // For delete-marker purge intents: the exact version id each target assigned to the + // replicated marker, keyed by target ARN. A generic S3 target mints its own version ids + // and answers a DELETE of an unknown id with 204, so a replay that fell back to the source + // marker id would be acknowledged while the real marker stayed behind (backlog#2290). + // Old files lack this key; default=empty means "unknown" and replay keeps the source-id + // fallback it always had. + #[serde(rename = "targetDeleteMarkerVersionIDs", skip_serializing_if = "HashMap::is_empty", default)] + pub target_delete_marker_version_ids: HashMap, + + // Companion to the map above: the source metadata disagreed about the recorded ids when + // the intent was journaled, so the live path refused to guess and reported the target as + // failed. Replay must keep refusing instead of falling back to the source id. Old files + // lack this key; default=false. + #[serde( + rename = "targetDeleteMarkerVersionIDsCorrupt", + skip_serializing_if = "std::ops::Not::not", + default + )] + pub target_delete_marker_version_ids_corrupt: bool, + #[serde(rename = "targetARNs", skip_serializing_if = "Vec::is_empty", default)] pub target_arns: Vec, diff --git a/crates/replication/src/lib.rs b/crates/replication/src/lib.rs index 1aa20daca..295340026 100644 --- a/crates/replication/src/lib.rs +++ b/crates/replication/src/lib.rs @@ -41,9 +41,9 @@ pub use config::{ }; pub use delete::{ DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, - delete_replication_creates_marker, is_object_lock_denied_delete, is_retryable_delete_replication_head_error, - is_version_delete_replication, replicate_delete_outcome, resync_existing_delete_replication_info, - should_retry_delete_marker_purge, target_delete_version_id, + delete_replication_creates_marker, delete_replication_target_version_id, is_object_lock_denied_delete, + is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, + resync_existing_delete_replication_info, should_retry_delete_marker_purge, target_delete_version_id, }; pub use filemeta::{ NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING, diff --git a/crates/replication/src/mrf.rs b/crates/replication/src/mrf.rs index 8698e7c2a..285c8bf82 100644 --- a/crates/replication/src/mrf.rs +++ b/crates/replication/src/mrf.rs @@ -31,8 +31,13 @@ const CAPABILITY_OPERATION_KIND: u64 = 1 << 0; const CAPABILITY_TARGET_ARNS: u64 = 1 << 1; const CAPABILITY_FORCE_DELETE: u64 = 1 << 2; const CAPABILITY_DELETE_MARKER_MTIME: u64 = 1 << 3; -const MRF_KNOWN_CAPABILITIES: u64 = - CAPABILITY_OPERATION_KIND | CAPABILITY_TARGET_ARNS | CAPABILITY_FORCE_DELETE | CAPABILITY_DELETE_MARKER_MTIME; +// Per-ARN target-assigned delete-marker version ids on purge intents (backlog#2290). +const CAPABILITY_TARGET_DELETE_MARKER_VERSION_IDS: u64 = 1 << 4; +const MRF_KNOWN_CAPABILITIES: u64 = CAPABILITY_OPERATION_KIND + | CAPABILITY_TARGET_ARNS + | CAPABILITY_FORCE_DELETE + | CAPABILITY_DELETE_MARKER_MTIME + | CAPABILITY_TARGET_DELETE_MARKER_VERSION_IDS; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MrfCapability { @@ -40,6 +45,7 @@ pub enum MrfCapability { TargetArns, ForceDelete, DeleteMarkerMtime, + TargetDeleteMarkerVersionIds, } impl MrfCapability { @@ -49,6 +55,7 @@ impl MrfCapability { Self::TargetArns => CAPABILITY_TARGET_ARNS, Self::ForceDelete => CAPABILITY_FORCE_DELETE, Self::DeleteMarkerMtime => CAPABILITY_DELETE_MARKER_MTIME, + Self::TargetDeleteMarkerVersionIds => CAPABILITY_TARGET_DELETE_MARKER_VERSION_IDS, } } } @@ -601,9 +608,17 @@ pub fn decode_mrf_file(data: &[u8]) -> Result> { #[cfg(test)] mod tests { use super::*; + use std::collections::HashMap; use uuid::Uuid; + // Capability word 31 = OperationKind | TargetArns | ForceDelete | DeleteMarkerMtime | + // TargetDeleteMarkerVersionIds (backlog#2290). const ENVELOPE_FIXTURE: &[u8] = &[ + b'M', b'R', b'F', b'E', 1, 0, 1, 0, 1, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 2, 3, + ]; + + // The envelope a binary from before backlog#2290 writes: same header, capability word 15. + const PRE_TARGET_MARKER_IDS_ENVELOPE_FIXTURE: &[u8] = &[ b'M', b'R', b'F', b'E', 1, 0, 1, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 2, 3, ]; @@ -626,6 +641,8 @@ mod tests { delete_marker_version_id: None, delete_marker: false, delete_marker_mtime: None, + target_delete_marker_version_ids: HashMap::new(), + target_delete_marker_version_ids_corrupt: false, target_arns: vec!["arn:target-a".to_string()], }, MrfReplicateEntry { @@ -642,6 +659,8 @@ mod tests { delete_marker_version_id: None, delete_marker: false, delete_marker_mtime: None, + target_delete_marker_version_ids: HashMap::new(), + target_delete_marker_version_ids_corrupt: false, target_arns: vec!["arn:target-a".to_string(), "arn:target-b".to_string()], }, MrfReplicateEntry { @@ -658,6 +677,11 @@ mod tests { delete_marker_version_id: Some(del_vid), delete_marker: true, delete_marker_mtime: Some(1_705_312_200_123_456_789), + target_delete_marker_version_ids: HashMap::from([ + ("arn:target-a".to_string(), "remote-marker-a".to_string()), + ("arn:target-b".to_string(), "remote-marker-b".to_string()), + ]), + target_delete_marker_version_ids_corrupt: false, target_arns: vec!["arn:target-a".to_string()], }, ]; @@ -685,6 +709,54 @@ mod tests { Some(1_705_312_200_123_456_789), "delete-marker mtime must survive the MRF disk round-trip" ); + assert!(decoded[0].target_delete_marker_version_ids.is_empty()); + assert!(decoded[1].target_delete_marker_version_ids.is_empty()); + assert_eq!( + decoded[2].target_delete_marker_version_ids, + HashMap::from([ + ("arn:target-a".to_string(), "remote-marker-a".to_string()), + ("arn:target-b".to_string(), "remote-marker-b".to_string()), + ]), + "target-assigned marker version ids must survive the MRF disk round-trip (backlog#2290)" + ); + assert!(!decoded[2].target_delete_marker_version_ids_corrupt); + } + + /// backlog#2290: the corrupt flag rides the same journal round trip, and an + /// entry that carries neither field encodes exactly as it did before the + /// field existed (both keys are skipped when empty/false). + #[test] + fn mrf_file_round_trips_target_marker_ids_corrupt_flag_and_skips_empty_keys() { + let corrupt = MrfReplicateEntry { + bucket: "bucket-a".to_string(), + object: "delete-a".to_string(), + op: MrfOpKind::Delete, + delete_marker: true, + delete_marker_version_id: Some(Uuid::new_v4()), + target_delete_marker_version_ids_corrupt: true, + target_arns: vec!["arn:target-a".to_string()], + ..Default::default() + }; + let decoded = decode_mrf_file(&encode_mrf_file(std::slice::from_ref(&corrupt)).expect("mrf file should encode")) + .expect("mrf file should decode"); + assert_eq!(decoded, vec![corrupt]); + assert!(decoded[0].target_delete_marker_version_ids_corrupt); + + let plain = MrfReplicateEntry { + bucket: "bucket-a".to_string(), + object: "delete-a".to_string(), + op: MrfOpKind::Delete, + delete_marker: true, + target_arns: vec!["arn:target-a".to_string()], + ..Default::default() + }; + let encoded = encode_mrf_file(std::slice::from_ref(&plain)).expect("mrf file should encode"); + let payload = String::from_utf8_lossy(&encoded); + assert!( + !payload.contains("targetDeleteMarkerVersionIDs"), + "an entry without recorded ids must not grow the new keys: {payload}" + ); + assert_eq!(decode_mrf_file(&encoded).expect("mrf file should decode"), vec![plain]); } #[test] @@ -719,6 +791,99 @@ mod tests { // Old files lack the deleteMarkerMtime key; it must default to None so replay keeps the // pre-#867 fallback to the current time. assert_eq!(decoded[0].delete_marker_mtime, None); + // Old files also lack the target marker id keys; they must default to an empty map + // and a clear corrupt flag so replay keeps the pre-#2290 source-id fallback. + assert!(decoded[0].target_delete_marker_version_ids.is_empty()); + assert!(!decoded[0].target_delete_marker_version_ids_corrupt); + } + + /// backlog#2290: a delete-marker entry written by a binary that predates the + /// `targetDeleteMarkerVersionIDs` key decodes with an empty map and a clear + /// corrupt flag — the exact shape replay handled before the field existed. + #[test] + fn mrf_pre_target_marker_ids_delete_entry_decodes_with_empty_map() { + let marker_version_id = Uuid::new_v4(); + let mut payload = Vec::new(); + rmp::encode::write_array_len(&mut payload, 1).expect("array len should encode"); + rmp::encode::write_map_len(&mut payload, 9).expect("map len should encode"); + rmp::encode::write_str(&mut payload, "bucket").expect("bucket key should encode"); + rmp::encode::write_str(&mut payload, "old-bucket").expect("bucket value should encode"); + rmp::encode::write_str(&mut payload, "object").expect("object key should encode"); + rmp::encode::write_str(&mut payload, "old-key").expect("object value should encode"); + rmp::encode::write_str(&mut payload, "retryCount").expect("retry key should encode"); + rmp::encode::write_i32(&mut payload, 0).expect("retry value should encode"); + rmp::encode::write_str(&mut payload, "size").expect("size key should encode"); + rmp::encode::write_i64(&mut payload, 0).expect("size value should encode"); + rmp::encode::write_str(&mut payload, "op").expect("op key should encode"); + rmp::encode::write_str(&mut payload, "delete").expect("op value should encode"); + rmp::encode::write_str(&mut payload, "forceDelete").expect("forceDelete key should encode"); + rmp::encode::write_bool(&mut payload, false).expect("forceDelete value should encode"); + rmp::encode::write_str(&mut payload, "deleteMarkerVersionID").expect("marker id key should encode"); + // Uuid serializes as a 16-byte bin in the MessagePack journal. + rmp::encode::write_bin(&mut payload, marker_version_id.as_bytes()).expect("marker id value should encode"); + rmp::encode::write_str(&mut payload, "deleteMarker").expect("deleteMarker key should encode"); + rmp::encode::write_bool(&mut payload, true).expect("deleteMarker value should encode"); + rmp::encode::write_str(&mut payload, "targetARNs").expect("targetARNs key should encode"); + rmp::encode::write_array_len(&mut payload, 1).expect("targetARNs len should encode"); + rmp::encode::write_str(&mut payload, "arn:target-a").expect("targetARNs value should encode"); + + let mut data = Vec::with_capacity(4 + payload.len()); + data.extend_from_slice(&MRF_META_FORMAT.to_le_bytes()); + data.extend_from_slice(&MRF_META_VERSION.to_le_bytes()); + data.extend_from_slice(&payload); + + let decoded = decode_mrf_file(&data).expect("pre-#2290 delete-marker entry should decode"); + + assert_eq!(decoded.len(), 1); + assert_eq!(decoded[0].op, MrfOpKind::Delete); + assert!(decoded[0].delete_marker); + assert_eq!(decoded[0].delete_marker_version_id, Some(marker_version_id)); + assert_eq!(decoded[0].target_arns, vec!["arn:target-a".to_string()]); + assert!(decoded[0].target_delete_marker_version_ids.is_empty()); + assert!(!decoded[0].target_delete_marker_version_ids_corrupt); + } + + /// backlog#2290: the new field is fenced by its own capability bit exactly + /// like the earlier optional fields — a reader without the bit refuses an + /// envelope that advertises it, while the current reader still accepts the + /// pre-#2290 envelope. + #[test] + fn envelope_target_marker_ids_capability_is_fenced_and_backward_compatible() { + assert!(MrfCapabilities::current().contains(MrfCapability::TargetDeleteMarkerVersionIds)); + assert_eq!(MrfCapabilities::with(MrfCapability::TargetDeleteMarkerVersionIds).bits(), 1 << 4); + + // Old envelope, current reader: accepted, and the negotiated set lacks the new bit. + let legacy = MrfEnvelope::decode(PRE_TARGET_MARKER_IDS_ENVELOPE_FIXTURE, MrfProtocolCapabilities::current()) + .expect("pre-#2290 envelope should decode"); + assert_eq!(legacy.protocol().capabilities().bits(), 15); + assert!( + !legacy + .protocol() + .capabilities() + .contains(MrfCapability::TargetDeleteMarkerVersionIds) + ); + assert_eq!(legacy.payload(), &[1, 2, 3]); + + // Current envelope, reader that only knows the pre-#2290 bits: refused. + let pre_2290_reader = MrfProtocolCapabilities::new(1, 1, MrfCapabilities::from_bits(15).expect("known bits")); + assert_eq!( + MrfEnvelope::decode(ENVELOPE_FIXTURE, pre_2290_reader), + Err(MrfEnvelopeError::MissingCapabilities { + required: 31, + available: 15, + }) + ); + + // Negotiation with such a peer drops the bit instead of failing. + let negotiated = MrfProtocolCapabilities::current() + .negotiate(pre_2290_reader) + .expect("negotiation with a pre-#2290 peer should succeed"); + assert!( + !negotiated + .capabilities() + .contains(MrfCapability::TargetDeleteMarkerVersionIds) + ); + assert!(negotiated.capabilities().contains(MrfCapability::DeleteMarkerMtime)); } #[test] diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 51c7dda99..318c8a093 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -66,8 +66,8 @@ use rustfs_madmin::{ ReplicateRemoveStatus, ResyncBucketStatus, SITE_REPL_API_VERSION, SR_IAM_ITEM_STS_ACC, SR_IAM_ITEM_STS_ACC_LEGACY, SRBucketInfo, SRBucketMeta, SRBucketStatsSummary, SRGroupInfo, SRGroupStatsSummary, SRIAMItem, SRIAMUser, SRILMExpiryStatsSummary, SRInfo, SRMetric, SRMetricsSummary, SRPeerError, SRPeerJoinReq, SRPendingOperation, SRPolicyMapping, - SRPolicyStatsSummary, SRRemoveReq, SRResyncOpStatus, SRSTSCredential, SRSessionPolicy, SRSiteSummary, SRStateEditReq, - SRStateInfo, SRStatusInfo, SRSvcAccChange, SRSvcAccCreate, SRUserStatsSummary, SiteReplicationInfo, SyncStatus, WorkerStat, + SRPolicyStatsSummary, SRRemoveReq, SRResyncOpStatus, SRSTSCredential, SRSiteSummary, SRStateEditReq, SRStateInfo, + SRStatusInfo, SRSvcAccChange, SRSvcAccCreate, SRUserStatsSummary, SiteReplicationInfo, SyncStatus, WorkerStat, }; use rustfs_policy::policy::{ Policy, @@ -98,7 +98,6 @@ use uuid::Uuid; // paths keep resolving while this file keeps only the HTTP handlers. pub(crate) use crate::site_replication::*; -const SERVICE_ACCOUNT_ENVELOPE_VERSION: u64 = 2; // Serializes peer-join admission (staleness check -> IAM upsert -> state // commit) across every node of this site; see admit_peer_join. Never an // actual object — only a namespace-lock key, like the repair execution lock. @@ -1479,6 +1478,7 @@ async fn set_site_replicator_service_account_secret(parent_user: &str, secret_ke expiration: None, allow_site_replicator_account: true, claims: None, + status: None, }, ) .await @@ -1721,6 +1721,7 @@ async fn reconcile_site_replicator_service_account() -> S3Result<()> { expiration: None, allow_site_replicator_account: true, claims: None, + status: None, }, ) .await @@ -1980,7 +1981,7 @@ async fn bootstrap_existing_metadata_after_add( return errors; } }; - let plan = match site_replication_bootstrap_plan(&info) { + let plan = match build_site_replication_bootstrap_plan(&info).await { Ok(plan) => plan, Err(err) => { let mut errors = SiteReplicationErrorSummary::default(); @@ -3498,6 +3499,7 @@ fn remove_sites(mut state: SiteReplicationState, req: SRRemoveReq) -> SiteReplic state.resync_status.clear(); state.retry_queue.clear(); state.iam_deletion_replays.clear(); + state.iam_deletion_marks.clear(); state.pending_endpoint_refresh = None; state.updated_at = Some(OffsetDateTime::now_utc()); return state; @@ -3509,6 +3511,7 @@ fn remove_sites(mut state: SiteReplicationState, req: SRRemoveReq) -> SiteReplic state.resync_status.clear(); state.retry_queue.clear(); state.iam_deletion_replays.clear(); + state.iam_deletion_marks.clear(); state.pending_endpoint_refresh = None; state.updated_at = Some(OffsetDateTime::now_utc()); return state; @@ -5386,6 +5389,38 @@ fn is_stale_update(local_updated_at: OffsetDateTime, incoming_updated_at: Option incoming_updated_at.is_some_and(|incoming_updated_at| incoming_updated_at < local_updated_at) } +/// Verdict for an incoming IAM item judged against the local record it would +/// overwrite or delete (backlog#2291). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum IamItemVerdict { + /// Apply the item: there is no local record, the item carries no source + /// timestamp (older peer), or it is at least as new as the local record. + Apply, + /// The local record was written from a newer source change; acknowledge the + /// item without touching the record. Covers both directions: a delayed + /// grant must not undo a newer revoke, and a delayed revoke must not undo a + /// newer grant. + SkipStale, +} + +/// Ordering rule shared by the `policy`, `policy-mapping` and `group-info` +/// item paths (and matching `iam-user` / `service-account`). +/// +/// `local_record_updated_at` is `None` when the targeted record does not +/// exist locally: nothing can be stale relative to an absent record, so a +/// create is applied and a delete falls through to the idempotent no-op paths +/// (backlog#2071). A record that exists but predates timestamps passes +/// `Some(UNIX_EPOCH)` and therefore never rejects an item. +fn judge_iam_item_staleness( + local_record_updated_at: Option, + incoming_updated_at: Option, +) -> IamItemVerdict { + match local_record_updated_at { + Some(local_updated_at) if is_stale_update(local_updated_at, incoming_updated_at) => IamItemVerdict::SkipStale, + _ => IamItemVerdict::Apply, + } +} + fn bucket_meta_local_updated_at( bucket_meta: &crate::admin::storage_api::bucket::metadata::BucketMetadata, config_file: &str, @@ -5586,6 +5621,18 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { _ => unreachable!(), }; + // Persist the SOURCE `updated_at` as the stored `*_config_updated_at` + // stamp, on writes and on deletes alike (backlog#2292). The staleness + // gate above compares the next item's source time against that stamp, so + // stamping the local apply time would reject a newer source edit that was + // merely delivered after this write or delete (two quick edits under + // delivery delay, or a peer clock ahead of ours). + // Items without a source time keep the local stamp; lc-config keeps it + // too: its staleness axis is the in-document `expiry_updated_at` the merge + // above records, and the whole-config time is only its deletion / legacy + // lower bound. + let source_updated_at = if item.r#type == "lc-config" { None } else { item.updated_at }; + if !skip_config_write { if let Some(data) = data { if item.r#type == "quota-config" { @@ -5604,13 +5651,25 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { "durable quota capability is not confirmed across the cluster".to_string(), ) })?; - metadata_sys::update_quota_if_incarnation(&item.bucket, data, expected_incarnation_id, &proof) - .await - .map_err(ApiError::from)?; + match source_updated_at { + Some(source_updated_at) => { + metadata_sys::update_quota_if_incarnation_at( + &item.bucket, + data, + expected_incarnation_id, + &proof, + source_updated_at, + ) + .await + } + None => { + metadata_sys::update_quota_if_incarnation(&item.bucket, data, expected_incarnation_id, &proof).await + } + } + .map_err(ApiError::from)?; } else { - metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) - .await - .map_err(ApiError::from)?; + write_replicated_bucket_config(&item.bucket, config_file, data, expected_incarnation_id, source_updated_at) + .await?; } } else { if let Some(guard) = lifecycle_guard.as_ref() { @@ -5618,9 +5677,8 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { .await .map_err(ApiError::from)?; } else { - metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) - .await - .map_err(ApiError::from)?; + write_replicated_bucket_config(&item.bucket, config_file, data, expected_incarnation_id, source_updated_at) + .await?; } } } else { @@ -5629,9 +5687,23 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { .await .map_err(ApiError::from)?; } else { - metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id) - .await - .map_err(ApiError::from)?; + // A delete is stamped like a write: the source time survives + // as the config's `*_config_updated_at`, so a newer source + // re-create delivered later is not judged stale against the + // local time this delete landed (backlog#2292). + match source_updated_at { + Some(source_updated_at) => { + metadata_sys::delete_if_incarnation_at( + &item.bucket, + config_file, + expected_incarnation_id, + source_updated_at, + ) + .await + } + None => metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id).await, + } + .map_err(ApiError::from)?; } } } @@ -5656,43 +5728,28 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { Ok(()) } -fn group_info_requires_upsert(update: &rustfs_madmin::GroupAddRemove) -> bool { - !update.is_remove +/// Write one replicated bucket config, stamped with the item's source +/// `updated_at` when it carries one and with the local clock otherwise +/// (backlog#2292; see [`apply_bucket_meta_item`]). +async fn write_replicated_bucket_config( + bucket: &str, + config_file: &str, + data: Vec, + expected_incarnation_id: Uuid, + source_updated_at: Option, +) -> S3Result<()> { + match source_updated_at { + Some(source_updated_at) => { + metadata_sys::update_if_incarnation_at(bucket, config_file, data, expected_incarnation_id, source_updated_at).await + } + None => metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await, + } + .map_err(ApiError::from)?; + Ok(()) } -pub(crate) fn encode_service_account_replication_policy( - claims: &HashMap, - session_policy: Option<&str>, -) -> S3Result<(SRSessionPolicy, Option)> { - if !claims.contains_key(OIDC_VIRTUAL_PARENT_CLAIM) { - return session_policy - .map(SRSessionPolicy::from_json) - .transpose() - .map(|policy| policy.unwrap_or_default()) - .map(|policy| (policy, None)) - .map_err(|err| s3_error!(InvalidArgument, "marshal policy failed: {:?}", err)); - } - - let policy = match session_policy { - Some(policy) => serde_json::from_str::(policy) - .map_err(|err| s3_error!(InvalidArgument, "invalid service account replication policy: {:?}", err))?, - None => Policy::default(), - }; - if policy.statements.is_empty() && (!policy.id.is_empty() || !policy.version.is_empty()) - || policy.version.is_empty() && !policy.statements.is_empty() - { - return Err(s3_error!(InvalidArgument, "service account replication policy is not normalized")); - } - let policy = serde_json::to_string(&policy) - .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?; - let policy = SRSessionPolicy::from_json(&policy) - .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?; - Ok(( - policy, - Some(rustfs_madmin::SRSvcAccReplicationEnvelope { - version: SERVICE_ACCOUNT_ENVELOPE_VERSION, - }), - )) +fn group_info_requires_upsert(update: &rustfs_madmin::GroupAddRemove) -> bool { + !update.is_remove } #[derive(Debug)] @@ -5761,32 +5818,96 @@ async fn apply_iam_item(item: SRIAMItem) -> S3Result<()> { let Some(iam_sys) = current_iam_handle() else { return Err(s3_error!(InvalidRequest, "iam not init")); }; - let incoming_updated_at = item.updated_at; - match item.r#type.as_str() { - "policy" => apply_iam_policy_item(&iam_sys, &item.name, item.policy).await, - "policy-mapping" => apply_iam_policy_mapping_item(&iam_sys, item.policy_mapping).await, - "group-info" => apply_iam_group_info_item(&iam_sys, item.group_info).await, // MinIO madmin-go sends `SRIAMItemSTSAcc = "sts-account"`. The legacy alias // `sts-credential` (emitted by older RustFS releases) stays accepted permanently // so mixed-version RustFS sites keep replicating STS credentials during rolling // upgrades; it is a compatibility layer, not temporary code. - SR_IAM_ITEM_STS_ACC | SR_IAM_ITEM_STS_ACC_LEGACY => apply_iam_sts_account_item(&iam_sys, item.sts_credential).await, - "iam-user" => apply_iam_user_item(&iam_sys, item.iam_user, incoming_updated_at).await, - "service-account" => apply_iam_service_account_item(&iam_sys, item.svc_acc_change, incoming_updated_at).await, - _ => Err(s3_error!( - NotImplemented, - "site replication IAM item type `{}` is not supported", - item.r#type - )), + // + // STS credentials carry no source revision and leave no deletion mark, + // so they stay outside the ordered transaction below. + SR_IAM_ITEM_STS_ACC | SR_IAM_ITEM_STS_ACC_LEGACY => { + return apply_iam_sts_account_item(&iam_sys, item.sts_credential).await; + } + "policy" | "policy-mapping" | "group-info" | "iam-user" | "service-account" => {} + _ => { + return Err(s3_error!( + NotImplemented, + "site replication IAM item type `{}` is not supported", + item.r#type + )); + } } + + // One transaction per item (backlog#2291). The staleness verdict, the IAM + // write and the deletion-mark commit run under the distributed + // state-object lock, so an older grant and a newer revoke delivered + // concurrently — to this node or to a sibling node of this site — are + // applied one after the other, each judged against what the other left + // behind. The write stamps the record with the item's source + // `updated_at`, which is what the next item is judged against: stamping + // the local apply time would reject a newer source edit that was merely + // delivered later. A committed deletion leaves no record, so its source + // timestamp is kept as a mark in the same commit; failing to persist the + // mark fails the item, and the sender retries the (idempotent) deletion + // rather than leaving a revoke that a stale grant could still undo. + with_site_replication_state_transaction(move |mut state| async move { + let incoming_updated_at = item.updated_at; + let deletion_mark_entities = iam_item_deletion_mark_entities(&item); + let verdict = match item.r#type.as_str() { + "policy" => apply_iam_policy_item(&iam_sys, &state, &item.name, item.policy, incoming_updated_at).await?, + "policy-mapping" => apply_iam_policy_mapping_item(&iam_sys, &state, item.policy_mapping, incoming_updated_at).await?, + "group-info" => apply_iam_group_info_item(&iam_sys, &state, item.group_info, incoming_updated_at).await?, + "iam-user" => apply_iam_user_item(&iam_sys, &state, item.iam_user, incoming_updated_at).await?, + "service-account" => { + apply_iam_service_account_item(&iam_sys, &state, item.svc_acc_change, incoming_updated_at).await? + } + _ => unreachable!("unsupported IAM item types are rejected before the transaction"), + }; + let changed = verdict == IamItemVerdict::Apply + && incoming_updated_at + .filter(|_| !deletion_mark_entities.is_empty()) + .is_some_and(|deleted_at| record_iam_deletion_marks(&mut state, &deletion_mark_entities, deleted_at)); + Ok(((), changed.then_some(state))) + }) + .await } -async fn apply_iam_policy_item(iam_sys: &IamSys, name: &str, policy: Option) -> S3Result<()> { +/// The stamp a replicated write persists on the record: the item's source +/// `updated_at`, or the local clock for an item from a peer that predates +/// timestamps (those keep last-writer-wins, see [`judge_iam_item_staleness`]). +fn replicated_write_stamp(incoming_updated_at: Option) -> OffsetDateTime { + incoming_updated_at.unwrap_or_else(OffsetDateTime::now_utc) +} + +async fn apply_iam_policy_item( + iam_sys: &IamSys, + marks: &SiteReplicationState, + name: &str, + policy: Option, + incoming_updated_at: Option, +) -> S3Result { + // Judge the item against the local document's own timestamp — the source + // time of the edit that wrote it — so a delayed older body (or delete) + // cannot overwrite a newer edit; once the document is deleted, its + // deletion mark stands in for it (backlog#2291). + let local_updated_at = match iam_sys.get_policy_doc(name).await { + Ok(doc) => Some(doc.update_date.unwrap_or(OffsetDateTime::UNIX_EPOCH)), + Err(err) if rustfs_iam::error::is_err_no_such_policy(&err) => { + iam_deletion_mark(marks, &[iam_policy_deletion_mark_entity(name)]) + } + Err(err) => return Err(ApiError::from(err).into()), + }; + if judge_iam_item_staleness(local_updated_at, incoming_updated_at) == IamItemVerdict::SkipStale { + return Ok(IamItemVerdict::SkipStale); + } if let Some(policy) = policy { let policy: Policy = serde_json::from_value(policy).map_err(|e| s3_error!(InvalidRequest, "invalid policy body: {}", e))?; - iam_sys.set_policy(name, policy).await.map_err(ApiError::from)?; + iam_sys + .set_policy_at(name, policy, replicated_write_stamp(incoming_updated_at)) + .await + .map_err(ApiError::from)?; } else { // Idempotent delete: the retry drain replays recorded deletions, and // an entity already absent here IS the converged outcome — erroring @@ -5797,26 +5918,87 @@ async fn apply_iam_policy_item(iam_sys: &IamSys, name: &str, policy Err(err) => return Err(ApiError::from(err).into()), } } - Ok(()) + Ok(IamItemVerdict::Apply) } -async fn apply_iam_policy_mapping_item(iam_sys: &IamSys, policy_mapping: Option) -> S3Result<()> { +async fn apply_iam_policy_mapping_item( + iam_sys: &IamSys, + marks: &SiteReplicationState, + policy_mapping: Option, + incoming_updated_at: Option, +) -> S3Result { let Some(mapping) = policy_mapping else { return Err(s3_error!(InvalidRequest, "policyMapping is required")); }; let user_type = user_type_from_sr_wire(mapping.user_type).ok_or_else(|| s3_error!(InvalidRequest, "invalid userType"))?; + // Judge the item against the stored mapping's timestamp so a delayed older + // attach (or an older detach, `policy == ""`) cannot overwrite a newer one + // (backlog#2291). A detach removes the mapping outright, so once it is + // gone the detach's deletion mark stands in for the record. + let local_updated_at = match iam_sys + .get_mapped_policy_record(&mapping.user_or_group, user_type, mapping.is_group) + .await + { + Some(record) => Some(record.update_at), + None => iam_deletion_mark( + marks, + &[iam_policy_mapping_deletion_mark_entity( + &mapping.user_or_group, + mapping.user_type, + mapping.is_group, + )], + ), + }; + if judge_iam_item_staleness(local_updated_at, incoming_updated_at) == IamItemVerdict::SkipStale { + return Ok(IamItemVerdict::SkipStale); + } iam_sys - .policy_db_set(&mapping.user_or_group, user_type, mapping.is_group, &mapping.policy) + .policy_db_set_at( + &mapping.user_or_group, + user_type, + mapping.is_group, + &mapping.policy, + replicated_write_stamp(incoming_updated_at), + ) .await .map_err(ApiError::from)?; - Ok(()) + Ok(IamItemVerdict::Apply) } -async fn apply_iam_group_info_item(iam_sys: &IamSys, group_info: Option) -> S3Result<()> { +async fn apply_iam_group_info_item( + iam_sys: &IamSys, + marks: &SiteReplicationState, + group_info: Option, + incoming_updated_at: Option, +) -> S3Result { let Some(group_info) = group_info else { return Err(s3_error!(InvalidRequest, "groupInfo is required")); }; let update = group_info.update_req; + // The record is the group itself: its own timestamp moves on every + // membership or status change, so a delayed older add cannot re-add a + // member a newer removal took out, and a delayed older removal (or group + // delete) cannot undo a newer add (backlog#2291). Once the group is gone + // the marks of its deletion and of its members' removals stand in for it, + // so a stale add cannot re-create it or re-add a removed member. + let local_updated_at = match iam_sys.get_group_info(&update.group).await { + Some(group) => Some(group.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH)), + None => { + let entities: Vec = std::iter::once(iam_group_deletion_mark_entity(&update.group)) + .chain( + update + .members + .iter() + .map(|member| iam_group_member_deletion_mark_entity(&update.group, member)), + ) + .collect(); + iam_deletion_mark(marks, &entities) + } + }; + if judge_iam_item_staleness(local_updated_at, incoming_updated_at) == IamItemVerdict::SkipStale { + return Ok(IamItemVerdict::SkipStale); + } + let stamp = replicated_write_stamp(incoming_updated_at); if !group_info_requires_upsert(&update) { // Idempotent removal: a replayed deletion may find the group or a // member already gone (deleted here earlier, or the user tombstone @@ -5831,25 +6013,25 @@ async fn apply_iam_group_info_item(iam_sys: &IamSys, group_info: Op } } if members.is_empty() && !update.members.is_empty() { - return Ok(()); + return Ok(IamItemVerdict::Apply); } - match iam_sys.remove_users_from_group(&update.group, members).await { + match iam_sys.remove_users_from_group_at(&update.group, members, stamp).await { Ok(_) => {} Err(err) if rustfs_iam::error::is_err_no_such_group(&err) => {} Err(err) => return Err(ApiError::from(err).into()), } - return Ok(()); + return Ok(IamItemVerdict::Apply); } iam_sys - .add_users_to_group(&update.group, update.members) + .add_users_to_group_at(&update.group, update.members, stamp) .await .map_err(ApiError::from)?; iam_sys - .set_group_status(&update.group, matches!(update.status, GroupStatus::Enabled)) + .set_group_status_at(&update.group, matches!(update.status, GroupStatus::Enabled), stamp) .await .map_err(ApiError::from)?; - Ok(()) + Ok(IamItemVerdict::Apply) } async fn apply_iam_sts_account_item(iam_sys: &IamSys, sts_credential: Option) -> S3Result<()> { @@ -5889,17 +6071,23 @@ async fn apply_iam_sts_account_item(iam_sys: &IamSys, sts_credentia async fn apply_iam_user_item( iam_sys: &IamSys, + marks: &SiteReplicationState, iam_user: Option, incoming_updated_at: Option, -) -> S3Result<()> { +) -> S3Result { let Some(user) = iam_user else { return Err(s3_error!(InvalidRequest, "iamUser is required")); }; - if let Some(local) = iam_sys.get_user(&user.access_key).await - && is_stale_update(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH), incoming_updated_at) - { - return Ok(()); + // Once the identity is deleted, its deletion mark stands in for the + // record so a stale re-create cannot resurrect it (backlog#2291). + let local_updated_at = match iam_sys.get_user(&user.access_key).await { + Some(local) => Some(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH)), + None => iam_deletion_mark(marks, &[iam_user_deletion_mark_entity(&user.access_key)]), + }; + if judge_iam_item_staleness(local_updated_at, incoming_updated_at) == IamItemVerdict::SkipStale { + return Ok(IamItemVerdict::SkipStale); } + let stamp = replicated_write_stamp(incoming_updated_at); if user.is_delete_req { iam_sys.delete_user(&user.access_key, true).await.map_err(ApiError::from)?; } else { @@ -5909,36 +6097,42 @@ async fn apply_iam_user_item( let is_status_only_update = user_req.secret_key.is_empty() && user_req.policy.is_none(); if is_status_only_update { iam_sys - .set_user_status(&user.access_key, user_req.status) + .set_user_status_at(&user.access_key, user_req.status, stamp) .await .map_err(ApiError::from)?; } else { iam_sys - .create_user(&user.access_key, &user_req) + .create_user_at(&user.access_key, &user_req, stamp) .await .map_err(ApiError::from)?; } } - Ok(()) + Ok(IamItemVerdict::Apply) } async fn apply_iam_service_account_item( iam_sys: &IamSys, + marks: &SiteReplicationState, svc_acc_change: Option, incoming_updated_at: Option, -) -> S3Result<()> { +) -> S3Result { let Some(change) = svc_acc_change else { return Err(s3_error!(InvalidRequest, "serviceAccountChange is required")); }; let envelope = change.oidc_service_account_envelope; + let stamp = replicated_write_stamp(incoming_updated_at); if let Some(create) = change.create { - let local_updated_at = iam_sys - .get_user(&create.access_key) - .await - .map(|local| local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH)); + // Like the user path: with the account already deleted here, the + // recorded deletion mark is the timestamp a stale create/update + // (a snapshot or a delayed delivery) has to beat (backlog#2291). + let local_updated_at = match iam_sys.get_user(&create.access_key).await { + Some(local) => Some(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH)), + None if create.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT => None, + None => iam_deletion_mark(marks, &[format!("svc-acc:{}", create.access_key)]), + }; let replicated_policy = if create.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT { if local_updated_at.is_some_and(|local_updated_at| is_stale_update(local_updated_at, incoming_updated_at)) { - return Ok(()); + return Ok(IamItemVerdict::SkipStale); } ReplicatedServiceAccountPolicy { policy: Some(site_replicator_service_account_policy()?), @@ -5948,7 +6142,7 @@ async fn apply_iam_service_account_item( let Some(replicated_policy) = decode_service_account_replication_policy(&create, envelope.as_ref(), incoming_updated_at, local_updated_at)? else { - return Ok(()); + return Ok(IamItemVerdict::SkipStale); }; replicated_policy }; @@ -5962,7 +6156,7 @@ async fn apply_iam_service_account_item( )); } iam_sys - .update_service_account( + .update_service_account_at( &create.access_key, UpdateServiceAccountOpts { name: replicated_policy.metadata_for_existing_account(create.name), @@ -5974,13 +6168,19 @@ async fn apply_iam_service_account_item( parent_user: None, allow_site_replicator_account: create.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT, }, + stamp, ) .await .map_err(ApiError::from)?; } Err(err) if is_err_no_such_service_account(&err) => { + // A snapshot (bootstrap / repair / retry resend) carries the + // account's current status, and the account is created with + // it in the same write: a disabled account must never exist + // enabled here, not even between a create and a follow-up + // status write that might fail (backlog#2289). iam_sys - .new_service_account( + .new_service_account_at( &create.parent, Some(create.groups), NewServiceAccountOpts { @@ -5992,21 +6192,23 @@ async fn apply_iam_service_account_item( expiration: create.expiration, allow_site_replicator_account: true, claims: Some(create.claims), + status: (!create.status.is_empty()).then_some(create.status), }, + stamp, ) .await .map_err(ApiError::from)?; } Err(err) => return Err(ApiError::from(err).into()), } - return Ok(()); + return Ok(IamItemVerdict::Apply); } if let Some(update) = change.update { if let Some(local) = iam_sys.get_user(&update.access_key).await && is_stale_update(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH), incoming_updated_at) { - return Ok(()); + return Ok(IamItemVerdict::SkipStale); } let allow_site_replicator_account = update.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT; let session_policy = if allow_site_replicator_account { @@ -6015,7 +6217,7 @@ async fn apply_iam_service_account_item( update.session_policy.as_str().and_then(|raw| serde_json::from_str(raw).ok()) }; iam_sys - .update_service_account( + .update_service_account_at( &update.access_key, UpdateServiceAccountOpts { session_policy, @@ -6029,23 +6231,24 @@ async fn apply_iam_service_account_item( parent_user: None, allow_site_replicator_account, }, + stamp, ) .await .map_err(ApiError::from)?; - return Ok(()); + return Ok(IamItemVerdict::Apply); } if let Some(delete) = change.delete { if let Some(local) = iam_sys.get_user(&delete.access_key).await && is_stale_update(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH), incoming_updated_at) { - return Ok(()); + return Ok(IamItemVerdict::SkipStale); } iam_sys .delete_service_account(&delete.access_key, true) .await .map_err(ApiError::from)?; - return Ok(()); + return Ok(IamItemVerdict::Apply); } Err(s3_error!(InvalidRequest, "serviceAccountChange is empty")) @@ -6111,6 +6314,7 @@ fn adopt_add_commit_state(state: &mut SiteReplicationState, next_state: SiteRepl sync_state_initialized, edit_generation: _, applied_edit_generations: _, + iam_deletion_marks: _, } = next_state; state.name = name; state.service_account_access_key = service_account_access_key; @@ -6707,6 +6911,7 @@ async fn apply_peer_join_service_account(join_req: SRPeerJoinReq) -> S3Result<() expiration: None, allow_site_replicator_account: join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT, claims: None, + status: None, }, ) .await @@ -7692,7 +7897,7 @@ impl Operation for SiteReplicationRepairHandler { let local_peer = current_local_peer(&req, &state); let body: SiteReplicationRepairRequest = read_site_replication_json(req, "", false).await?; let info = build_sr_info(&state, &local_peer).await?; - let plan = site_replication_bootstrap_plan(&info)?; + let plan = build_site_replication_bootstrap_plan(&info).await?; let signing_key = current_token_signing_key().ok_or_else(|| { S3Error::with_message(S3ErrorCode::InternalError, "token signing key is not initialized".to_string()) })?; @@ -7885,6 +8090,7 @@ impl Operation for SRRotateServiceAccountHandler { mod tests { use super::*; use crate::site_replication::identity::deployment_id_for_endpoint; + use rustfs_madmin::SRSessionPolicy; /// A peer the status probe could not reach must render as offline. /// @@ -8039,6 +8245,343 @@ mod tests { } } + // --- Review regressions on rustfs#7195 (backlog#2289 / #2291 / #2292): + // delivery order and concurrency through the real receiver. + + /// Two-peer state so the apply transaction's persist keeps the state + /// object (a single-peer state is cleared on write) and the deletion + /// marks it records survive between items. + async fn seed_two_peer_state_for_iam_apply() { + let seed = SiteReplicationState { + peers: BTreeMap::from([ + ( + "site-a".to_string(), + PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://a.example:9000") + }, + ), + ( + "site-b".to_string(), + PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://b.example:9000") + }, + ), + ]), + ..Default::default() + }; + save_site_replication_state(&seed).await.expect("seed state"); + } + + async fn clear_seeded_state() { + save_site_replication_state(&SiteReplicationState::default()) + .await + .expect("clear state"); + } + + fn sr_item(item_type: &str, updated_at: OffsetDateTime) -> SRIAMItem { + SRIAMItem { + r#type: item_type.to_string(), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + } + } + + fn allow_actions_policy(actions: &[&str]) -> serde_json::Value { + serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{"Effect": "Allow", "Action": actions, "Resource": ["arn:aws:s3:::*"]}] + }) + } + + fn sr_policy_item(name: &str, body: serde_json::Value, updated_at: OffsetDateTime) -> SRIAMItem { + let mut item = sr_item("policy", updated_at); + item.name = name.to_string(); + item.policy = Some(body); + item + } + + fn sr_mapping_item(user: &str, policy: &str, updated_at: OffsetDateTime) -> SRIAMItem { + let mut item = sr_item("policy-mapping", updated_at); + item.policy_mapping = Some(SRPolicyMapping { + user_or_group: user.to_string(), + user_type: 0, + is_group: false, + policy: policy.to_string(), + ..Default::default() + }); + item + } + + fn sr_group_item(group: &str, members: &[&str], is_remove: bool, updated_at: OffsetDateTime) -> SRIAMItem { + let mut item = sr_item("group-info", updated_at); + item.group_info = Some(SRGroupInfo { + update_req: rustfs_madmin::GroupAddRemove { + group: group.to_string(), + members: members.iter().map(|member| member.to_string()).collect(), + status: rustfs_madmin::GroupStatus::Enabled, + is_remove, + }, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }); + item + } + + fn sr_user_item( + access_key: &str, + user_req: Option, + updated_at: OffsetDateTime, + ) -> SRIAMItem { + let mut item = sr_item("iam-user", updated_at); + item.iam_user = Some(SRIAMUser { + access_key: access_key.to_string(), + is_delete_req: user_req.is_none(), + user_req, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }); + item + } + + fn user_req(secret_key: &str, status: rustfs_madmin::AccountStatus) -> rustfs_madmin::AddOrUpdateUserReq { + rustfs_madmin::AddOrUpdateUserReq { + secret_key: secret_key.to_string(), + policy: None, + status, + } + } + + fn sr_service_account_create_item(parent: &str, access_key: &str, status: &str, updated_at: OffsetDateTime) -> SRIAMItem { + let mut item = sr_item("service-account", updated_at); + item.svc_acc_change = Some(SRSvcAccChange { + create: Some(SRSvcAccCreate { + parent: parent.to_string(), + access_key: access_key.to_string(), + secret_key: "replicated-svc-secret-123".to_string(), + groups: Vec::new(), + claims: HashMap::new(), + session_policy: rustfs_madmin::SRSessionPolicy::default(), + status: status.to_string(), + name: String::new(), + description: String::new(), + expiration: None, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + ..Default::default() + }); + item + } + + async fn stored_policy_json(name: &str) -> (String, Option) { + let iam = current_iam_handle().expect("test IAM"); + let doc = iam.get_policy_doc(name).await.expect("policy doc"); + (serde_json::to_string(&doc.policy).expect("serialize policy"), doc.update_date) + } + + /// Review finding on rustfs#7195 (P1): two source edits T1 < T2 that both + /// predate their delivery. The record T1 writes must carry T1 — not the + /// later receive time — or T2 is judged stale against it and the newer + /// revoke is silently dropped. Exercised through the real receiver for + /// every gated item type. + #[tokio::test] + #[serial] + async fn apply_iam_item_applies_delayed_in_order_updates_through_the_receiver() { + publish_ready_iam_context().await; + seed_two_peer_state_for_iam_apply().await; + let iam = current_iam_handle().expect("test IAM"); + let t1 = OffsetDateTime::now_utc() - time::Duration::hours(2); + let t2 = t1 + time::Duration::minutes(5); + + // policy: the grant, then the narrower revision. + let policy = "sr-delayed-order-policy"; + apply_iam_item(sr_policy_item(policy, allow_actions_policy(&["s3:GetObject", "s3:PutObject"]), t1)) + .await + .expect("T1 grant"); + apply_iam_item(sr_policy_item(policy, allow_actions_policy(&["s3:GetObject"]), t2)) + .await + .expect("T2 narrowed body"); + let (stored, stamp) = stored_policy_json(policy).await; + assert_eq!(stamp, Some(t2), "the stored stamp is the source time of the last applied edit"); + assert!( + !stored.contains("s3:PutObject"), + "the narrower T2 body must replace the T1 grant: {stored}" + ); + + // policy-mapping: attach the wide policy, then the narrow one. + for (name, actions) in [ + ("sr-delayed-order-wide", &["s3:*"][..]), + ("sr-delayed-order-narrow", &["s3:GetObject"][..]), + ] { + let body: rustfs_policy::policy::Policy = serde_json::from_value(allow_actions_policy(actions)).expect("policy body"); + iam.set_policy(name, body).await.expect("local policy"); + } + let user = "sr-delayed-order-user"; + apply_iam_item(sr_mapping_item(user, "sr-delayed-order-wide", t1)) + .await + .expect("T1 attach"); + apply_iam_item(sr_mapping_item(user, "sr-delayed-order-narrow", t2)) + .await + .expect("T2 attach"); + let mapping = iam + .get_mapped_policy_record(user, rustfs_iam::store::UserType::Reg, false) + .await + .expect("mapping"); + assert_eq!(mapping.policies, "sr-delayed-order-narrow"); + assert_eq!(mapping.update_at, t2); + + // group: add the member, then remove it. + let member = "sr-delayed-order-member"; + iam.create_user(member, &user_req("member-secret-key-123", rustfs_madmin::AccountStatus::Enabled)) + .await + .expect("member"); + let group = "sr-delayed-order-group"; + apply_iam_item(sr_group_item(group, &[member], false, t1)) + .await + .expect("T1 add"); + apply_iam_item(sr_group_item(group, &[member], true, t2)) + .await + .expect("T2 remove"); + let info = iam.get_group_info(group).await.expect("group"); + assert!(info.members.is_empty(), "the T2 removal must land after the delayed T1 add"); + assert_eq!(info.update_at, Some(t2)); + + // iam-user: create enabled, then the status-only disable. + let access_key = "sr-delayed-order-account"; + apply_iam_item(sr_user_item( + access_key, + Some(user_req("account-secret-key-123", rustfs_madmin::AccountStatus::Enabled)), + t1, + )) + .await + .expect("T1 create"); + apply_iam_item(sr_user_item(access_key, Some(user_req("", rustfs_madmin::AccountStatus::Disabled)), t2)) + .await + .expect("T2 disable"); + let identity = iam.get_user(access_key).await.expect("user"); + assert_eq!(identity.credentials.status, "off", "the T2 disable must land after the delayed T1 create"); + assert_eq!(identity.update_at, Some(t2)); + + clear_seeded_state().await; + } + + /// Review finding on rustfs#7195: an older grant and a newer revoke for + /// the same record delivered concurrently must always leave the revoke, + /// whichever request reaches the transaction first — the verdict and + /// the write of one cannot interleave with the other's. + #[tokio::test] + #[serial] + async fn apply_iam_item_serializes_a_concurrent_older_grant_and_newer_revoke() { + publish_ready_iam_context().await; + seed_two_peer_state_for_iam_apply().await; + let t1 = OffsetDateTime::now_utc() - time::Duration::hours(2); + let t2 = t1 + time::Duration::minutes(5); + + for round in 0..6u32 { + let policy = format!("sr-race-policy-{round}"); + let grant = tokio::spawn(apply_iam_item(sr_policy_item( + &policy, + allow_actions_policy(&["s3:GetObject", "s3:PutObject"]), + t1, + ))); + let revoke = tokio::spawn(apply_iam_item(sr_policy_item(&policy, allow_actions_policy(&["s3:GetObject"]), t2))); + let (grant, revoke) = if round % 2 == 0 { + tokio::join!(grant, revoke) + } else { + let (revoke, grant) = tokio::join!(revoke, grant); + (grant, revoke) + }; + grant.expect("join grant").expect("grant delivery is acknowledged"); + revoke.expect("join revoke").expect("revoke delivery is acknowledged"); + let (stored, stamp) = stored_policy_json(&policy).await; + assert!( + !stored.contains("s3:PutObject"), + "round {round}: the grant won over the newer revoke: {stored}" + ); + assert_eq!(stamp, Some(t2), "round {round}"); + } + + clear_seeded_state().await; + } + + /// Review finding on rustfs#7195: a replicated delete followed by the + /// delayed delivery of the older create must not resurrect the entity, + /// and the mark that fences it is committed by the same transaction that + /// applied the delete; a genuinely newer create still lands. + #[tokio::test] + #[serial] + async fn apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete() { + publish_ready_iam_context().await; + seed_two_peer_state_for_iam_apply().await; + let iam = current_iam_handle().expect("test IAM"); + let t1 = OffsetDateTime::now_utc() - time::Duration::hours(2); + let t3 = t1 + time::Duration::minutes(10); + let t4 = t3 + time::Duration::minutes(10); + let access_key = "sr-recreate-account"; + let create = |at| { + sr_user_item( + access_key, + Some(user_req("recreate-secret-key-123", rustfs_madmin::AccountStatus::Enabled)), + at, + ) + }; + + apply_iam_item(create(t1)).await.expect("T1 create"); + assert!(iam.get_user(access_key).await.is_some()); + apply_iam_item(sr_user_item(access_key, None, t3)).await.expect("T3 delete"); + assert!(iam.get_user(access_key).await.is_none()); + let state = load_site_replication_state().await.expect("state"); + assert_eq!( + state.iam_deletion_marks.get(&iam_user_deletion_mark_entity(access_key)), + Some(&t3), + "the delete's mark is committed with the delete" + ); + + apply_iam_item(create(t1)).await.expect("the stale replay is acknowledged"); + assert!( + iam.get_user(access_key).await.is_none(), + "a create older than the recorded deletion must not re-create the user" + ); + + apply_iam_item(create(t4)).await.expect("T4 create"); + let identity = iam.get_user(access_key).await.expect("a newer create lands"); + assert_eq!(identity.update_at, Some(t4)); + + clear_seeded_state().await; + } + + /// Review finding on rustfs#7195 (backlog#2289): a replicated disabled + /// service account is created disabled in one write, never enabled and + /// then switched off, and carries the source stamp. + #[tokio::test] + #[serial] + async fn apply_iam_item_creates_a_replicated_service_account_with_its_status() { + publish_ready_iam_context().await; + seed_two_peer_state_for_iam_apply().await; + let iam = current_iam_handle().expect("test IAM"); + let t1 = OffsetDateTime::now_utc() - time::Duration::hours(2); + let parent = "sr-svc-parent"; + iam.create_user(parent, &user_req("parent-secret-key-123", rustfs_madmin::AccountStatus::Enabled)) + .await + .expect("parent"); + + for (access_key, status, expected) in [ + ("sr-svc-disabled", "off", "off"), + ("sr-svc-enabled", "on", "on"), + ("sr-svc-default", "", "on"), + ] { + apply_iam_item(sr_service_account_create_item(parent, access_key, status, t1)) + .await + .expect("service account create"); + let (credentials, _) = iam.get_service_account(access_key).await.expect("service account"); + assert_eq!(credentials.status, expected, "{access_key}"); + let identity = iam.get_user(access_key).await.expect("identity"); + assert_eq!(identity.update_at, Some(t1), "{access_key} carries the source stamp"); + } + + clear_seeded_state().await; + } + #[tokio::test] #[serial] async fn apply_iam_item_accepts_minio_sts_account_item_type() { @@ -12186,6 +12729,355 @@ mod tests { assert!(!is_stale_update(local, None)); } + /// Minimal model of one replicated IAM record (a policy document body, a + /// user/group mapping, or a group's member set) as the apply paths treat + /// it: `None` is "absent", `Some((content, stamp))` is the local record + /// with the timestamp of the change that last wrote it. Applying an item + /// goes through `judge_iam_item_staleness` exactly like the three apply + /// functions do; a delete (`incoming == None`) on an absent record is the + /// idempotent no-op of backlog#2071. + fn apply_iam_item_to_model( + record: &mut Option<(&'static str, OffsetDateTime)>, + incoming: Option<&'static str>, + incoming_updated_at: Option, + ) -> IamItemVerdict { + let verdict = judge_iam_item_staleness(record.map(|(_, stamp)| stamp), incoming_updated_at); + if verdict == IamItemVerdict::Apply { + *record = incoming.map(|content| (content, incoming_updated_at.unwrap_or(OffsetDateTime::UNIX_EPOCH))); + } + verdict + } + + fn at(seconds: i64) -> OffsetDateTime { + OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(seconds) + } + + /// backlog#2291: a revoke (narrowed policy body, detached mapping, member + /// removed from the group) followed by the delayed delivery of the older + /// grant must leave the revoke in place. + #[test] + fn test_iam_item_stale_grant_after_revoke_is_not_applied() { + let mut record = None; + assert_eq!(apply_iam_item_to_model(&mut record, Some("grant"), Some(at(10))), IamItemVerdict::Apply); + assert_eq!(apply_iam_item_to_model(&mut record, Some("revoke"), Some(at(20))), IamItemVerdict::Apply); + + // The older grant is redelivered (retry drain, slow peer) after the revoke. + assert_eq!( + apply_iam_item_to_model(&mut record, Some("grant"), Some(at(10))), + IamItemVerdict::SkipStale, + "a grant older than the local revoke must be acknowledged without being applied" + ); + assert_eq!(record, Some(("revoke", at(20))), "the revoke must survive the stale grant"); + } + + /// backlog#2291: the mirror image — a grant followed by the delayed delivery + /// of an older revoke (older body, older detach, older member removal, or + /// an older delete) must leave the grant in place. + #[test] + fn test_iam_item_stale_revoke_after_grant_is_not_applied() { + let mut record = None; + assert_eq!(apply_iam_item_to_model(&mut record, Some("revoke"), Some(at(10))), IamItemVerdict::Apply); + assert_eq!(apply_iam_item_to_model(&mut record, Some("grant"), Some(at(20))), IamItemVerdict::Apply); + + assert_eq!( + apply_iam_item_to_model(&mut record, Some("revoke"), Some(at(10))), + IamItemVerdict::SkipStale, + "a revoke older than the local grant must not be applied" + ); + assert_eq!( + apply_iam_item_to_model(&mut record, None, Some(at(15))), + IamItemVerdict::SkipStale, + "a delete older than the local record must not remove it" + ); + assert_eq!(record, Some(("grant", at(20)))); + } + + /// backlog#2291: an item at least as new as the local record is applied, + /// including a newer delete; equal timestamps are not stale (same rule as + /// `iam-user`). + #[test] + fn test_iam_item_newer_than_local_record_is_applied() { + let mut record = Some(("grant", at(20))); + assert_eq!(apply_iam_item_to_model(&mut record, Some("revoke"), Some(at(20))), IamItemVerdict::Apply); + assert_eq!(record, Some(("revoke", at(20)))); + + assert_eq!(apply_iam_item_to_model(&mut record, Some("grant"), Some(at(30))), IamItemVerdict::Apply); + assert_eq!(record, Some(("grant", at(30)))); + + assert_eq!(apply_iam_item_to_model(&mut record, None, Some(at(40))), IamItemVerdict::Apply); + assert_eq!(record, None, "a newer delete removes the record"); + } + + /// backlog#2291: peers that predate item timestamps keep today's + /// last-writer-wins behaviour — an item without `updatedAt` is applied even + /// over a newer local record. + #[test] + fn test_iam_item_without_source_timestamp_is_applied() { + let mut record = Some(("grant", at(20))); + assert_eq!(apply_iam_item_to_model(&mut record, Some("revoke"), None), IamItemVerdict::Apply); + assert_eq!(record.map(|(content, _)| content), Some("revoke")); + + assert_eq!(judge_iam_item_staleness(Some(at(20)), None), IamItemVerdict::Apply); + assert_eq!(judge_iam_item_staleness(None, None), IamItemVerdict::Apply); + } + + /// backlog#2291: nothing is stale relative to an absent record. A create + /// with any timestamp is applied, and a delete falls through to the + /// idempotent no-op paths (backlog#2071) instead of being judged. + #[test] + fn test_iam_item_targeting_absent_record_is_applied() { + assert_eq!(judge_iam_item_staleness(None, Some(at(1))), IamItemVerdict::Apply); + + let mut record = None; + assert_eq!(apply_iam_item_to_model(&mut record, None, Some(at(1))), IamItemVerdict::Apply); + assert_eq!(record, None); + assert_eq!(apply_iam_item_to_model(&mut record, Some("grant"), Some(at(1))), IamItemVerdict::Apply); + assert_eq!(record, Some(("grant", at(1)))); + + // A record that predates timestamps is reported as UNIX_EPOCH by the + // apply paths and therefore never rejects an item. + assert_eq!( + judge_iam_item_staleness(Some(OffsetDateTime::UNIX_EPOCH), Some(at(1))), + IamItemVerdict::Apply + ); + } + + /// The apply paths once the record is gone: the local timestamp the gate + /// sees is the deletion mark (or `None` when no deletion was recorded), + /// and a committed deletion records its source timestamp as the mark — + /// the same sequence `apply_iam_item` runs. + fn apply_iam_item_to_deleted_record_model( + marks: &mut SiteReplicationState, + entity: &str, + incoming_is_delete: bool, + incoming_updated_at: Option, + ) -> IamItemVerdict { + let entities = vec![entity.to_string()]; + let verdict = judge_iam_item_staleness(iam_deletion_mark(marks, &entities), incoming_updated_at); + if verdict == IamItemVerdict::Apply + && incoming_is_delete + && let Some(deleted_at) = incoming_updated_at + { + // Pruning is judged from the deletion's own clock in the model. + record_iam_deletion_marks_at(marks, &entities, deleted_at, deleted_at); + } + verdict + } + + /// backlog#2291 (real-VM case R6.3a of backlog#2080): a detach deletes the + /// mapping outright, so the older grant that arrives afterwards finds no + /// record — the deletion mark must stand in for it and reject the grant. + /// The same holds for a deleted policy document, user or group. + #[test] + fn test_iam_item_stale_grant_after_record_deletion_is_not_applied() { + let mut marks = SiteReplicationState::default(); + let entity = "policy-mapping:alice:0:false"; + + // The revoke (detach) is applied first: the record is gone, the mark stays. + assert_eq!( + apply_iam_item_to_deleted_record_model(&mut marks, entity, true, Some(at(20))), + IamItemVerdict::Apply + ); + assert_eq!(marks.iam_deletion_marks.get(entity), Some(&at(20))); + + // The older grant is delivered after the revoke. + assert_eq!( + apply_iam_item_to_deleted_record_model(&mut marks, entity, false, Some(at(10))), + IamItemVerdict::SkipStale, + "a grant older than the recorded deletion must not re-create the record" + ); + // A replayed copy of the same revoke stays a no-op and keeps the mark. + assert_eq!( + apply_iam_item_to_deleted_record_model(&mut marks, entity, true, Some(at(20))), + IamItemVerdict::Apply + ); + assert_eq!(marks.iam_deletion_marks.get(entity), Some(&at(20))); + // An older replayed revoke is stale against the newer one. + assert_eq!( + apply_iam_item_to_deleted_record_model(&mut marks, entity, true, Some(at(5))), + IamItemVerdict::SkipStale + ); + assert_eq!( + marks.iam_deletion_marks.get(entity), + Some(&at(20)), + "an older deletion never lowers the mark" + ); + } + + /// backlog#2291: a mark only fences items older than the deletion. A grant + /// newer than (or as new as) the recorded deletion re-creates the record, + /// an unmarked entity and an item without a source timestamp keep today's + /// behaviour. + #[test] + fn test_iam_item_newer_than_deletion_mark_is_applied() { + let mut marks = SiteReplicationState::default(); + let entity = "policy:readonly"; + assert_eq!( + apply_iam_item_to_deleted_record_model(&mut marks, entity, true, Some(at(20))), + IamItemVerdict::Apply + ); + + assert_eq!( + apply_iam_item_to_deleted_record_model(&mut marks, entity, false, Some(at(20))), + IamItemVerdict::Apply, + "a grant as new as the deletion is not stale" + ); + assert_eq!( + apply_iam_item_to_deleted_record_model(&mut marks, entity, false, Some(at(30))), + IamItemVerdict::Apply + ); + assert_eq!( + apply_iam_item_to_deleted_record_model(&mut marks, entity, false, None), + IamItemVerdict::Apply, + "an item from a peer without timestamps keeps last-writer-wins" + ); + assert_eq!( + apply_iam_item_to_deleted_record_model(&mut marks, "policy:other", false, Some(at(1))), + IamItemVerdict::Apply, + "no mark, no record: nothing to be stale against" + ); + assert_eq!( + judge_iam_item_staleness(iam_deletion_mark(&marks, &[]), Some(at(1))), + IamItemVerdict::Apply + ); + } + + /// backlog#2291: a group's removal marks are per member (plus the group + /// itself for a group delete), so with the group gone a stale add is + /// judged against the newest mark among the group and the members it + /// would add. + #[test] + fn test_iam_group_item_after_deletion_is_judged_against_member_marks() { + let mut marks = SiteReplicationState::default(); + let bob = iam_group_member_deletion_mark_entity("devs", "bob"); + let group = iam_group_deletion_mark_entity("devs"); + record_iam_deletion_marks_at(&mut marks, std::slice::from_ref(&bob), at(20), at(20)); + record_iam_deletion_marks_at(&mut marks, std::slice::from_ref(&group), at(30), at(30)); + + // The gate for an add of `bob` to the (deleted) group. + let add_bob = [group.clone(), bob.clone()]; + assert_eq!(iam_deletion_mark(&marks, &add_bob), Some(at(30))); + assert_eq!( + judge_iam_item_staleness(iam_deletion_mark(&marks, &add_bob), Some(at(25))), + IamItemVerdict::SkipStale + ); + assert_eq!( + judge_iam_item_staleness(iam_deletion_mark(&marks, &add_bob), Some(at(30))), + IamItemVerdict::Apply + ); + + // An add of `carol` to a group that was only ever partially emptied + // (no group delete) is judged against carol's own mark only. + marks.iam_deletion_marks.remove(&group); + let add_carol = [group.clone(), iam_group_member_deletion_mark_entity("devs", "carol")]; + assert_eq!(iam_deletion_mark(&marks, &add_carol), None); + assert_eq!( + judge_iam_item_staleness(iam_deletion_mark(&marks, &add_carol), Some(at(1))), + IamItemVerdict::Apply + ); + let add_bob = [group, bob]; + assert_eq!( + judge_iam_item_staleness(iam_deletion_mark(&marks, &add_bob), Some(at(10))), + IamItemVerdict::SkipStale + ); + } + + /// Cheap wiring guard for backlog#2291: every one of the `policy`, + /// `policy-mapping`, `group-info`, `iam-user` and `service-account` apply + /// paths must judge the item against the local record (falling back to + /// the deletion marks of the transaction's state when the record is + /// absent) before it writes or deletes anything, must stamp every write + /// with the item's source time, and `apply_iam_item` must run verdict, + /// write and mark commit inside one state transaction. The ordering rule + /// itself is covered by the `test_iam_item_*` model tests above and the + /// `apply_iam_item_*` receiver tests; this only pins that no path bypasses + /// it again. + #[test] + fn test_iam_policy_mapping_and_group_items_gate_on_incoming_updated_at() { + let source = include_str!("site_replication.rs"); + let locally_stamped_writes = [ + ".set_policy(", + ".policy_db_set(", + ".add_users_to_group(", + ".remove_users_from_group(", + ".set_group_status(", + ".create_user(", + ".set_user_status(", + ".new_service_account(", + ".update_service_account(", + ]; + for (start, end, judged_by_shared_verdict) in [ + ("async fn apply_iam_policy_item(", "async fn apply_iam_policy_mapping_item(", true), + ("async fn apply_iam_policy_mapping_item(", "async fn apply_iam_group_info_item(", true), + ("async fn apply_iam_group_info_item(", "async fn apply_iam_sts_account_item(", true), + ("async fn apply_iam_user_item(", "async fn apply_iam_service_account_item(", true), + ("async fn apply_iam_service_account_item(", "fn claims_unix_timestamp(", false), + ] { + let body = source + .split(start) + .nth(1) + .and_then(|rest| rest.split(end).next()) + .expect(start); + if judged_by_shared_verdict { + assert!( + body.contains("judge_iam_item_staleness(local_updated_at, incoming_updated_at)"), + "{start} must judge the item against the local record before applying it" + ); + } else { + assert!( + body.contains("is_stale_update(local_updated_at, incoming_updated_at)"), + "{start} must judge the item against the local record before applying it" + ); + } + assert!( + body.contains("iam_deletion_mark("), + "{start} must fall back to the deletion marks of the transaction's state when the record is absent" + ); + assert!( + body.contains("replicated_write_stamp(incoming_updated_at)"), + "{start} must stamp its writes with the item's source time" + ); + for write in locally_stamped_writes { + assert!( + !body.contains(write), + "{start} must not stamp a replicated write with the local clock ({write})" + ); + } + } + let dispatch = source + .split("async fn apply_iam_item(") + .nth(1) + .and_then(|rest| rest.split("fn replicated_write_stamp(").next()) + .expect("apply_iam_item"); + assert!( + dispatch.contains("with_site_replication_state_transaction(move |mut state| async move {"), + "apply_iam_item must run verdict, write and mark commit in one state transaction" + ); + assert!( + dispatch.contains("record_iam_deletion_marks(&mut state, &deletion_mark_entities, deleted_at)"), + "apply_iam_item must record the mark of a deletion it committed in the same transaction" + ); + assert!( + !dispatch.contains("commit_iam_deletion_marks("), + "the mark must not be committed in a second transaction" + ); + // backlog#2289: a replicated service account is created with its + // status; a second status write could fail and leave it enabled. + let create_branch = source + .split("Err(err) if is_err_no_such_service_account(&err) => {") + .nth(1) + .and_then(|rest| rest.split("Err(err) => return Err(ApiError::from(err).into()),").next()) + .expect("service account create branch"); + assert!( + create_branch.contains("status: (!create.status.is_empty()).then_some(create.status),"), + "the service account must be created with the source status" + ); + assert!( + !create_branch.contains("update_service_account"), + "the created service account's status must not depend on a second write" + ); + } + #[test] fn test_apply_state_edit_req_only_updates_ilm_expiry_flags() { let mut state = SiteReplicationState::default(); @@ -13717,4 +14609,76 @@ mod tests { ); } } + + /// backlog#2292: the receiver persists the SOURCE `updated_at` of an + /// applied bucket config and judges the next item's source time against + /// it. Stamping the local apply time instead rejected a source edit that + /// was newer than the applied one but delivered after the local stamp + /// (two quick source edits under delivery delay; a peer clock ahead of + /// ours) and acknowledged it with 200. + #[test] + fn test_bucket_meta_staleness_is_judged_against_the_applied_source_timestamp() { + let apply_wall_clock = OffsetDateTime::now_utc(); + let source_edit_t1 = apply_wall_clock - time::Duration::seconds(30); + let source_edit_t2 = source_edit_t1 + time::Duration::seconds(2); + let source_edit_t0 = source_edit_t1 - time::Duration::seconds(2); + assert!( + source_edit_t2 < apply_wall_clock, + "T2 is newer at the source yet older than the local apply clock" + ); + + // Edit T1 arrives first and is applied the way apply_bucket_meta_item + // persists a replicated config: stamped with its source time. + let mut meta = crate::admin::storage_api::bucket::metadata::BucketMetadata::new("photos"); + meta.update_config_at( + BUCKET_POLICY_CONFIG, + br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(), + source_edit_t1, + ) + .expect("apply edit T1"); + let local_updated_at = bucket_meta_local_updated_at(&meta, BUCKET_POLICY_CONFIG); + assert_eq!( + local_updated_at, source_edit_t1, + "the stored stamp is the source time, not the apply clock" + ); + + // Edit T2 is newer at the source but delivered late: it must apply. + assert!( + !is_stale_update(local_updated_at, Some(source_edit_t2)), + "edit T2 ({source_edit_t2}) is newer than applied edit T1 ({source_edit_t1}) but is rejected against local stamp {local_updated_at}" + ); + // Edit T0 predates the applied edit: it stays rejected. + assert!( + is_stale_update(local_updated_at, Some(source_edit_t0)), + "edit T0 ({source_edit_t0}) is older than applied edit T1 ({source_edit_t1}) and must be rejected" + ); + // An item without a source time is never judged stale (unchanged). + assert!(!is_stale_update(local_updated_at, None)); + } + + /// backlog#2292: the replicated-config write in `apply_bucket_meta_item` + /// must go through the source-stamped entries; a plain + /// `update_if_incarnation` there would reintroduce local stamping. + #[test] + fn test_apply_bucket_meta_item_writes_through_the_source_stamped_entries() { + let source = include_str!("site_replication.rs"); + let apply = source + .split("async fn apply_bucket_meta_item") + .nth(1) + .and_then(|rest| rest.split("fn group_info_requires_upsert").next()) + .expect("apply_bucket_meta_item source"); + assert!( + apply.contains("update_quota_if_incarnation_at("), + "durable quota must carry the source stamp" + ); + assert!(apply.contains("update_if_incarnation_at("), "bucket configs must carry the source stamp"); + assert!( + apply.contains("delete_if_incarnation_at("), + "bucket config deletes must carry the source stamp too, or a newer re-create is judged stale" + ); + assert!( + !apply.contains("metadata_sys::update_if_incarnation(&item.bucket"), + "no replicated config write may bypass the source stamp" + ); + } } diff --git a/rustfs/src/admin/handlers/user.rs b/rustfs/src/admin/handlers/user.rs index a9d4ee4f0..04d28290f 100644 --- a/rustfs/src/admin/handlers/user.rs +++ b/rustfs/src/admin/handlers/user.rs @@ -1131,6 +1131,7 @@ impl Operation for ImportIam { expiration: req.expiration, allow_site_replicator_account: false, claims: Some(req.claims), + status: None, }; let groups = if req.groups.is_empty() { None } else { Some(req.groups) }; diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 024fd2f59..97e87c68c 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -335,6 +335,25 @@ pub(crate) mod metadata_sys { super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await } + /// [`update_if_incarnation`] stamping the config with a replicated edit's + /// source `updated_at` instead of the local clock (backlog#2292). + pub(crate) async fn update_if_incarnation_at( + bucket: &str, + config_file: &str, + data: Vec, + expected_incarnation_id: uuid::Uuid, + updated_at: OffsetDateTime, + ) -> Result { + super::ecstore_bucket::metadata_sys::update_if_incarnation_at( + bucket, + config_file, + data, + expected_incarnation_id, + updated_at, + ) + .await + } + pub(crate) async fn update_quota_if_incarnation( bucket: &str, data: Vec, @@ -344,6 +363,25 @@ pub(crate) mod metadata_sys { super::ecstore_bucket::metadata_sys::update_quota_if_incarnation(bucket, data, expected_incarnation_id, proof).await } + /// [`update_quota_if_incarnation`] stamping the quota with a replicated + /// edit's source `updated_at` instead of the local clock (backlog#2292). + pub(crate) async fn update_quota_if_incarnation_at( + bucket: &str, + data: Vec, + expected_incarnation_id: uuid::Uuid, + proof: &super::ecstore_notification::CrossPoolFenceFleetProofToken, + updated_at: OffsetDateTime, + ) -> Result { + super::ecstore_bucket::metadata_sys::update_quota_if_incarnation_at( + bucket, + data, + expected_incarnation_id, + proof, + updated_at, + ) + .await + } + pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result { super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await } @@ -398,6 +436,18 @@ pub(crate) mod metadata_sys { super::ecstore_bucket::metadata_sys::delete_if_incarnation(bucket, config_file, expected_incarnation_id).await } + /// [`delete_if_incarnation`] stamping the cleared config with a replicated + /// deletion's source `updated_at` instead of the local clock (backlog#2292). + pub(crate) async fn delete_if_incarnation_at( + bucket: &str, + config_file: &str, + expected_incarnation_id: uuid::Uuid, + updated_at: OffsetDateTime, + ) -> Result { + super::ecstore_bucket::metadata_sys::delete_if_incarnation_at(bucket, config_file, expected_incarnation_id, updated_at) + .await + } + pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> { super::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await } diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 2719ccb72..7a48d5ff1 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -718,6 +718,8 @@ pub(crate) mod bucket { delete_marker_version_id: None, delete_marker: false, delete_marker_mtime: None, + target_delete_marker_version_ids: Default::default(), + target_delete_marker_version_ids_corrupt: false, target_arns, force_delete_id: Some(operation_id), force_delete_generation: Some(i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX)), diff --git a/rustfs/src/site_replication/hooks.rs b/rustfs/src/site_replication/hooks.rs index 95a830500..2af6aa000 100644 --- a/rustfs/src/site_replication/hooks.rs +++ b/rustfs/src/site_replication/hooks.rs @@ -302,7 +302,164 @@ pub(crate) fn site_replication_state_replicates_ilm_expiry(state: &SiteReplicati state.peers.values().any(|peer| peer.replicate_ilm_expiry) } -pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result { +/// Secret-bearing half of the IAM snapshot. `SRInfo` is served to admin +/// callers (`site-replication/info`, status, add preflight) and must stay +/// secret-free, so the bootstrap plan receives credentials through this +/// separate value, built only on the paths that deliver to peers (site add +/// bootstrap, repair, retry snapshot resend). Never persisted, never served. +#[derive(Debug, Clone, Default)] +pub(crate) struct SiteReplicationIamCredentials { + /// Built-in users (access key -> credential); temp and service accounts + /// are excluded, external/IdP users never appear here. + pub(crate) users: BTreeMap, + /// Every service account except the site replicator's own, already + /// shaped as the `service-account` create item the live hook emits. + pub(crate) service_accounts: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct SiteReplicationUserCredential { + pub(crate) secret_key: String, + pub(crate) status: AccountStatus, + /// The user record's own update time (the axis the receiver's staleness + /// check compares against), unlike `UserInfo::updated_at` which + /// `list_users` overwrites with the policy mapping's time. + pub(crate) updated_at: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct SiteReplicationServiceAccountSnapshot { + pub(crate) create: SRSvcAccCreate, + pub(crate) envelope: Option, + pub(crate) updated_at: Option, +} + +pub(crate) const SERVICE_ACCOUNT_ENVELOPE_VERSION: u64 = 2; + +pub(crate) fn encode_service_account_replication_policy( + claims: &HashMap, + session_policy: Option<&str>, +) -> S3Result<(SRSessionPolicy, Option)> { + if !claims.contains_key(OIDC_VIRTUAL_PARENT_CLAIM) { + return session_policy + .map(SRSessionPolicy::from_json) + .transpose() + .map(|policy| policy.unwrap_or_default()) + .map(|policy| (policy, None)) + .map_err(|err| s3_error!(InvalidArgument, "marshal policy failed: {:?}", err)); + } + + let policy = match session_policy { + Some(policy) => serde_json::from_str::(policy) + .map_err(|err| s3_error!(InvalidArgument, "invalid service account replication policy: {:?}", err))?, + None => Policy::default(), + }; + if policy.statements.is_empty() && (!policy.id.is_empty() || !policy.version.is_empty()) + || policy.version.is_empty() && !policy.statements.is_empty() + { + return Err(s3_error!(InvalidArgument, "service account replication policy is not normalized")); + } + let policy = serde_json::to_string(&policy) + .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?; + let policy = SRSessionPolicy::from_json(&policy) + .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?; + Ok(( + policy, + Some(SRSvcAccReplicationEnvelope { + version: SERVICE_ACCOUNT_ENVELOPE_VERSION, + }), + )) +} + +/// Read the credentials the IAM snapshot needs straight from the IAM store: +/// `list_users` deliberately strips secret keys and skips service accounts, +/// which is right for an admin listing and wrong for a peer snapshot (the +/// plan builder used to drop every user for lack of a secret, so a status +/// change or secret rotation committed while a peer was unreachable never +/// reached it — backlog#2289). +pub(crate) async fn build_sr_iam_credentials() -> S3Result { + let mut credentials = SiteReplicationIamCredentials::default(); + let Some(iam_sys) = current_iam_handle() else { + return Ok(credentials); + }; + + let mut users = HashMap::new(); + iam_sys.load_users(UserType::Reg, &mut users).await.map_err(ApiError::from)?; + for (access_key, identity) in users { + if identity.credentials.is_temp() || identity.credentials.is_service_account() { + continue; + } + credentials.users.insert( + access_key, + SiteReplicationUserCredential { + secret_key: identity.credentials.secret_key, + status: if identity.credentials.status == "off" { + AccountStatus::Disabled + } else { + AccountStatus::Enabled + }, + updated_at: identity.update_at, + }, + ); + } + + let mut service_accounts = HashMap::new(); + iam_sys + .load_users(UserType::Svc, &mut service_accounts) + .await + .map_err(ApiError::from)?; + let mut service_accounts: Vec<_> = service_accounts.into_iter().collect(); + service_accounts.sort_by(|(a, _), (b, _)| a.cmp(b)); + for (access_key, identity) in service_accounts { + // The replicator account is installed by join / rotate, never by a snapshot. + if access_key == SITE_REPLICATOR_SERVICE_ACCOUNT || !identity.credentials.is_service_account() { + continue; + } + let claims = iam_sys.get_claims_for_svc_acc(&access_key).await.map_err(ApiError::from)?; + let (account, session_policy) = iam_sys.get_service_account(&access_key).await.map_err(ApiError::from)?; + let session_policy = session_policy + .map(|policy| serde_json::to_string(&policy)) + .transpose() + .map_err(|err| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("marshal service account session policy failed: {err:?}"), + ) + })?; + let (session_policy, envelope) = encode_service_account_replication_policy(&claims, session_policy.as_deref())?; + credentials.service_accounts.push(SiteReplicationServiceAccountSnapshot { + create: SRSvcAccCreate { + parent: identity.credentials.parent_user, + access_key, + secret_key: identity.credentials.secret_key, + groups: identity.credentials.groups.unwrap_or_default(), + claims, + session_policy, + status: identity.credentials.status, + name: account.name.unwrap_or_default(), + description: account.description.unwrap_or_default(), + expiration: account.expiration, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }, + envelope, + updated_at: identity.update_at, + }); + } + + Ok(credentials) +} + +/// The bootstrap plan for peer delivery: `info` (secret-free) plus the IAM +/// credentials read at this moment. +pub(crate) async fn build_site_replication_bootstrap_plan(info: &SRInfo) -> S3Result { + let credentials = build_sr_iam_credentials().await?; + site_replication_bootstrap_plan(info, &credentials) +} + +pub(crate) fn site_replication_bootstrap_plan( + info: &SRInfo, + credentials: &SiteReplicationIamCredentials, +) -> S3Result { let mut plan = SiteReplicationBootstrapPlan::default(); let replicate_ilm_expiry = site_replication_info_replicates_ilm_expiry(info); @@ -318,24 +475,57 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result S3Result<()> { let Some(runtime) = runtime_site_replication_targets().await? else { return Ok(()); }; + // A local revoke must out-rank a stale grant a peer delivers later, so its + // mark is committed before the broadcast (backlog#2291). The broadcast + // still goes out when the mark cannot be persisted: the peers' own records + // remain the primary gate, the mark only covers the deleted case. + if let Err(err) = record_iam_deletion_marks_for_item(&item).await { + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + item_type = %item.r#type, + result = "iam_deletion_mark_not_recorded", + error = ?err, + "failed to record local IAM deletion mark before broadcast" + ); + } let mut first_error: Option = None; for peer in runtime.state.peers.values() { if peer.deployment_id == runtime.local_peer.deployment_id diff --git a/rustfs/src/site_replication/mod.rs b/rustfs/src/site_replication/mod.rs index 630c50576..675aa5b5d 100644 --- a/rustfs/src/site_replication/mod.rs +++ b/rustfs/src/site_replication/mod.rs @@ -79,13 +79,16 @@ use http::header::{CONTENT_TYPE, HOST}; use http::{HeaderMap, HeaderValue, Uri}; use hyper::{Method, StatusCode}; use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_TLS_PATH}; +use rustfs_iam::federation::OIDC_VIRTUAL_PARENT_CLAIM; use rustfs_iam::store::{MappedPolicy, UserType, sr_wire_user_type}; use rustfs_iam::sys::SITE_REPLICATOR_SERVICE_ACCOUNT; use rustfs_madmin::{ - AddOrUpdateUserReq, GroupAddRemove, GroupStatus, PeerInfo, PeerSite, ReplicateEditStatus, SITE_REPL_API_VERSION, - SRBucketInfo, SRBucketMeta, SRGroupInfo, SRIAMItem, SRIAMPolicy, SRInfo, SRPolicyMapping, SRRemoveReq, SRResyncOpStatus, - SRRetryStats, SRStateInfo, SyncStatus, + AccountStatus, AddOrUpdateUserReq, GroupAddRemove, GroupStatus, PeerInfo, PeerSite, ReplicateEditStatus, + SITE_REPL_API_VERSION, SRBucketInfo, SRBucketMeta, SRGroupInfo, SRIAMItem, SRIAMPolicy, SRInfo, SRPolicyMapping, SRRemoveReq, + SRResyncOpStatus, SRRetryStats, SRSessionPolicy, SRStateInfo, SRSvcAccChange, SRSvcAccCreate, SRSvcAccDelete, + SRSvcAccReplicationEnvelope, SyncStatus, }; +use rustfs_policy::policy::Policy; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration}; @@ -107,6 +110,26 @@ use tracing::{info, warn}; use url::{Url, form_urlencoded}; use uuid::Uuid; +/// Serialize `value` with every JSON object's keys sorted, for hashing and +/// equality checks. `HashMap` fields (service-account claims) iterate in a +/// per-instance random order and `serde_json` is built with `preserve_order`, +/// so two identical plans would otherwise hash differently: the repair +/// preflight token went stale between dry-run and execute, and a retry +/// snapshot resend never looked "stable" (backlog#2289 follow-up). +pub(crate) fn canonical_json_vec(value: &T) -> serde_json::Result> { + fn sort_keys(value: Value) -> Value { + match value { + Value::Object(map) => { + let sorted: BTreeMap = map.into_iter().map(|(key, value)| (key, sort_keys(value))).collect(); + Value::Object(sorted.into_iter().collect()) + } + Value::Array(items) => Value::Array(items.into_iter().map(sort_keys).collect()), + other => other, + } + } + serde_json::to_vec(&sort_keys(serde_json::to_value(value)?)) +} + pub(crate) const LOG_COMPONENT_ADMIN: &str = "admin"; pub(crate) const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication"; diff --git a/rustfs/src/site_replication/repair.rs b/rustfs/src/site_replication/repair.rs index b2785c0fd..ec341d70f 100644 --- a/rustfs/src/site_replication/repair.rs +++ b/rustfs/src/site_replication/repair.rs @@ -234,9 +234,9 @@ impl SiteReplicationRepairTask<'_> { pub(crate) fn id(&self) -> S3Result { let payload = match self { - Self::Iam(item) => serde_json::to_vec(item), + Self::Iam(item) => canonical_json_vec(item), Self::BucketMake(_) | Self::Replication(_) => serde_json::to_vec(&serde_json::json!({})), - Self::BucketMetadata(item) => serde_json::to_vec(item), + Self::BucketMetadata(item) => canonical_json_vec(item), } .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair task failed: {err}")))?; let mut digest = Sha256::new(); @@ -726,7 +726,7 @@ pub(crate) async fn execute_site_replication_repair_locked( return Err(s3_error!(InvalidRequest, "site replication is not configured")); } let info = build_sr_info(&state, &request.local_peer).await?; - let plan = site_replication_bootstrap_plan(&info)?; + let plan = build_site_replication_bootstrap_plan(&info).await?; let plan_token = site_replication_repair_plan_token(&state, &plan)?; let preflight_token = site_replication_repair_preflight_token(&state, &plan, request.signing_key.as_bytes())?; let sites = site_replication_repair_sites(&state, &request.local_peer, &plan, request.signing_key.as_bytes())?; diff --git a/rustfs/src/site_replication/retry.rs b/rustfs/src/site_replication/retry.rs index 4b818c752..c9b9e73a0 100644 --- a/rustfs/src/site_replication/retry.rs +++ b/rustfs/src/site_replication/retry.rs @@ -397,12 +397,12 @@ pub(crate) fn iam_deletion_replay_matches(record: &SiteReplicationIamDeletionRep /// newer revision of one another. pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option { match item.r#type.as_str() { - "policy" if item.policy.is_none() => Some(format!("policy:{}", item.name)), + "policy" if item.policy.is_none() => Some(iam_policy_deletion_mark_entity(&item.name)), "iam-user" => item .iam_user .as_ref() .filter(|user| user.is_delete_req) - .map(|user| format!("iam-user:{}", user.access_key)), + .map(|user| iam_user_deletion_mark_entity(&user.access_key)), "group-info" => item .group_info .as_ref() @@ -416,7 +416,7 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option { .policy_mapping .as_ref() .filter(|mapping| mapping.policy.is_empty()) - .map(|mapping| format!("policy-mapping:{}:{}:{}", mapping.user_or_group, mapping.user_type, mapping.is_group)), + .map(|mapping| iam_policy_mapping_deletion_mark_entity(&mapping.user_or_group, mapping.user_type, mapping.is_group)), "service-account" => item .svc_acc_change .as_ref() @@ -426,6 +426,82 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option { } } +/// The entities whose deletion a deletion-shaped IAM item commits, keyed the +/// way the receive-side staleness gate looks them up once the local record is +/// gone (backlog#2291); empty for creates and updates. Group member removal +/// yields one entity per removed member so a stale re-add of that member can +/// be judged, and a group delete (no members) yields the group itself. +pub(crate) fn iam_item_deletion_mark_entities(item: &SRIAMItem) -> Vec { + if item.r#type == "group-info" { + let Some(update) = item + .group_info + .as_ref() + .map(|group| &group.update_req) + .filter(|update| update.is_remove) + else { + return Vec::new(); + }; + if update.members.is_empty() { + return vec![iam_group_deletion_mark_entity(&update.group)]; + } + return update + .members + .iter() + .map(|member| iam_group_member_deletion_mark_entity(&update.group, member)) + .collect(); + } + iam_item_deletion_entity(item).into_iter().collect() +} + +pub(crate) fn iam_policy_deletion_mark_entity(name: &str) -> String { + format!("policy:{name}") +} + +pub(crate) fn iam_user_deletion_mark_entity(access_key: &str) -> String { + format!("iam-user:{access_key}") +} + +/// `user_type` is the SR wire integer, as carried by the item on both sides. +pub(crate) fn iam_policy_mapping_deletion_mark_entity(user_or_group: &str, user_type: i64, is_group: bool) -> String { + format!("policy-mapping:{user_or_group}:{user_type}:{is_group}") +} + +pub(crate) fn iam_group_deletion_mark_entity(group: &str) -> String { + format!("group:{group}") +} + +pub(crate) fn iam_group_member_deletion_mark_entity(group: &str, member: &str) -> String { + format!("group-member:{group}:{member}") +} + +/// Persist the deletion marks of `item` (its source `updated_at` per entity +/// of [`iam_item_deletion_mark_entities`]) through the state transaction. +/// No-op for creates/updates and for items without a source timestamp +/// (older peers): a mark without a source clock could not be ordered against +/// later items. Called before a local deletion is broadcast and after a +/// replicated deletion is applied, so both sides out-rank a stale grant that +/// arrives later. +pub(crate) async fn record_iam_deletion_marks_for_item(item: &SRIAMItem) -> S3Result<()> { + let entities = iam_item_deletion_mark_entities(item); + let Some(deleted_at) = item.updated_at.filter(|_| !entities.is_empty()) else { + return Ok(()); + }; + commit_iam_deletion_marks(entities, deleted_at).await +} + +/// [`record_iam_deletion_marks`] under the state transaction; the write is +/// skipped when no mark moves. +pub(crate) async fn commit_iam_deletion_marks(entities: Vec, deleted_at: OffsetDateTime) -> S3Result<()> { + update_site_replication_state_when_changed(move |state| { + Ok(if record_iam_deletion_marks(state, &entities, deleted_at) { + StateCommit::Changed(()) + } else { + StateCommit::Unchanged(()) + }) + }) + .await +} + /// Failure bookkeeping for one IAM item delivery: upsert the collapsed retry /// event and, when the item is a deletion, record its body for replay. Both /// live in the same state so the caller commits them in one transaction — a @@ -791,8 +867,8 @@ impl RetrySnapshot { pub(crate) fn fingerprint(&self) -> S3Result>> { let mut payloads = match self { - Self::Iam(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), - Self::BucketMetadata(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), + Self::Iam(items) => items.iter().map(canonical_json_vec).collect::, _>>(), + Self::BucketMetadata(items) => items.iter().map(canonical_json_vec).collect::, _>>(), } .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry snapshot failed: {err}")))?; payloads.sort_unstable(); @@ -954,6 +1030,7 @@ pub(crate) enum IamSnapshotKey { User(String), Group(String), PolicyMapping { target: String, user_type: i64, is_group: bool }, + ServiceAccount(String), } pub(crate) fn iam_snapshot_key(item: &SRIAMItem) -> Option { @@ -972,6 +1049,11 @@ pub(crate) fn iam_snapshot_key(item: &SRIAMItem) -> Option { user_type: mapping.user_type, is_group: mapping.is_group, }), + "service-account" => item + .svc_acc_change + .as_ref() + .and_then(|change| change.create.as_ref()) + .map(|create| IamSnapshotKey::ServiceAccount(create.access_key.clone())), _ => None, } } @@ -1006,6 +1088,24 @@ pub(crate) fn iam_snapshot_tombstones(item: &SRIAMItem, observed_at: OffsetDateT mapping.policy.clear(); } } + "service-account" => { + let Some(access_key) = item + .svc_acc_change + .as_ref() + .and_then(|change| change.create.as_ref()) + .map(|create| create.access_key.clone()) + else { + return Vec::new(); + }; + tombstone.svc_acc_change = Some(SRSvcAccChange { + delete: Some(SRSvcAccDelete { + access_key, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }); + } _ => return Vec::new(), } vec![tombstone] @@ -1701,7 +1801,7 @@ pub(crate) async fn drain_site_replication_retry_queue_locked( // tick and only when a snapshot resend is actually due. let plan = if needs_plan { let info = build_sr_info(&runtime.state, &runtime.local_peer).await?; - Some(site_replication_bootstrap_plan(&info)?) + Some(build_site_replication_bootstrap_plan(&info).await?) } else { None }; @@ -1841,7 +1941,7 @@ pub(crate) async fn drain_one_site_replication_retry_event( } } let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?; - let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?; + let fresh_plan = build_site_replication_bootstrap_plan(&fresh_info).await?; let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot"); if fresh_snapshot.fingerprint()? == current_fingerprint { if is_iam { diff --git a/rustfs/src/site_replication/state.rs b/rustfs/src/site_replication/state.rs index 04619a9ac..efc8b16e5 100644 --- a/rustfs/src/site_replication/state.rs +++ b/rustfs/src/site_replication/state.rs @@ -64,6 +64,104 @@ pub(crate) struct SiteReplicationState { /// newer edit that already landed. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub(crate) applied_edit_generations: BTreeMap, + /// Source timestamp of the newest IAM deletion committed on this site, + /// keyed by the deleted entity (`iam_item_deletion_mark_entities`). A + /// deletion leaves no local record to judge a later item against, so this + /// is what lets the receive-side staleness gate reject a grant that is + /// older than the revoke it would otherwise undo (backlog#2291). Marks + /// are kept for [`SITE_REPLICATION_IAM_DELETION_MARK_RETENTION`] and never + /// evicted by count: see that constant for why a count bound would open + /// exactly the window the marks exist to close. + #[serde(default, with = "rfc3339_map", skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) iam_deletion_marks: BTreeMap, +} + +/// How long an IAM deletion mark outlives the deletion it records. +/// +/// A mark fences the delivery paths that can still carry an older grant for +/// the deleted entity: a live delivery delayed in transit, the same grant +/// arriving on a sibling node while the revoke is being applied, and a +/// snapshot (bootstrap / repair / resend) built by a peer that has not yet +/// received the deletion — which is bounded by this site's own retry queue +/// towards that peer, whose backoff tops out at one day +/// (`SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS`). The retry drain itself +/// never replays a stale grant: it resends snapshots of the current records +/// and the recorded deletion bodies. Thirty days is an order of magnitude +/// beyond every one of those windows. Marks are pruned by age only — a count +/// bound would drop a mark that is still inside the delivery window as soon +/// as enough newer deletions happen, letting the delayed grant re-create the +/// entity, which is the very hole the marks close. +pub(crate) const SITE_REPLICATION_IAM_DELETION_MARK_RETENTION: time::Duration = time::Duration::days(30); + +/// Record that deletions of `entities` with source timestamp `deleted_at` +/// were committed here. Newest wins per entity: an older deletion never +/// lowers a mark. Marks older than the retention are pruned in the same +/// pass. Returns whether the state changed. +pub(crate) fn record_iam_deletion_marks( + state: &mut SiteReplicationState, + entities: &[String], + deleted_at: OffsetDateTime, +) -> bool { + record_iam_deletion_marks_at(state, entities, deleted_at, OffsetDateTime::now_utc()) +} + +/// [`record_iam_deletion_marks`] pruning against an explicit `now`. +pub(crate) fn record_iam_deletion_marks_at( + state: &mut SiteReplicationState, + entities: &[String], + deleted_at: OffsetDateTime, + now: OffsetDateTime, +) -> bool { + let mut changed = false; + for entity in entities { + if state + .iam_deletion_marks + .get(entity) + .is_some_and(|existing| *existing >= deleted_at) + { + continue; + } + state.iam_deletion_marks.insert(entity.clone(), deleted_at); + changed = true; + } + let expired_before = now - SITE_REPLICATION_IAM_DELETION_MARK_RETENTION; + let before = state.iam_deletion_marks.len(); + state.iam_deletion_marks.retain(|_, deleted_at| *deleted_at >= expired_before); + changed || state.iam_deletion_marks.len() != before +} + +/// Newest deletion mark among `entities`, or `None` when no deletion of any +/// of them was recorded here. The receive-side staleness gate feeds this in +/// as the local timestamp when the targeted record is absent. +pub(crate) fn iam_deletion_mark(state: &SiteReplicationState, entities: &[String]) -> Option { + entities + .iter() + .filter_map(|entity| state.iam_deletion_marks.get(entity).copied()) + .max() +} + +/// RFC 3339 map values, matching the other timestamps in the state object +/// (`time::serde::rfc3339` only applies to a single field). +mod rfc3339_map { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use std::collections::BTreeMap; + use time::OffsetDateTime; + + #[derive(Serialize, Deserialize)] + #[serde(transparent)] + struct Stamp(#[serde(with = "time::serde::rfc3339")] OffsetDateTime); + + pub(super) fn serialize(map: &BTreeMap, serializer: S) -> Result { + serializer.collect_map(map.iter().map(|(entity, deleted_at)| (entity, Stamp(*deleted_at)))) + } + + pub(super) fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let map = BTreeMap::::deserialize(deserializer)?; + Ok(map + .into_iter() + .map(|(entity, Stamp(deleted_at))| (entity, deleted_at)) + .collect()) + } } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -323,6 +421,33 @@ where update_site_replication_state_when_changed(move |state| update(state).map(StateCommit::Changed)).await } +/// The state transaction for work that has to await inside it: an IAM write +/// that must be ordered with the staleness verdict taken before it and the +/// deletion mark committed after it (backlog#2291). Same boundary as +/// [`update_site_replication_state`] — load and persist under the +/// distributed state-object write lock, so two nodes of this site cannot +/// interleave their verdicts and writes — and the same rules inside: no peer +/// network calls and no other config locks. The closure hands the state back +/// as `Some` when it changed it; `None` skips the write. +pub(crate) async fn with_site_replication_state_transaction(transaction: F) -> S3Result +where + T: Send + 'static, + F: FnOnce(SiteReplicationState) -> Fut + Send + 'static, + Fut: std::future::Future)>> + Send + 'static, +{ + with_site_replication_state_lock(move || async move { + let store = current_object_store_handle() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + let state = load_site_replication_state_no_lock(store.clone()).await?; + let (result, changed) = transaction(state).await?; + if let Some(state) = changed { + persist_site_replication_state_no_lock(store, state).await?; + } + Ok(result) + }) + .await +} + /// [`update_site_replication_state`] for closures that may find nothing to /// do — see [`StateCommit`]. pub(crate) async fn update_site_replication_state_when_changed(update: F) -> S3Result diff --git a/rustfs/src/site_replication/tests.rs b/rustfs/src/site_replication/tests.rs index 3b2c48666..f1c06131f 100644 --- a/rustfs/src/site_replication/tests.rs +++ b/rustfs/src/site_replication/tests.rs @@ -554,6 +554,145 @@ fn test_iam_item_deletion_entity_shapes() { assert!(iam_item_deletion_entity(&policy_set).is_none()); } +/// Deletion marks (backlog#2291) key on the same entities as the replay +/// records, except that a group member removal is marked per member (so a +/// stale re-add of one member can be judged) and a group delete marks the +/// group itself. Creates and updates leave no mark. +#[test] +fn test_iam_item_deletion_mark_entities_shapes() { + assert_eq!( + iam_item_deletion_mark_entities(&user_delete_item("alice")), + vec!["iam-user:alice".to_string()] + ); + assert_eq!( + iam_item_deletion_mark_entities(&policy_delete_item("readonly")), + vec!["policy:readonly".to_string()] + ); + + let mut group_remove = SRIAMItem { + r#type: "group-info".to_string(), + group_info: Some(SRGroupInfo { + update_req: GroupAddRemove { + group: "devs".to_string(), + members: vec!["bob".to_string(), "alice".to_string()], + status: GroupStatus::Enabled, + is_remove: true, + }, + api_version: None, + }), + ..Default::default() + }; + assert_eq!( + iam_item_deletion_mark_entities(&group_remove), + vec!["group-member:devs:bob".to_string(), "group-member:devs:alice".to_string()] + ); + group_remove + .group_info + .as_mut() + .expect("group info") + .update_req + .members + .clear(); + assert_eq!( + iam_item_deletion_mark_entities(&group_remove), + vec!["group:devs".to_string()], + "a removal without members deletes the group" + ); + group_remove.group_info.as_mut().expect("group info").update_req.is_remove = false; + assert!(iam_item_deletion_mark_entities(&group_remove).is_empty()); + + let mapping_clear = SRIAMItem { + r#type: "policy-mapping".to_string(), + policy_mapping: Some(SRPolicyMapping { + user_or_group: "alice".to_string(), + user_type: 0, + is_group: false, + policy: String::new(), + ..Default::default() + }), + ..Default::default() + }; + assert_eq!( + iam_item_deletion_mark_entities(&mapping_clear), + vec!["policy-mapping:alice:0:false".to_string()] + ); + + let mut user_create = user_delete_item("alice"); + user_create.iam_user.as_mut().expect("iam user").is_delete_req = false; + assert!(iam_item_deletion_mark_entities(&user_create).is_empty()); +} + +/// Newest wins per entity, marks are pruned by age only (never by count: a +/// count bound would drop a mark still inside the delivery window as soon as +/// enough newer deletions happen), and the timestamps survive the state +/// object as RFC 3339. +#[test] +fn test_record_iam_deletion_marks_newest_wins_and_expires_by_age_only() { + let at = |seconds: i64| OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(seconds); + let now = at(1_000_000); + let mut state = SiteReplicationState::default(); + let alice = vec!["iam-user:alice".to_string()]; + + assert!(record_iam_deletion_marks_at(&mut state, &alice, at(20), now)); + assert!( + !record_iam_deletion_marks_at(&mut state, &alice, at(10), now), + "an older deletion does not move the mark" + ); + assert!( + !record_iam_deletion_marks_at(&mut state, &alice, at(20), now), + "a replayed deletion is not a change" + ); + assert_eq!(iam_deletion_mark(&state, &alice), Some(at(20))); + assert!(record_iam_deletion_marks_at(&mut state, &alice, at(30), now)); + assert_eq!(iam_deletion_mark(&state, &alice), Some(at(30))); + assert_eq!(iam_deletion_mark(&state, &["iam-user:bob".to_string()]), None); + assert!(!record_iam_deletion_marks_at(&mut state, &[], at(40), now)); + + // Many newer deletions never evict an older mark that is still within the retention. + let members: Vec = (0..4096).map(|index| format!("group-member:devs:user-{index:04}")).collect(); + for (index, member) in members.iter().enumerate() { + record_iam_deletion_marks_at(&mut state, std::slice::from_ref(member), at(100 + index as i64), now); + } + assert_eq!(state.iam_deletion_marks.len(), members.len() + 1); + assert_eq!(iam_deletion_mark(&state, &alice), Some(at(30)), "no count-based eviction"); + + // Marks older than the retention are pruned, on the pass that records a + // newer one and on a pass that changes nothing else; younger ones stay. + let later = at(100) + SITE_REPLICATION_IAM_DELETION_MARK_RETENTION; + assert!( + record_iam_deletion_marks_at(&mut state, &["iam-user:carol".to_string()], at(200_000), later), + "pruning alone is a change" + ); + assert_eq!(iam_deletion_mark(&state, &alice), None, "alice's mark aged out"); + assert_eq!( + iam_deletion_mark(&state, &members[..1]), + Some(at(100)), + "a mark exactly at the retention edge stays, and so do the younger ones" + ); + assert_eq!(state.iam_deletion_marks.len(), members.len() + 1); + assert_eq!(iam_deletion_mark(&state, &["iam-user:carol".to_string()]), Some(at(200_000))); + let mut state = SiteReplicationState::default(); + record_iam_deletion_marks_at(&mut state, &alice, at(30), now); + let past_edge = at(30) + SITE_REPLICATION_IAM_DELETION_MARK_RETENTION + time::Duration::seconds(1); + assert!( + record_iam_deletion_marks_at(&mut state, &[], at(0), past_edge), + "a pass that only prunes reports the change" + ); + assert_eq!(iam_deletion_mark(&state, &alice), None); + record_iam_deletion_marks_at(&mut state, &alice, at(30), now); + + let json = serde_json::to_value(&state).expect("serialize state"); + assert_eq!(json["iam_deletion_marks"]["iam-user:alice"], serde_json::json!("1970-01-01T00:00:30Z")); + let reloaded = parse_site_replication_state(&serde_json::to_vec(&state).expect("serialize state")).expect("parse state"); + assert_eq!(reloaded.iam_deletion_marks, state.iam_deletion_marks); + assert!( + parse_site_replication_state(br#"{"name":"a","service_account_access_key":"","service_account_parent":"","peers":{},"updated_at":null,"resync_status":{}}"#) + .expect("state without marks") + .iam_deletion_marks + .is_empty() + ); +} + /// A failed deletion delivery persists a replay record next to the collapsed /// retry entry; a fresh entry is stamped `deletions_recorded` so a later /// replay can settle it, and a repeated deletion of the same entity keeps the @@ -1679,7 +1818,8 @@ fn test_site_replication_bootstrap_plan_includes_replayable_snapshot_items() { }, ); - let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build"); + let plan = + site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build"); assert_eq!(plan.iam_items.iter().map(|item| item.r#type.as_str()).collect::>(), { vec!["policy", "iam-user", "group-info", "policy-mapping"] @@ -1717,7 +1857,8 @@ fn test_site_replication_bootstrap_plan_skips_lifecycle_by_default() { }, ); - let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build"); + let plan = + site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build"); assert!(!plan.bucket_items.iter().any(|item| item.r#type == "lc-config")); } @@ -1748,7 +1889,8 @@ fn test_site_replication_bootstrap_plan_emits_timestamped_lifecycle_delete() { }, ); - let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build"); + let plan = + site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build"); let item = plan .bucket_items @@ -1935,8 +2077,8 @@ fn test_site_replication_repair_preflight_token_is_deterministic_for_equal_state }, ); - let plan_a = site_replication_bootstrap_plan(&info).expect("first plan"); - let plan_b = site_replication_bootstrap_plan(&info).expect("second plan"); + let plan_a = site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("first plan"); + let plan_b = site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("second plan"); let token_a = site_replication_repair_preflight_token(&state, &plan_a, b"test-signing-key").expect("first token"); let token_b = site_replication_repair_preflight_token(&state, &plan_b, b"test-signing-key").expect("second token"); @@ -3219,3 +3361,345 @@ fn test_reconcile_adds_missing_peer_rules_to_existing_config() { assert!(rule_ids.contains(&"site-repl-dep-b")); assert!(rule_ids.contains(&"site-repl-dep-c")); } + +/// backlog#2289: the IAM snapshot (retry resend, repair, site-add bootstrap) +/// used to be built from `list_users`, whose `UserInfo` never carries a +/// secret key, so the plan dropped every user and a status change or secret +/// rotation committed while a peer was unreachable never reached it. The +/// credentials now come from a separate store read; SRInfo stays secret-free. +#[test] +fn test_bootstrap_plan_carries_users_from_the_credential_snapshot() { + let mut info = SRInfo::default(); + // Exactly what `list_users` builds: status, policy, updated_at — never secret_key. + info.user_info_map.insert( + "alice".to_string(), + rustfs_madmin::UserInfo { + status: rustfs_madmin::AccountStatus::Disabled, + policy_name: Some("readwrite".to_string()), + updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")), + ..Default::default() + }, + ); + info.user_info_map.insert( + "external-idp-user".to_string(), + rustfs_madmin::UserInfo { + status: rustfs_madmin::AccountStatus::Enabled, + ..Default::default() + }, + ); + let user_updated_at = OffsetDateTime::from_unix_timestamp(1_700_000_500).expect("timestamp"); + let mut credentials = SiteReplicationIamCredentials::default(); + credentials.users.insert( + "alice".to_string(), + SiteReplicationUserCredential { + secret_key: "alice-secret".to_string(), + status: rustfs_madmin::AccountStatus::Disabled, + updated_at: Some(user_updated_at), + }, + ); + + let plan = site_replication_bootstrap_plan(&info, &credentials).expect("bootstrap plan should build"); + + let users: Vec<_> = plan.iam_items.iter().filter(|item| item.r#type == "iam-user").collect(); + assert_eq!(users.len(), 1, "only the user with a credential travels: {:?}", plan.iam_items); + let alice = users[0].iam_user.as_ref().expect("iam user body"); + assert_eq!(alice.access_key, "alice"); + let req = alice.user_req.as_ref().expect("user request"); + assert_eq!(req.secret_key, "alice-secret"); + assert_eq!(req.status, rustfs_madmin::AccountStatus::Disabled); + assert_eq!(req.policy.as_deref(), Some("readwrite")); + // the user record's own axis, not the policy-mapping time list_users reports + assert_eq!(users[0].updated_at, Some(user_updated_at)); +} + +fn service_account_snapshot(access_key: &str, parent: &str, status: &str) -> SiteReplicationServiceAccountSnapshot { + SiteReplicationServiceAccountSnapshot { + create: rustfs_madmin::SRSvcAccCreate { + parent: parent.to_string(), + access_key: access_key.to_string(), + secret_key: format!("{access_key}-secret"), + groups: Vec::new(), + claims: HashMap::new(), + session_policy: SRSessionPolicy::default(), + status: status.to_string(), + name: String::new(), + description: String::new(), + expiration: None, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }, + envelope: None, + updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_600).expect("timestamp")), + } +} + +/// backlog#2289: service accounts were absent from every snapshot (the +/// listing filters them). They now travel as the create item the live hook +/// emits — after their parents — carrying secret and status. +#[test] +fn test_bootstrap_plan_emits_service_accounts_after_their_parents() { + let mut info = SRInfo::default(); + info.user_info_map + .insert("alice".to_string(), rustfs_madmin::UserInfo::default()); + let mut credentials = SiteReplicationIamCredentials::default(); + credentials.users.insert( + "alice".to_string(), + SiteReplicationUserCredential { + secret_key: "alice-secret".to_string(), + status: rustfs_madmin::AccountStatus::Enabled, + updated_at: None, + }, + ); + credentials + .service_accounts + .push(service_account_snapshot("alice-svc", "alice", "off")); + + let plan = site_replication_bootstrap_plan(&info, &credentials).expect("bootstrap plan should build"); + + let types: Vec<_> = plan.iam_items.iter().map(|item| item.r#type.as_str()).collect(); + assert_eq!(types, vec!["iam-user", "service-account"]); + let change = plan.iam_items[1].svc_acc_change.as_ref().expect("service account change"); + let create = change.create.as_ref().expect("create body"); + assert_eq!((create.access_key.as_str(), create.parent.as_str()), ("alice-svc", "alice")); + assert_eq!(create.secret_key, "alice-svc-secret"); + assert_eq!(create.status, "off", "a disabled account must arrive disabled"); + assert!(change.delete.is_none() && change.update.is_none()); +} + +/// A service account present in the previous snapshot but gone from the +/// fresh one is replayed as an explicit delete, like the other IAM kinds. +#[test] +fn test_retry_snapshot_tombstones_removed_service_accounts() { + let observed_at = OffsetDateTime::from_unix_timestamp(1_700_001_000).expect("timestamp"); + let mut info = SRInfo::default(); + info.user_info_map + .insert("alice".to_string(), rustfs_madmin::UserInfo::default()); + let mut credentials = SiteReplicationIamCredentials::default(); + credentials.users.insert( + "alice".to_string(), + SiteReplicationUserCredential { + secret_key: "alice-secret".to_string(), + status: rustfs_madmin::AccountStatus::Enabled, + updated_at: None, + }, + ); + let mut with_account = credentials.clone(); + with_account + .service_accounts + .push(service_account_snapshot("alice-svc", "alice", "on")); + let previous = site_replication_bootstrap_plan(&info, &with_account).expect("previous plan"); + let fresh = site_replication_bootstrap_plan(&info, &credentials).expect("fresh plan"); + + let replay = RetrySnapshot::replay_after_change( + &RetrySnapshot::Iam(previous.iam_items), + &RetrySnapshot::Iam(fresh.iam_items), + observed_at, + ); + let RetrySnapshot::Iam(items) = replay else { + panic!("IAM snapshot expected"); + }; + let tombstone = items + .iter() + .find(|item| item.r#type == "service-account") + .expect("service account tombstone"); + let change = tombstone.svc_acc_change.as_ref().expect("change"); + assert_eq!(change.delete.as_ref().map(|delete| delete.access_key.as_str()), Some("alice-svc")); + assert!(change.create.is_none()); + assert_eq!(tombstone.updated_at, Some(observed_at)); +} + +/// Spawns a one-shot HTTP peer that answers 200 and flips the returned flag +/// once a request head has arrived. +async fn spawn_reached_probe_peer() -> (String, Arc, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind healthy peer"); + let endpoint = format!("http://{}", listener.local_addr().expect("healthy peer address")); + let reached = Arc::new(AtomicBool::new(false)); + let reached_by_server = reached.clone(); + let server = tokio::spawn(async move { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let Ok(read) = stream.read(&mut buffer).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + reached_by_server.store(true, Ordering::SeqCst); + let _ = stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nok") + .await; + }); + (endpoint, reached, server) +} + +/// Three-peer runtime whose local peer is `local`; BTreeMap order visits the +/// failing peer `b` before the healthy peer `c`. +fn broadcast_runtime_with_failing_peer_before_healthy(failing_endpoint: &str, healthy_endpoint: &str) -> SiteReplicationRuntime { + let local_peer = PeerInfo { + deployment_id: "local".to_string(), + ..peer("local", "http://127.0.0.1:9") + }; + let mut state = SiteReplicationState { + name: "local".to_string(), + service_account_access_key: "site-replicator-0".to_string(), + ..Default::default() + }; + state.peers.insert("local".to_string(), local_peer.clone()); + state.peers.insert( + "b".to_string(), + PeerInfo { + deployment_id: "b".to_string(), + ..peer("b", failing_endpoint) + }, + ); + state.peers.insert( + "c".to_string(), + PeerInfo { + deployment_id: "c".to_string(), + ..peer("c", healthy_endpoint) + }, + ); + SiteReplicationRuntime { + state, + local_peer, + service_account_secret_key: "site-replicator-secret".to_string(), + } +} + +const BROADCAST_PROBE_DELETE_BUCKET_PATH: &str = + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=delete-bucket"; + +/// The generic JSON broadcast (bucket make/delete, bucket-meta hook, bucket +/// ops) attempts every remote peer: a peer whose request fails must not stop +/// delivery to the peers that follow it in deployment-id order, and the +/// failure is still reported to the caller (backlog#2293). +#[tokio::test] +#[serial] +async fn test_broadcast_json_reaches_healthy_peers_after_a_failed_peer() { + // Peer "b": nothing listens on the port, so the connect is refused. + let refused = TcpListener::bind("127.0.0.1:0").await.expect("bind refused-peer probe"); + let refused_endpoint = format!("http://{}", refused.local_addr().expect("refused-peer address")); + drop(refused); + + let (healthy_endpoint, reached, server) = spawn_reached_probe_peer().await; + let runtime = broadcast_runtime_with_failing_peer_before_healthy(&refused_endpoint, &healthy_endpoint); + + let result = temp_env::async_with_vars([(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV, Some("true"))], async { + broadcast_site_replication_json_with_runtime(&runtime, BROADCAST_PROBE_DELETE_BUCKET_PATH, &serde_json::json!({})).await + }) + .await; + + let err = result.expect_err("peer b refuses connections, the broadcast must report it"); + assert!( + reached.load(Ordering::SeqCst), + "peer c never received the broadcast once peer b failed: {err}" + ); + server.abort(); +} + +/// Same guarantee when the failing peer never gets a transport: an endpoint +/// that `PeerTransport::for_runtime_peer` rejects must be skipped past (and +/// reported), not abort the broadcast before the healthy peers (backlog#2293). +#[tokio::test] +#[serial] +async fn test_broadcast_json_reaches_healthy_peers_after_a_peer_without_transport() { + // Peer "b": a scheme the peer connection validator refuses outright. + let forbidden_endpoint = "ftp://peer-b.example.com"; + + let (healthy_endpoint, reached, server) = spawn_reached_probe_peer().await; + let runtime = broadcast_runtime_with_failing_peer_before_healthy(forbidden_endpoint, &healthy_endpoint); + + let result = temp_env::async_with_vars([(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV, Some("true"))], async { + broadcast_site_replication_json_with_runtime(&runtime, BROADCAST_PROBE_DELETE_BUCKET_PATH, &serde_json::json!({})).await + }) + .await; + + let err = result.expect_err("peer b has no usable transport, the broadcast must report it"); + assert!( + err.to_string().contains("invalid persisted site replication peer"), + "the reported error must be peer b's transport failure: {err}" + ); + assert!( + reached.load(Ordering::SeqCst), + "peer c never received the broadcast once peer b failed to get a transport: {err}" + ); + server.abort(); +} + +fn service_account_item_with_claims(order: &[&str]) -> SRIAMItem { + let mut claims = HashMap::new(); + for key in order { + claims.insert((*key).to_string(), serde_json::json!(format!("value-of-{key}"))); + } + SRIAMItem { + r#type: "service-account".to_string(), + svc_acc_change: Some(SRSvcAccChange { + create: Some(rustfs_madmin::SRSvcAccCreate { + parent: "alice".to_string(), + access_key: "alice-svc".to_string(), + secret_key: "alice-svc-secret".to_string(), + groups: Vec::new(), + claims, + session_policy: SRSessionPolicy::default(), + status: "on".to_string(), + name: String::new(), + description: String::new(), + expiration: None, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }), + updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + } +} + +/// The repair preflight token and the retry-snapshot fingerprint hash the +/// serialized items. Service-account claims live in a `HashMap`, whose +/// iteration order differs between instances, so the hash must not depend on +/// it (the real-VM repair returned 412 "preflight is stale" between dry-run +/// and execute once snapshots carried service accounts). +#[test] +fn test_repair_task_id_and_retry_fingerprint_ignore_claim_map_order() { + let forward = service_account_item_with_claims(&["accessKey", "exp", "parent", "sa-policy", "sub", "tenant"]); + let backward = service_account_item_with_claims(&["tenant", "sub", "sa-policy", "parent", "exp", "accessKey"]); + + let canonical = canonical_json_vec(&forward).expect("canonical json"); + let text = String::from_utf8(canonical).expect("utf8"); + let positions: Vec = [ + "\"accessKey\"", + "\"exp\"", + "\"parent\"", + "\"sa-policy\"", + "\"sub\"", + "\"tenant\"", + ] + .iter() + .map(|key| text.find(key).expect("claim key present")) + .collect(); + assert!( + positions.windows(2).all(|pair| pair[0] < pair[1]), + "claim keys must serialize sorted: {text}" + ); + + assert_eq!( + SiteReplicationRepairTask::Iam(&forward).id().expect("id"), + SiteReplicationRepairTask::Iam(&backward).id().expect("id"), + "identical items must yield the same repair task id regardless of claim map order" + ); + assert_eq!( + RetrySnapshot::Iam(vec![forward]).fingerprint().expect("fingerprint"), + RetrySnapshot::Iam(vec![backward]).fingerprint().expect("fingerprint"), + "identical snapshots must fingerprint equal regardless of claim map order" + ); +} diff --git a/rustfs/src/site_replication/transport.rs b/rustfs/src/site_replication/transport.rs index bc941c9f0..18d4850b9 100644 --- a/rustfs/src/site_replication/transport.rs +++ b/rustfs/src/site_replication/transport.rs @@ -876,6 +876,14 @@ pub(crate) async fn broadcast_site_replication_json(path: &str, bo broadcast_site_replication_json_with_runtime(&runtime, path, body).await } +/// PUT `body` to `path` on every remote peer of the runtime. +/// +/// Every peer is attempted: one peer's failure — transport construction +/// included — must not skip the peers that follow it in deployment-id order, +/// or they silently miss the change with no retry record (backlog#2293). A +/// success settles the peer/path's queued retry event, a failure enqueues one +/// under the request `path` (so the drain classifies it as today), and the +/// first error is returned once all peers were attempted. pub(crate) async fn broadcast_site_replication_json_with_runtime( runtime: &SiteReplicationRuntime, path: &str, @@ -883,20 +891,30 @@ pub(crate) async fn broadcast_site_replication_json_with_runtime( ) -> S3Result<()> { let state = &runtime.state; let local_peer = &runtime.local_peer; + let mut first_error: Option = None; for peer in state.peers.values() { if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { continue; } - let transport = PeerTransport::for_runtime_peer(peer).await?; - PeerAdminRequest::put(&transport.connection, path, &state.service_account_access_key) - .with_client(&transport.client) - .send_with_retry_event(peer, &runtime.service_account_secret_key, body) - .await?; + let sent = match PeerTransport::for_runtime_peer(peer).await { + Ok(transport) => PeerAdminRequest::put(&transport.connection, path, &state.service_account_access_key) + .with_client(&transport.client) + .send_with_retry_event(peer, &runtime.service_account_secret_key, body) + .await + .map(|_| ()), + Err(err) => { + enqueue_site_replication_retry_event(peer, path, &err).await; + Err(err) + } + }; + if let Err(err) = sent { + first_error.get_or_insert(err); + } } - Ok(()) + first_error.map_or(Ok(()), Err) } pub(crate) fn parse_endpoint_refresh_status(peer: &PeerInfo, body: &[u8]) -> S3Result<()> { From 1a88870809896c989465540b519afc74731d7f33 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 14:12:56 +0800 Subject: [PATCH 09/20] fix(odm): preserve cursor compatibility and native source semantics (#7238) --- .../on_demand_migration/get_response.json | 2 +- .../on_demand_migration/set_request.json | 2 +- .../on_demand_migration/set_response.json | 2 +- .../on_demand_migration/source_config_e2a.rs | 80 ++ crates/madmin/src/on_demand_migration.rs | 40 +- docs/architecture/compat-cleanup-register.md | 1 + docs/operations/on-demand-migration.md | 24 +- .../on_demand_migration/source_config_e2a.rs | 80 ++ rustfs/src/app/bucket_list_through.rs | 1095 ++++++++++++++++- rustfs/src/app/bucket_usecase.rs | 22 +- rustfs/src/on_demand_migration/azure.rs | 451 ++++++- rustfs/src/on_demand_migration/config.rs | 47 +- rustfs/src/on_demand_migration/gcs.rs | 221 +++- .../src/on_demand_migration/list_through.rs | 327 ++++- rustfs/src/on_demand_migration/native_http.rs | 39 +- .../src/on_demand_migration/source_client.rs | 47 +- .../on_demand_migration/test_http_fixture.rs | 10 + 17 files changed, 2370 insertions(+), 120 deletions(-) create mode 100644 crates/madmin/fixtures/on_demand_migration/source_config_e2a.rs create mode 100644 rustfs/fixtures/on_demand_migration/source_config_e2a.rs diff --git a/crates/madmin/fixtures/on_demand_migration/get_response.json b/crates/madmin/fixtures/on_demand_migration/get_response.json index 4101403c3..aff3a808b 100644 --- a/crates/madmin/fixtures/on_demand_migration/get_response.json +++ b/crates/madmin/fixtures/on_demand_migration/get_response.json @@ -1 +1 @@ -{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"} +{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"} diff --git a/crates/madmin/fixtures/on_demand_migration/set_request.json b/crates/madmin/fixtures/on_demand_migration/set_request.json index a67d359da..e5b7fb03a 100644 --- a/crates/madmin/fixtures/on_demand_migration/set_request.json +++ b/crates/madmin/fixtures/on_demand_migration/set_request.json @@ -1 +1 @@ -{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}} +{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}} diff --git a/crates/madmin/fixtures/on_demand_migration/set_response.json b/crates/madmin/fixtures/on_demand_migration/set_response.json index 2ea089aa3..81bf69669 100644 --- a/crates/madmin/fixtures/on_demand_migration/set_response.json +++ b/crates/madmin/fixtures/on_demand_migration/set_response.json @@ -1 +1 @@ -{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}} +{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}} diff --git a/crates/madmin/fixtures/on_demand_migration/source_config_e2a.rs b/crates/madmin/fixtures/on_demand_migration/source_config_e2a.rs new file mode 100644 index 000000000..75331d48c --- /dev/null +++ b/crates/madmin/fixtures/on_demand_migration/source_config_e2a.rs @@ -0,0 +1,80 @@ +// Strict source reader frozen from e2a921bc1608823c8efec955d7463ab8350a8a01. +// Wire declarations and credential Debug are copied verbatim; runtime methods are omitted. +use serde::{Deserialize, Serialize}; +use std::fmt; + +const REDACTED: &str = "REDACTED"; + +/// The external S3-compatible source bucket. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceConfig { + pub provider: Provider, + /// `http(s)://host[:port]` with no path or query. Optional only for + /// [`Provider::Aws`], where it is derived from `region`. + #[serde(default)] + pub endpoint: Option, + pub region: String, + pub bucket: String, + #[serde(default)] + pub path_style: PathStyle, + /// `None` means anonymous access to a public source bucket. + #[serde(default)] + pub credentials: Option, + #[serde(default)] + pub tls: TlsConfig, +} + +/// Source vendor family. `azure` is deliberately absent from this version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Provider { + /// Generic S3-compatible endpoint. + S3, + Aws, + Minio, + Rustfs, + R2, + /// GCS XML interoperability API with HMAC keys. + Gcs, +} + +/// Bucket addressing style. `auto` is resolved by the source client builder. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum PathStyle { + #[default] + Auto, + Path, + Virtual, +} + +/// Static credentials for the source. `Debug` never prints the secret or +/// the session token. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceCredentials { + pub access_key: String, + pub secret_key: String, + #[serde(default)] + pub session_token: Option, +} + +impl fmt::Debug for SourceCredentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SourceCredentials") + .field("access_key", &self.access_key) + .field("secret_key", &REDACTED) + .field("session_token", &self.session_token.as_ref().map(|_| REDACTED)) + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TlsConfig { + #[serde(default)] + pub skip_verify: bool, + #[serde(default)] + pub ca_cert_pem: Option, +} diff --git a/crates/madmin/src/on_demand_migration.rs b/crates/madmin/src/on_demand_migration.rs index e92899717..989d4a991 100644 --- a/crates/madmin/src/on_demand_migration.rs +++ b/crates/madmin/src/on_demand_migration.rs @@ -85,10 +85,10 @@ pub struct OnDemandMigrationSource { #[serde(default)] pub tls: OnDemandMigrationTls, /// Required for `azure` and rejected for every other provider. - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub azure: Option, /// Required for `gcs_native` and rejected for every other provider. - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub gcs: Option, } @@ -651,6 +651,10 @@ mod tests { use super::*; use crate::test_support::TestServer; + mod before_native_sources { + include!("../fixtures/on_demand_migration/source_config_e2a.rs"); + } + const SET_REQUEST_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/set_request.json"); const SET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/set_response.json"); const GET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/get_response.json"); @@ -684,6 +688,20 @@ mod tests { assert_eq!(config.source.tls, OnDemandMigrationTls::default()); } + #[test] + fn s3_admin_writes_remain_readable_by_the_strict_pre_native_server() { + for provider in ["s3", "aws", "minio", "rustfs", "r2", "gcs"] { + let historical = SET_REQUEST_FIXTURE.replace("\"provider\":\"minio\"", &format!("\"provider\":\"{provider}\"")); + let config: OnDemandMigrationConfig = serde_json::from_str(&historical).expect("historical set request"); + let wire = serde_json::to_string(&config).expect("current admin set request"); + let actual: serde_json::Value = serde_json::from_str(&wire).expect("admin request JSON"); + let old_source: before_native_sources::SourceConfig = serde_json::from_value(actual["source"].clone()) + .expect("the strict e2a server must accept an ordinary S3 source from the new admin client"); + assert_eq!(serde_json::to_value(old_source).expect("old source wire"), actual["source"]); + assert_eq!(wire, historical.trim(), "provider={provider}: preserve the historical request bytes"); + } + } + #[test] fn set_response_fixture_round_trips_and_is_redacted() { let response: OnDemandMigrationSetResponse = round_trip(SET_RESPONSE_FIXTURE); @@ -878,11 +896,11 @@ mod tests { for (label, json) in [ ( "azure", - r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"},"gcs":null}"#, + r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"}}"#, ), ( "gcs_native", - r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#, + r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#, ), ] { let source: OnDemandMigrationSource = serde_json::from_str(json).unwrap_or_else(|err| panic!("{label}: {err}")); @@ -891,6 +909,16 @@ mod tests { json, "{label} must reproduce the server wire shape byte for byte" ); + let mut wire: serde_json::Value = serde_json::from_str(json).expect("native wire fixture"); + assert!( + serde_json::from_value::(wire.clone()).is_err(), + "native provider names and fields still require an upgraded server" + ); + wire[if label == "azure" { "gcs" } else { "azure" }] = serde_json::Value::Null; + assert_eq!( + serde_json::from_value::(wire).expect("the prior explicit-null wire still decodes"), + source + ); } let azure = OnDemandMigrationAzure { @@ -945,6 +973,10 @@ mod tests { .is_some_and(|auth| auth.starts_with("AWS4-HMAC-SHA256")) ); assert_eq!(request.body, SET_REQUEST_FIXTURE.trim(), "the body is the canonical config document"); + let body: serde_json::Value = serde_json::from_str(&request.body).expect("signed admin request JSON"); + let old_source: before_native_sources::SourceConfig = serde_json::from_value(body["source"].clone()) + .expect("the strict pre-native server must accept the actual signed PUT source"); + assert_eq!(old_source.provider, before_native_sources::Provider::Minio); } #[tokio::test] diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 52c50e042..0720cc8b3 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -11,6 +11,7 @@ ## Open Items +- `odm-list-bare-envelope` historical ODM continuation tokens: preserve complete bare v1/v2 envelopes. Framed issuance defaults on for the deployed framed-only generation; upgrades from older bare-only readers must explicitly disable it before starting new nodes and keep it off until reader convergence. Remove the legacy classifier and framing issuance override only after every supported reader accepts framing and outstanding bare listings have drained or clients explicitly restarted them; tokens have no automatic expiry. Exact full-envelope object keys remain intrinsically ambiguous during this compatibility period. - `backlog-2263` legacy heal MRF inspection: retained per-record journals remain readable while committed-snapshot ownership and writer activation are staged. Remove legacy import only after all supported direct-upgrade and rollback readers understand committed snapshots and migration tooling confirms that no retained or restorable legacy journal requires it. This does not enable a new writer or change the automatic legacy consumer. - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation. - `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains. diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md index 9717559f3..06d066fd4 100644 --- a/docs/operations/on-demand-migration.md +++ b/docs/operations/on-demand-migration.md @@ -23,13 +23,25 @@ Both builds can read, redact and preserve GCS configuration. A build without `gc ## List continuation token rollout -`RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape. +The two rollout switches have different defaults. Unset or invalid boolean values use the stated default. Both are node environment variables, not bucket settings: -Leave the switch off while deploying v2 reader support to every node that can receive a continuation request, including nodes behind other load-balancer routes. Then set it to `true` in each node's environment and restart those nodes to enable issuance. A v1-only binary rejects v2 with `400 InvalidArgument` before the source-error policy runs; neither `not_found` nor turning off list-through makes that old reader compatible. With issuance still off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. The default rollout does not claim to fix that chain until issuance is enabled. +- `RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false` and allows a v1 listing to first issue a v2 token after an empty truncated merged page. Existing v2 tokens keep their budget even on reader-only nodes. Consuming an object/common prefix or reaching a new EOF resets the budget to v1 without changing the chain's framing. +- `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS` defaults to `true`, preserving the framed output of the #7187 / `e1608fbd9` generation. It allows a bare/new merged listing to first issue a NUL-prefixed JSON envelope inside the existing base64 encoding. Existing framed chains stay framed even with this switch off, including a reset to v1 and local continuation after list-through is disabled. With framing issuance off, new bare v1 output keeps its historical bytes; ordinary local listings remain unchanged. This switch does not enable the v2 budget. -An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64 JSON, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries. +Choose the upgrade path from the binaries currently serving LIST requests, including every load-balancer route. This build reads complete historical bare envelopes and framed v1/v2 envelopes with the same strict version/count validation. There is no single writer format understood by both bare-only and framed-only readers. A bare-v1-only binary rejects bare v2 with `400 InvalidArgument`; a bare-only reader mistakes framed input for a local marker, while a framed-only reader mistakes bare input for one. These framing mismatches can restart a merged scan and lose its budget without returning an error. For example, with local keys `b,d` and source keys `a,c`, a new node issuing bare after `a` followed by an `e1608fbd9` reader can return `a` again. That old reader then emits framing, so the symptom need not be an infinite loop. -For rollback, first turn issuance off on every node. Keep v2-capable readers available for outstanding v2 chains: switching issuance off does not erase their budgets, and tokens have no expiration that proves those chains have drained. Route those continuations to compatible readers or have clients explicitly restart their listings before restoring v1-only binaries. Restarting a listing is a new scan and can repeat entries. Do not roll back readers while assuming the issuance switch makes existing v2 tokens disappear. +- **Upgrading from the framed-only #7187 / `e1608fbd9` generation:** before deploying the first new node, set `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS=true` in the new nodes' deployment environment, or leave it unset to use this build's `true` default. Remove any previous explicit `false` override. Existing framed-only nodes ignore this new variable and already emit framing. Keep framing enabled while any of those nodes serves continuations. While list-through remains active, new and existing framed v1/v2 chains then retain their cursors and any active budget in both directions. Only after all serving nodes are dual readers may you choose `false`; existing framed chains still stay framed, while newly started bare chains must stay on dual readers. +- **Upgrading from older, pre-#7187 bare-only binaries:** explicitly set `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS=false` on every new node **before its first start**. Keep it false until all serving readers accept both formats; this preserves bare v1 bytes and active bare cursors during that rollout. Keep `RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS=false` until every reader also supports v2. After reader convergence, you may enable framing and then the v2 budget; enabling framing also frames the next nonzero merged continuation of an existing bare chain, so do not do this while bare-only readers remain. + +Restart nodes after changing their environment. Do not directly mix bare-only and framed-only binaries on the same continuation routes. Existing bare tokens must be routed to dual readers while any framed-only nodes remain. The v2 issuance switch is independent: keep it off until all readers support v2, but turning it off never removes an existing v2 budget. + +Partial JSON-shaped object keys remain local markers. To retain already issued cursors, a bare JSON object with the ODM tag and every historical writer field (`v`, `local`, `local_done`, `source`, `source_done`, `last_key`) is treated as an envelope, then strictly validated. A valid object key can be identical to that complete envelope: the two byte strings are indistinguishable, so legacy compatibility necessarily gives the envelope interpretation precedence. Framing identifies new merged tokens unambiguously, but dual-format readers do not eliminate this old full-envelope key collision. There is no signature, session store, or automatic format negotiation. + +An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. A zero-sized request does not spend an existing budget. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64-encoded JSON, optionally framed, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries. + +With budget issuance off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. Default rollout does not fix that chain until the v2 switch is enabled. Framing alone does not impose the budget. + +For rollback, turn v2 issuance off, but choose framing for the readers being restored. When returning to the framed-only generation, keep framing `true` and route any outstanding bare tokens only to dual readers. Before restoring bare-only readers, set framing `false` on the remaining dual readers and deal with all outstanding framed tokens; v1-only readers also cannot resume v2 tokens. Neither switch rewrites existing framed or v2 chains, and tokens have no expiration that proves they have drained. Retain compatible readers for those continuations or have clients explicitly restart their listings before restoring incompatible binaries. Restarting a listing is a new scan and can repeat entries. Switching issuance off alone does not make outstanding tokens safe for older readers. ## Positioning @@ -188,9 +200,9 @@ No write, delete, ACL or versioning permission is required or used. Scope the po Behaviour a client can observe. The "Test" column names the case that pins it: `*_test.rs` files live under `crates/e2e_test/src/on_demand_migration/`, and the unit tests live next to the code in `rustfs/src/app/object/get.rs`, `head.rs` and `shared.rs`. -ODM merged continuation tokens use a NUL-prefixed JSON envelope inside the existing base64 encoding. NUL is not valid in a local object key, so a legitimate JSON-shaped key can never be mistaken for a merged cursor. Upgrade every node before using list-through, and restart any in-progress ODM listing issued by an older build: its unframed JSON tokens cannot be distinguished from legitimate local keys. Ordinary local listing tokens remain unchanged. Tokens issued by this build can still resume the local side after list-through is disabled. +ODM merged continuation tokens use bare or NUL-prefixed JSON inside the existing base64 encoding. The default writer preserves framed output; an explicit `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS=false` keeps historical bare output during older-reader rollouts. Compatible readers accept both formats and retain existing budgets. See [List continuation token rollout](#list-continuation-token-rollout) for the independent issuance switches, rolling-upgrade requirements, and the unavoidable ambiguity between a complete historical envelope and an identically named local key. -Source `HEAD` responses with status 404 require a successful bucket probe before being negative-cached. The source credential therefore needs permission for `HeadBucket` (S3 `ListBucket`); a prefix-restricted ListBucket policy can deny that probe, in which case the response is a source failure rather than a cached miss. A missing/inaccessible source bucket, a missing source version, or an ambiguous GET 404 is not proof that the requested key is absent. Conditional GET validators are checked against the actual source GET metadata as well as the advisory HEAD; a missing required validator fails with 424. Source LIST entries without a key or a non-negative size fail the page rather than fabricating an empty object. +Source `HEAD` responses with status 404 require a successful bucket probe before being negative-cached. The source credential therefore needs permission for `HeadBucket` (S3 `ListBucket`); a prefix-restricted ListBucket policy can deny that probe, in which case the response is a source failure rather than a cached miss. A missing/inaccessible source bucket or a missing source version is not proof that the requested key is absent. Native GCS verifies the bucket after either HEAD or GET returns 404 and preserves a failed probe as a source error. Azure accepts explicit `BlobNotFound` only on an unversioned object read with status 404; an ambiguous HEAD may make one container probe, while an ambiguous GET remains a source error. Native probes add at most one request and retain the existing per-request timeouts, rather than a single deadline for the pair. Conditional GET validators are checked against the actual source GET metadata as well as the advisory HEAD; a missing required validator fails with 424. Source LIST entries without a key or a non-negative size fail the page rather than fabricating an empty object. Write-back currently requires namespace locking enabled and exactly one pool with one erasure set. Other topologies fail write-back explicitly as `unsupported`: source reads remain available, but backfill cannot complete successfully or certify cutover. This restriction avoids relying on a set-local condition across distinct pool or lock domains; it does not restrict ordinary S3 writes. Full cross-pool migration requires a globally fenced commit protocol. diff --git a/rustfs/fixtures/on_demand_migration/source_config_e2a.rs b/rustfs/fixtures/on_demand_migration/source_config_e2a.rs new file mode 100644 index 000000000..75331d48c --- /dev/null +++ b/rustfs/fixtures/on_demand_migration/source_config_e2a.rs @@ -0,0 +1,80 @@ +// Strict source reader frozen from e2a921bc1608823c8efec955d7463ab8350a8a01. +// Wire declarations and credential Debug are copied verbatim; runtime methods are omitted. +use serde::{Deserialize, Serialize}; +use std::fmt; + +const REDACTED: &str = "REDACTED"; + +/// The external S3-compatible source bucket. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceConfig { + pub provider: Provider, + /// `http(s)://host[:port]` with no path or query. Optional only for + /// [`Provider::Aws`], where it is derived from `region`. + #[serde(default)] + pub endpoint: Option, + pub region: String, + pub bucket: String, + #[serde(default)] + pub path_style: PathStyle, + /// `None` means anonymous access to a public source bucket. + #[serde(default)] + pub credentials: Option, + #[serde(default)] + pub tls: TlsConfig, +} + +/// Source vendor family. `azure` is deliberately absent from this version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Provider { + /// Generic S3-compatible endpoint. + S3, + Aws, + Minio, + Rustfs, + R2, + /// GCS XML interoperability API with HMAC keys. + Gcs, +} + +/// Bucket addressing style. `auto` is resolved by the source client builder. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum PathStyle { + #[default] + Auto, + Path, + Virtual, +} + +/// Static credentials for the source. `Debug` never prints the secret or +/// the session token. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceCredentials { + pub access_key: String, + pub secret_key: String, + #[serde(default)] + pub session_token: Option, +} + +impl fmt::Debug for SourceCredentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SourceCredentials") + .field("access_key", &self.access_key) + .field("secret_key", &REDACTED) + .field("session_token", &self.session_token.as_ref().map(|_| REDACTED)) + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TlsConfig { + #[serde(default)] + pub skip_verify: bool, + #[serde(default)] + pub ca_cert_pem: Option, +} diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs index d4004c5bc..b0f276e32 100644 --- a/rustfs/src/app/bucket_list_through.rs +++ b/rustfs/src/app/bucket_list_through.rs @@ -33,9 +33,10 @@ use super::storage_api::bucket_usecase::s3_api::bucket::ListObjectsV2Params; use crate::app::object::shared::{odm_source_unavailable_error, odm_state_error_class}; use crate::error::ApiError; use crate::on_demand_migration::{ - BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, - MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan, - SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan, + BucketOdmState, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, + ListThroughToken, ListThroughTokenError, MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, + SourceError, SourceErrorPolicy, SourceListPlan, SourceListRequest, SourceObject, SourcePage, decode_continuation_token, + source_list_plan, }; use futures::StreamExt; use rustfs_utils::http::{SUFFIX_SOURCE_PROXY_REQUEST, get_header}; @@ -53,6 +54,9 @@ const SOURCE_STORAGE_CLASS: &str = "STANDARD"; /// Enable only after every node serving continuation requests can read v2. const ENV_LIST_PROGRESS_TOKENS: &str = "RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS"; +/// Independent of the budget version: bare-v2 readers cannot read framing. +const ENV_LIST_FRAMED_TOKENS: &str = "RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS"; + /// Concurrent local metadata probes when a versioned bucket has to check /// source-only keys for a shadowing delete marker. const DELETE_MARKER_PROBE_CONCURRENCY: usize = 32; @@ -89,6 +93,37 @@ pub(crate) fn local_cursor(decoded: Option<&str>, merged: Option<&ListThroughTok } } +/// A framed chain keeps its envelope when the bucket stops consulting source. +pub(crate) fn preserve_framed_local_cursor(info: &mut ListObjectsV2Info, previous: Option<&ListThroughToken>) { + let Some(previous) = previous.filter(|token| token.framed) else { + return; + }; + let Some(next) = info.next_continuation_token.take() else { + return; + }; + let mut token = previous.clone(); + token.local = Some(next); + token.local_done = false; + if let Some(last_key) = info + .objects + .iter() + .map(|object| object.name.as_str()) + .chain(info.prefixes.iter().map(String::as_str)) + .max() + { + token.last_key = Some( + token + .last_key + .as_deref() + .map_or(last_key, |previous| previous.max(last_key)) + .to_string(), + ); + token.v = LIST_THROUGH_TOKEN_VERSION; + token.no_progress = None; + } + info.next_continuation_token = Some(token.encode()); +} + fn invalid_continuation_token(err: &ListThroughTokenError) -> S3Error { debug!(error = %err, "rejected an on-demand migration list continuation token"); S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid continuation token".to_string()) @@ -289,6 +324,7 @@ pub(crate) async fn merged_list_objects_v2( } let issue_progress_tokens = rustfs_utils::get_env_bool(ENV_LIST_PROGRESS_TOKENS, false); + let framed = token.is_some_and(|token| token.framed) || rustfs_utils::get_env_bool(ENV_LIST_FRAMED_TOKENS, true); let outcome = match merger.finish(issue_progress_tokens) { Ok(outcome) => outcome, Err(ListPageError::NoProgress(MergeSide::Source)) => { @@ -326,7 +362,10 @@ pub(crate) async fn merged_list_objects_v2( info: ListObjectsV2Info { is_truncated: outcome.is_truncated, continuation_token: None, - next_continuation_token: outcome.next_token.map(|token| token.encode()), + next_continuation_token: outcome.next_token.map(|mut token| { + token.framed = framed; + token.encode() + }), objects, prefixes, }, @@ -481,6 +520,7 @@ mod tests { fn token(local: Option<&str>, local_done: bool) -> ListThroughToken { ListThroughToken { + framed: false, t: "odm-list".to_string(), v: 1, local: local.map(str::to_string), @@ -558,20 +598,23 @@ mod tests { #[test] fn a_v2_token_keeps_the_local_cursor_when_list_through_is_turned_off() { - let mut resume = token(Some("local-2"), false); - resume.v = 2; - resume.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 1); - let encoded = resume.encode(); - let decoded = decode_list_cursor(Some(&encoded)).expect("a v2 envelope decodes"); - assert_eq!(decoded.as_ref(), Some(&resume)); - assert!(matches!( - local_cursor(Some(&encoded), decoded.as_ref()), - LocalListCursor::Token(Some(local)) if local == "local-2" - )); - resume.local_done = true; - let encoded = resume.encode(); - let decoded = decode_list_cursor(Some(&encoded)).expect("v2 with local EOF decodes"); - assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted)); + for framed in [false, true] { + let mut resume = token(Some("local-2"), false); + resume.framed = framed; + resume.v = 2; + resume.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 1); + let encoded = resume.encode(); + let decoded = decode_list_cursor(Some(&encoded)).expect("a v2 envelope decodes"); + assert_eq!(decoded.as_ref(), Some(&resume)); + assert!(matches!( + local_cursor(Some(&encoded), decoded.as_ref()), + LocalListCursor::Token(Some(local)) if local == "local-2" + )); + resume.local_done = true; + let encoded = resume.encode(); + let decoded = decode_list_cursor(Some(&encoded)).expect("v2 with local EOF decodes"); + assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted)); + } } #[test] @@ -623,6 +666,17 @@ mod tests { String, tokio_util::task::AbortOnDropHandle>, tokio_util::sync::CancellationToken, + ) { + list_source_with_response(pages, |_, body| body).await + } + + async fn list_source_with_response( + pages: impl Iterator + Send + 'static, + mut response_body: impl FnMut(&str, String) -> String + Send + 'static, + ) -> ( + String, + tokio_util::task::AbortOnDropHandle>, + tokio_util::sync::CancellationToken, ) { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await @@ -656,6 +710,7 @@ mod tests { "expected a path-style bucket-root LIST request, got {first_line:?}" ); assert!(first_line.contains("list-type=2"), "expected a ListObjectsV2 query, got {first_line:?}"); + let body = response_body(&first_line, body); requests.push(first_line); let response = format!( "HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", @@ -762,6 +817,7 @@ mod tests { ); let continuation_token = resume_source.map(|source| { let token = ListThroughToken { + framed: false, t: "odm-list".into(), v: 1, local: None, @@ -809,6 +865,286 @@ mod tests { .expect("listing must complete within its bounded source budget") } + async fn native_list_source( + provider: Provider, + pages: Vec<(String, String)>, + ) -> ( + String, + tokio_util::task::AbortOnDropHandle>, + tokio_util::sync::CancellationToken, + ) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind native listing source"); + let address = listener.local_addr().expect("native source address"); + let stop = tokio_util::sync::CancellationToken::new(); + let server_stop = stop.clone(); + let server = tokio::spawn(async move { + let mut pages = pages.into_iter(); + let mut requests = Vec::new(); + loop { + let (mut stream, _) = tokio::select! { + _ = server_stop.cancelled() => break, + accepted = listener.accept() => accepted.expect("accept native source request"), + }; + let (target, body) = pages.next().expect("native source must not receive an extra request"); + let mut request = Vec::new(); + let mut chunk = [0; 4096]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = stream.read(&mut chunk).await.expect("read native source request"); + assert!(count > 0, "native request needs complete headers"); + request.extend_from_slice(&chunk[..count]); + assert!(request.len() <= 32 * 1024, "native request headers must be bounded"); + } + let text = String::from_utf8(request).expect("native HTTP request text"); + let first_line = text.lines().next().expect("native request line"); + assert_eq!(first_line, format!("GET {target} HTTP/1.1")); + let authorization = text + .lines() + .filter_map(|line| line.split_once(':')) + .find(|(name, _)| name.eq_ignore_ascii_case("authorization")) + .map(|(_, value)| value.trim()) + .expect("native credential must be used"); + assert!(authorization.starts_with(if provider == Provider::Azure { + "SharedKey acct:" + } else { + "Bearer " + })); + requests.push(first_line.to_string()); + let response = format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()); + stream.write_all(response.as_bytes()).await.expect("write native source page"); + stream.shutdown().await.expect("finish native source response"); + } + assert!(pages.next().is_none(), "every scripted native page must have been requested"); + requests + }); + (format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server), stop) + } + + fn native_list_target(provider: Provider, cursor: bool, max_keys: i32) -> String { + if provider == Provider::Azure { + format!( + "/source-bucket?restype=container&comp=list{}&maxresults={max_keys}", + if cursor { "&marker=opaque%2B%2F%3D" } else { "" } + ) + } else { + format!( + "/storage/v1/b/source-bucket/o?{}maxResults={max_keys}", + if cursor { "pageToken=opaque%2B%2F%3D&" } else { "" } + ) + } + } + + fn native_list_body(provider: Provider, entries: &str, prefixes: bool, next: bool) -> String { + if provider == Provider::Azure { + format!( + "{entries}{}{}", + if prefixes { + "目录/子/" + } else { + "" + }, + if next { "opaque+/=" } else { "" } + ) + } else { + format!( + r#"{{"items":[{entries}],"prefixes":{},"nextPageToken":{}}}"#, + if prefixes { r#"["目录/子/"]"# } else { "[]" }, + if next { r#""opaque+/=""# } else { "null" } + ) + } + } + + #[cfg(feature = "gcs")] + fn native_test_service_account() -> String { + // The real Google credentials implementation signs locally. Generate a + // disposable key instead of storing private key material in the fixture. + let key = rcgen::KeyPair::generate_for(&rcgen::PKCS_RSA_SHA256).expect("generate fixture service-account key"); + serde_json::json!({ + "type": "service_account", + "client_email": "fixture@example.invalid", + "private_key_id": "fixture-key", + "private_key": key.serialize_pem(), + "project_id": "fixture-project" + }) + .to_string() + } + + async fn native_source_policy_request( + provider: Provider, + policy: SourceErrorPolicy, + pages: Vec<(String, String)>, + service_account: &str, + max_keys: i32, + ) -> (S3Result>, Vec) { + let (endpoint, server, stop) = native_list_source(provider, pages).await; + let (_state_guard, mut input) = source_policy_input(endpoint.clone(), policy, None, None).await; + input.max_keys = Some(max_keys); + let sys = OnDemandMigrationSys::get(); + let installed = sys.state(&input.bucket).expect("installed source state"); + let mut config = installed.config().clone(); + config.source = serde_json::from_value(serde_json::json!({ + "provider": provider, + "endpoint": endpoint, + "region": "us-east-1", + "bucket": "source-bucket", + "azure": if provider == Provider::Azure { serde_json::json!({ "account": "acct", "account_key": "c2VjcmV0LWtleQ==" }) } else { serde_json::Value::Null }, + "gcs": if provider == Provider::GcsNative { serde_json::json!({ "service_account_json": service_account }) } else { serde_json::Value::Null } + })).expect("native source configuration"); + sys.apply_for_incarnation(&input.bucket, installed.incarnation_id(), Some(&config)) + .await; + let state = sys.state(&input.bucket).expect("native source state"); + state + .client() + .unwrap_or_else(|error| panic!("{provider:?} native client must build: {error:?}")); + let result = execute_source_list(input).await; + stop.cancel(); + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("native source server must finish") + .expect("native source requests must match the script"); + (result, requests) + } + + #[test] + #[serial_test::serial] + fn native_list_through_malformed_fields_follow_both_source_policies() { + run_large_stack_test("native-list-through-fields", || async { + temp_env::async_with_vars( + [("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), ("HTTP_PROXY", None), ("HTTPS_PROXY", None), + ("ALL_PROXY", None), ("http_proxy", None), ("https_proxy", None), ("all_proxy", None), + ("NO_PROXY", Some("*")), ("no_proxy", Some("*"))], + async { + #[cfg(feature = "gcs")] + let service_account = native_test_service_account(); + #[cfg(not(feature = "gcs"))] + let service_account = String::new(); + for provider in [Provider::Azure, #[cfg(feature = "gcs")] Provider::GcsNative] { + let invalid = if provider == Provider::Azure { + ["bad", + "bad-1", + "bad18446744073709551616", + "1"] + } else { + [r#"{"name":"bad"}"#, r#"{"name":"bad","size":"-1"}"#, + r#"{"name":"bad","size":"18446744073709551616"}"#, r#"{"size":"1"}"#] + }; + let valid = if provider == Provider::Azure { + "a-source1" + } else { r#"{"name":"a-source","size":"1"}"# }; + for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] { + for entry in invalid { + for refill in [false, true] { + let mut pages = Vec::new(); + if refill { + pages.push((native_list_target(provider, false, 2), native_list_body(provider, valid, false, true))); + } + let entries = if refill { entry.to_string() } else if provider == Provider::Azure { + format!("{valid}{entry}") + } else { format!("{valid},{entry}") }; + pages.push((native_list_target(provider, refill, 2), native_list_body(provider, &entries, false, false))); + let (result, requests) = native_source_policy_request(provider, policy, pages, &service_account, 2).await; + assert_eq!(requests.len(), if refill { 2 } else { 1 }, "{provider:?} {policy:?} {entry}"); + if policy == SourceErrorPolicy::Propagate { + let err = result.expect_err("malformed native page must propagate"); + assert_eq!(err.status_code(), Some(http::StatusCode::FAILED_DEPENDENCY)); + assert_eq!(err.code(), &S3ErrorCode::Custom("SourceUnavailable".into())); + assert_eq!(err.message(), Some("other")); + } else { + // Reuse the complete local-only assertions, including no + // leaked source objects, no cursor and the degraded header. + assert_source_policy_result(result, policy); + } + } + } + } + } + }, + ).await; + }); + } + + #[test] + #[serial_test::serial] + fn native_list_through_preserves_valid_empty_pages_and_zero_size_objects() { + run_large_stack_test("native-list-through-valid", || async { + temp_env::async_with_vars( + [ + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + #[cfg(feature = "gcs")] + let service_account = native_test_service_account(); + #[cfg(not(feature = "gcs"))] + let service_account = String::new(); + for provider in [ + Provider::Azure, + #[cfg(feature = "gcs")] + Provider::GcsNative, + ] { + let valid = if provider == Provider::Azure { + "目录/空0" + } else { + r#"{"name":"目录/空","size":"0"}"# + }; + for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] { + let (result, requests) = native_source_policy_request( + provider, + policy, + vec![ + (native_list_target(provider, false, 3), native_list_body(provider, "", false, true)), + (native_list_target(provider, true, 3), native_list_body(provider, valid, true, false)), + ], + &service_account, + 3, + ) + .await; + assert_eq!(requests.len(), 2); + let response = result.expect("valid native listing must succeed under either policy"); + assert_ne!( + response + .headers + .get("x-rustfs-on-demand-migration-list") + .and_then(|value| value.to_str().ok()), + Some("local_only") + ); + let output = response.output; + let objects = output.contents.expect("local and source objects"); + assert_eq!( + objects + .iter() + .map(|object| (object.key.as_deref(), object.size)) + .collect::>(), + vec![(Some("z-local"), Some(1)), (Some("目录/空"), Some(0))] + ); + assert_eq!( + output + .common_prefixes + .expect("native prefix") + .into_iter() + .map(|prefix| prefix.prefix) + .collect::>(), + vec![Some("目录/子/".to_string())] + ); + assert_eq!(output.key_count, Some(3)); + assert_eq!(output.is_truncated, Some(false)); + assert!(output.next_continuation_token.is_none()); + } + } + }, + ) + .await; + }); + } + async fn source_policy_request( pages: Vec, policy: SourceErrorPolicy, @@ -1162,6 +1498,7 @@ mod tests { temp_env::async_with_vars( [ (ENV_LIST_PROGRESS_TOKENS, Some("true")), + (ENV_LIST_FRAMED_TOKENS, Some("false")), ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), ("HTTP_PROXY", None), ("HTTPS_PROXY", None), @@ -1205,6 +1542,7 @@ mod tests { seen.insert(next.clone()), "a cross-request source cursor cycle must not return an identical empty merged token" ); + assert!(!decode_wire_token(&next).framed, "the budget switch cannot enable framing"); empty_pages += 1; input.continuation_token = Some(next); } @@ -1241,6 +1579,7 @@ mod tests { temp_env::async_with_vars( [ (ENV_LIST_PROGRESS_TOKENS, None), + (ENV_LIST_FRAMED_TOKENS, Some("false")), ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), ("HTTP_PROXY", None), ("HTTPS_PROXY", None), @@ -1252,7 +1591,10 @@ mod tests { ("no_proxy", Some("*")), ], async { - for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] { + for (policy, framed) in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] + .into_iter() + .flat_map(|policy| [false, true].map(|framed| (policy, framed))) + { let pages = ["B", "C", "A"].map(|next| source_xml(Some(next), true, None)); let (endpoint, server, stop) = list_source(pages.into_iter().cycle()).await; let (_state_guard, mut input) = source_policy_input(endpoint, policy, Some("A"), None).await; @@ -1266,6 +1608,7 @@ mod tests { let raw = base64_simd::STANDARD.decode_to_vec(&next).expect("base64 continuation token"); let decoded = std::str::from_utf8(&raw).expect("JSON token"); let token = decode_list_cursor(Some(decoded)).expect("v1 reader").expect("merged token"); + assert!(!token.framed, "explicit false must keep issuing bare tokens for older readers"); assert_eq!(token.v, 1, "the default rollout cannot begin issuing v2"); assert_eq!(token.no_progress, None); assert!(!decoded.contains("no_progress"), "ordinary v1 wire shape stays unchanged"); @@ -1279,6 +1622,7 @@ mod tests { let mut token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token"))) .expect("v1 reader") .expect("merged token"); + token.framed = framed; token.v = 2; token.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 2); input.continuation_token = Some(base64_simd::STANDARD.encode_to_string(token.encode().as_bytes())); @@ -1290,6 +1634,7 @@ mod tests { let token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token"))) .expect("v2 reader") .expect("merged token"); + assert_eq!(token.framed, framed, "reader-only nodes retain the incoming framing"); assert_eq!(token.v, 2); assert_eq!(token.no_progress, Some(MAX_LIST_NO_PROGRESS_PAGES - 1)); input.continuation_token = Some(next); @@ -1318,6 +1663,7 @@ mod tests { temp_env::async_with_vars( [ (ENV_LIST_PROGRESS_TOKENS, Some("true")), + (ENV_LIST_FRAMED_TOKENS, None), ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), ("HTTP_PROXY", None), ("HTTPS_PROXY", None), @@ -1393,6 +1739,717 @@ mod tests { }); } + // The e160 active-ODM route decoded before entering this same merge core. + // Freeze both its reader and writer; use real local storage and HTTP source + // calls here, without claiming that this harness executes an old binary. + async fn execute_e160_source_list(input: ListObjectsV2Input) -> S3Result> { + use crate::app::storage_api::bucket_usecase::s3_api::bucket::{ + build_list_objects_v2_output, parse_list_objects_v2_params, + }; + use crate::on_demand_migration::list_through::e160_framed_reader as old; + + let params = parse_list_objects_v2_params( + input.prefix.clone(), + input.delimiter.clone(), + input.max_keys, + input.continuation_token.clone(), + input.start_after.clone(), + )?; + assert!(params.max_keys > 0, "the frozen route models active, nonzero merged requests"); + let token = params.decoded_continuation_token.as_deref().and_then(|raw| { + match old::decode_continuation_token(raw).expect("valid frozen-reader input") { + old::ListThroughCursor::Local(_) => None, + old::ListThroughCursor::Merged(token) => Some(ListThroughToken { + framed: true, + t: token.t, + v: token.v, + local: token.local, + local_done: token.local_done, + source: token.source, + source_done: token.source_done, + last_key: token.last_key, + no_progress: token.no_progress, + }), + } + }); + let store = shared_gating_ecstore().await; + let state = OnDemandMigrationSys::get() + .state(&input.bucket) + .expect("installed real source state"); + assert!(state.config().policy.list_through); + let mut outcome = tokio::time::timeout( + Duration::from_secs(10), + merged_list_objects_v2( + &store, + &state, + &input.bucket, + ¶ms, + input.fetch_owner.unwrap_or_default(), + false, + token.as_ref(), + ), + ) + .await + .expect("frozen-reader listing must remain bounded")?; + assert!(!outcome.degraded, "the compatibility matrix exercises successful source pages"); + if let Some(raw) = outcome.info.next_continuation_token.as_mut() { + // The shared core supplies semantic fields. Only the frozen e160 + // writer decides the old node's outgoing framing and JSON bytes. + let old_token: old::ListThroughToken = + serde_json::from_str(raw.strip_prefix("\0odm-list:").unwrap_or(raw)).expect("merge output fields"); + *raw = old_token.encode(); + } + Ok(S3Response::new(build_list_objects_v2_output( + outcome.info, + input.fetch_owner.unwrap_or_default(), + params.max_keys, + input.bucket, + params.prefix, + params.delimiter, + input.encoding_type, + params.response_continuation_token, + params.response_start_after, + ))) + } + + fn e160_wire_cursor(wire: &str) -> crate::on_demand_migration::list_through::e160_framed_reader::ListThroughCursor { + let raw = base64_simd::STANDARD.decode_to_vec(wire).expect("wire base64"); + crate::on_demand_migration::list_through::e160_framed_reader::decode_continuation_token( + std::str::from_utf8(&raw).expect("wire UTF-8"), + ) + .expect("frozen e160 decoder") + } + + #[test] + #[serial_test::serial] + fn list_through_e160_framed_rollout_and_reader_convergence_preserve_sequence() { + run_large_stack_test("list-through-e160-framed-rollout", || async { + for initial_framing in [None, Some("true")] { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, None), + (ENV_LIST_FRAMED_TOKENS, initial_framing), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + // Respond to the actual cursor, including a repeated start + // from an incompatible reader, rather than to request order. + let (endpoint, server, _) = + list_source_with_response(std::iter::repeat_n(String::new(), 6), |request, _| { + let uri: http::Uri = request + .split_whitespace() + .nth(1) + .expect("request target") + .parse() + .expect("source LIST URI"); + let cursors: Vec<_> = url::form_urlencoded::parse(uri.query().expect("LIST query").as_bytes()) + .filter_map(|(key, cursor)| (key == "continuation-token").then_some(cursor)) + .collect(); + match cursors.as_slice() { + [] => source_xml(Some("S"), true, Some("a")), + [cursor] if cursor == "S" => source_xml(None, false, Some("c")), + _ => panic!("unexpected source cursor in {request}"), + } + }) + .await; + let (_guard, mut input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await; + let store = shared_gating_ecstore().await; + for key in ["b", "d"] { + store + .put_object( + &input.bucket, + key, + &mut StoragePutObjReader::from_vec(vec![1]), + &StorageObjectOptions::default(), + ) + .await + .expect("seed interleaved local keys"); + } + input.max_keys = Some(1); + // First new writer -> e160 reader -> new reader. After all old + // readers leave, disabling issuance must retain this live frame. + let mut framed_pages = Vec::new(); + for (page, key) in ["a", "b", "c", "d", "z-local"].into_iter().enumerate() { + let response = if page == 1 { + execute_e160_source_list(input.clone()).await + } else if page >= 3 { + temp_env::async_with_vars( + [(ENV_LIST_FRAMED_TOKENS, Some("false"))], + execute_source_list(input.clone()), + ) + .await + } else { + execute_source_list(input.clone()).await + } + .expect("compatible framed reader must continue the same scan"); + assert_eq!(response.output.key_count, Some(1)); + assert_eq!(response.output.contents.as_ref().expect("one object")[0].key.as_deref(), Some(key)); + assert_eq!(response.output.is_truncated, Some(page != 4)); + input.continuation_token = response.output.next_continuation_token; + if page != 4 { + let wire = input.continuation_token.as_deref().expect("framed continuation"); + framed_pages.push((wire.to_string(), key)); + } else { + assert!(input.continuation_token.is_none()); + } + } + for (wire, key) in &framed_pages { + let wire = wire.as_str(); + let key = *key; + let crate::on_demand_migration::list_through::e160_framed_reader::ListThroughCursor::Merged(old) = + e160_wire_cursor(wire) + else { + panic!("e160 must recognize every active framed v1 token") + }; + assert_eq!(old.v, 1); + assert_eq!(old.last_key.as_deref(), Some(key)); + assert_eq!(old.no_progress, None); + assert!(decode_wire_token(wire).framed); + } + // A fresh bare chain is safe only after every reader is dual. + temp_env::async_with_vars([(ENV_LIST_FRAMED_TOKENS, Some("false"))], async { + for (page, key) in ["a", "b", "c", "d", "z-local"].into_iter().enumerate() { + let response = execute_source_list(input.clone()).await.expect("converged dual readers"); + assert_eq!(response.output.key_count, Some(1)); + assert_eq!(response.output.contents.as_ref().expect("one object")[0].key.as_deref(), Some(key)); + assert_eq!(response.output.is_truncated, Some(page != 4)); + input.continuation_token = response.output.next_continuation_token; + if page != 4 { + let wire = input.continuation_token.as_deref().expect("bare continuation"); + assert!(!decode_wire_token(wire).framed); + assert!( + matches!(e160_wire_cursor(wire), + crate::on_demand_migration::list_through::e160_framed_reader::ListThroughCursor::Local(_)), + "a remaining e160 reader would make this switch unsafe" + ); + } else { + assert!(input.continuation_token.is_none()); + } + } + }) + .await; + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("both finite source scans complete") + .expect("source server"); + assert_eq!(requests.len(), 6, "framing changes add no source fetches"); + for scan in requests.as_chunks::<3>().0.iter() { + assert!(!scan[0].contains("continuation-token=")); + for request in &scan[1..] { + assert!(request.contains("continuation-token=S"), "{request}"); + } + } + }, + ) + .await; + } + }); + } + + #[test] + #[serial_test::serial] + fn list_through_e160_explicit_bare_negative_control_repeats_an_object() { + run_large_stack_test("list-through-e160-bare-negative-control", || async { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, None), + (ENV_LIST_FRAMED_TOKENS, Some("false")), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + let (endpoint, server) = scripted_list_source(vec![ + source_xml(Some("S"), true, Some("a")), + source_xml(Some("S"), true, Some("a")), + source_xml(None, false, Some("c")), + ]) + .await; + let (_guard, mut input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await; + let store = shared_gating_ecstore().await; + for key in ["b", "d"] { + store + .put_object( + &input.bucket, + key, + &mut StoragePutObjReader::from_vec(vec![1]), + &StorageObjectOptions::default(), + ) + .await + .expect("seed interleaved local keys"); + } + input.max_keys = Some(1); + let first = execute_source_list(input.clone()).await.expect("explicit bare first page"); + assert_eq!(first.output.contents.as_ref().expect("first object")[0].key.as_deref(), Some("a")); + input.continuation_token = first.output.next_continuation_token; + let bare = input.continuation_token.as_deref().expect("new bare cursor"); + assert!(!decode_wire_token(bare).framed); + assert!(matches!( + e160_wire_cursor(bare), + crate::on_demand_migration::list_through::e160_framed_reader::ListThroughCursor::Local(_) + )); + let repeated = execute_e160_source_list(input.clone()) + .await + .expect("old reader silently restarts"); + assert_eq!(repeated.output.key_count, Some(1)); + assert_eq!( + repeated.output.contents.as_ref().expect("repeated object")[0].key.as_deref(), + Some("a"), + "negative control: the incompatible bare setting loses last_key and repeats a" + ); + input.continuation_token = repeated.output.next_continuation_token; + assert!(decode_wire_token(input.continuation_token.as_deref().expect("old framed cursor")).framed); + let resumed = execute_source_list(input).await.expect("old writer now supplies framing"); + assert_eq!( + resumed.output.contents.as_ref().expect("next object")[0].key.as_deref(), + Some("b"), + "this mismatch need not loop forever: e160 subsequently emits a frame" + ); + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("finite negative-control source") + .expect("source server"); + assert_eq!(requests.len(), 3); + assert!(!requests[0].contains("continuation-token=")); + assert!(!requests[1].contains("continuation-token="), "the old reader restarted the source scan"); + assert!(requests[2].contains("continuation-token=S")); + }, + ) + .await; + }); + } + + #[test] + #[serial_test::serial] + fn list_through_e160_framed_budget_survives_readers_but_bare_budget_is_lost() { + run_large_stack_test("list-through-e160-budget", || async { + temp_env::async_with_vars([ + (ENV_LIST_PROGRESS_TOKENS, None), (ENV_LIST_FRAMED_TOKENS, Some("false")), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), ("HTTPS_PROXY", None), ("ALL_PROXY", None), + ("http_proxy", None), ("https_proxy", None), ("all_proxy", None), + ("NO_PROXY", Some("*")), ("no_proxy", Some("*")), + ], async { + // Fixed writer bytes, independent of the current token encoder. + let framed13 = "\0odm-list:{\"t\":\"odm-list\",\"v\":2,\"local\":null,\"local_done\":true,\"source\":\"A\",\"source_done\":false,\"last_key\":null,\"no_progress\":13}"; + let (endpoint, server) = scripted_list_source(["B", "C", "D", "E", "F", "G"].into_iter() + .map(|cursor| source_xml(Some(cursor), true, None)).collect()).await; + let (_guard, mut input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await; + input.max_keys = Some(1); + input.continuation_token = Some(base64_simd::STANDARD.encode_to_string(framed13.as_bytes())); + let fourteen = execute_source_list(input.clone()).await.expect("new reader preserves the old framed budget"); + assert_eq!(fourteen.output.key_count, Some(0)); + assert_eq!(fourteen.output.is_truncated, Some(true)); + input.continuation_token = fourteen.output.next_continuation_token; + let crate::on_demand_migration::list_through::e160_framed_reader::ListThroughCursor::Merged(old) = + e160_wire_cursor(input.continuation_token.as_deref().expect("fourteenth empty page")) else { + panic!("the new writer must preserve a framed v2 token for e160"); + }; + assert_eq!(old.v, 2); + assert_eq!(old.no_progress, Some(14)); + assert_eq!(old.source.as_deref(), Some("C")); + assert!(old.local_done); + assert_eq!(old.last_key, None); + let fifteen = execute_e160_source_list(input.clone()).await.expect("e160 preserves an existing v2 budget"); + assert_eq!(fifteen.output.key_count, Some(0)); + assert_eq!(fifteen.output.is_truncated, Some(true)); + input.continuation_token = fifteen.output.next_continuation_token; + let next = decode_wire_token(input.continuation_token.as_deref().expect("fifteenth empty page")); + assert!(next.framed); + assert_eq!(next.v, 2); + assert_eq!(next.no_progress, Some(15)); + assert_eq!(next.source.as_deref(), Some("E")); + assert!(next.local_done); + assert_eq!(next.last_key, None); + assert_source_policy_result(execute_source_list(input).await, SourceErrorPolicy::Propagate); + let requests = tokio::time::timeout(Duration::from_secs(5), server).await + .expect("bounded framed budget source").expect("source server"); + assert_eq!(requests.len(), 6, "each of the three reader hops spends exactly two source fetches"); + for (request, cursor) in requests.iter().zip(["A", "B", "C", "D", "E", "F"]) { + assert!(request.contains(&format!("continuation-token={cursor}")), "{request}"); + } + + let (endpoint, server) = scripted_list_source(vec![ + source_xml(Some("B"), true, None), source_xml(Some("C"), true, None), + ]).await; + let (_guard, mut input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await; + let bare15 = r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":null,"no_progress":15}"#; + input.continuation_token = Some(base64_simd::STANDARD.encode_to_string(bare15.as_bytes())); + assert!(matches!(e160_wire_cursor(input.continuation_token.as_deref().expect("bare v2")), + crate::on_demand_migration::list_through::e160_framed_reader::ListThroughCursor::Local(_))); + let restarted = execute_e160_source_list(input).await.expect("negative control: old reader loses the budget"); + assert_eq!(restarted.output.key_count, Some(0)); + assert_eq!(restarted.output.is_truncated, Some(true)); + let reset = decode_wire_token(restarted.output.next_continuation_token.as_deref().expect("restarted cursor")); + assert!(reset.framed); + assert_eq!(reset.v, 1); + assert_eq!(reset.no_progress, None, "bare v2 was never recognized by the e160 reader"); + let requests = tokio::time::timeout(Duration::from_secs(5), server).await + .expect("bounded bare negative-control source").expect("source server"); + assert_eq!(requests.len(), 2); + assert!(!requests[0].contains("continuation-token=")); + assert!(requests[1].contains("continuation-token=B")); + }).await; + }); + } + + fn decode_wire_token(wire: &str) -> ListThroughToken { + let raw = base64_simd::STANDARD.decode_to_vec(wire).expect("base64 continuation token"); + decode_list_cursor(Some(std::str::from_utf8(&raw).expect("UTF-8 cursor"))) + .expect("valid continuation token") + .expect("merged continuation token") + } + + #[test] + fn framed_local_continuations_preserve_json_markers_and_zero_sized_budgets() { + let json_key = r#"{"t":"odm-list","v":1,"local_done":true}"#; + let mut resume = token(Some("local-2"), false); + resume.framed = true; + resume.v = 2; + resume.no_progress = Some(15); + let mut page = ListObjectsV2Info { + is_truncated: true, + next_continuation_token: Some(json_key.to_string()), + objects: vec![info(json_key)], + ..Default::default() + }; + preserve_framed_local_cursor(&mut page, Some(&resume)); + let raw = page.next_continuation_token.expect("local continuation"); + assert!(raw.starts_with("\0odm-list:")); + let decoded = decode_list_cursor(Some(&raw)) + .expect("framed local continuation") + .expect("envelope"); + assert!(decoded.framed); + assert_eq!( + decoded.local.as_deref(), + Some(json_key), + "the local marker is embedded without another encoding" + ); + assert_eq!(decoded.source, resume.source); + assert_eq!(decoded.last_key.as_deref(), Some(json_key)); + assert_eq!(decoded.v, 1); + assert_eq!(decoded.no_progress, None); + assert!(matches!(local_cursor(Some(&raw), Some(&decoded)), LocalListCursor::Token(Some(local)) if local == json_key)); + + let mut prefix_page = ListObjectsV2Info { + is_truncated: true, + next_continuation_token: Some("photos/".to_string()), + prefixes: vec!["photos/".to_string()], + ..Default::default() + }; + preserve_framed_local_cursor(&mut prefix_page, Some(&resume)); + let prefix = decode_list_cursor(prefix_page.next_continuation_token.as_deref()) + .expect("prefix continuation") + .expect("framed prefix envelope"); + assert!(prefix.framed); + assert_eq!(prefix.last_key.as_deref(), Some("photos/")); + assert_eq!(prefix.v, 1); + assert_eq!(prefix.no_progress, None); + + let mut zero = ListObjectsV2Info { + is_truncated: true, + next_continuation_token: resume.local.clone(), + ..Default::default() + }; + preserve_framed_local_cursor(&mut zero, Some(&resume)); + assert_eq!( + decode_list_cursor(zero.next_continuation_token.as_deref()).expect("zero-sized continuation"), + Some(resume.clone()) + ); + + resume.framed = false; + let mut ordinary = ListObjectsV2Info { + is_truncated: true, + next_continuation_token: Some(json_key.to_string()), + ..Default::default() + }; + preserve_framed_local_cursor(&mut ordinary, Some(&resume)); + assert_eq!(ordinary.next_continuation_token.as_deref(), Some(json_key)); + } + + #[test] + #[serial_test::serial] + fn list_through_historical_v2_budget_exhausts_on_the_next_reader_only_request() { + run_large_stack_test("list-through-historical-budget", || async { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, None), + (ENV_LIST_FRAMED_TOKENS, None), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), ("HTTPS_PROXY", None), ("ALL_PROXY", None), + ("http_proxy", None), ("https_proxy", None), ("all_proxy", None), + ("NO_PROXY", Some("*")), ("no_proxy", Some("*")), + ], + async { + for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] { + let (endpoint, server) = scripted_list_source(vec![ + source_xml(Some("B"), true, None), source_xml(Some("C"), true, None), + ]).await; + let (_state_guard, mut input) = source_policy_input(endpoint, policy, None, None).await; + // Fixed bytes from the pre-framing writer, independent of today's encoder. + input.continuation_token = Some("eyJ0Ijoib2RtLWxpc3QiLCJ2IjoyLCJsb2NhbCI6bnVsbCwibG9jYWxfZG9uZSI6ZmFsc2UsInNvdXJjZSI6IkEiLCJzb3VyY2VfZG9uZSI6ZmFsc2UsImxhc3Rfa2V5IjpudWxsLCJub19wcm9ncmVzcyI6MTV9".to_string()); + assert_source_policy_result(execute_source_list(input).await, policy); + let requests = tokio::time::timeout(Duration::from_secs(5), server).await + .expect("finite source server must finish").expect("source server must not panic"); + assert_eq!(requests.len(), 2, "the old count=15 must terminate without starting a new budget"); + for (request, cursor) in requests.iter().zip(["A", "B"]) { + assert!(request.contains(&format!("continuation-token={cursor}")), "{request}"); + } + } + }, + ).await; + }); + } + + #[test] + #[serial_test::serial] + fn list_through_framing_survives_budget_reset_and_local_only_pagination() { + run_large_stack_test("list-through-framing-local-pagination", || async { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, None), + (ENV_LIST_FRAMED_TOKENS, Some("true")), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + let (endpoint, server) = scripted_list_source(vec![ + source_xml(Some("A"), true, None), + source_xml(Some("B"), true, None), + source_xml(Some("C"), true, None), + source_xml(Some("D"), true, None), + source_xml(None, false, Some("0-source")), + ]) + .await; + let (_state_guard, mut input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await; + let json_key = r#"{"t":"odm-list","v":1,"local_done":true}"#; + let expected_local = ["a-local", "b-local", "z-local", json_key, "~last"]; + let store = shared_gating_ecstore().await; + for key in ["a-local", "b-local", json_key, "~last"] { + store + .put_object( + &input.bucket, + key, + &mut StoragePutObjReader::from_vec(vec![1]), + &StorageObjectOptions::default(), + ) + .await + .expect("seed paginated local keys"); + } + input.max_keys = Some(1); + let first = execute_source_list(input.clone()).await.expect("legitimate empty page"); + assert_eq!(first.output.key_count, Some(0)); + assert_eq!(first.output.is_truncated, Some(true)); + let next = first.output.next_continuation_token.expect("first framed cursor"); + let token = decode_wire_token(&next); + assert!(token.framed, "the independent switch permits first framing issuance"); + assert_eq!(token.v, 1, "framing issuance cannot enable the no-progress budget"); + assert_eq!(token.no_progress, None); + input.continuation_token = Some(next); + let budget_page = + temp_env::async_with_vars([(ENV_LIST_PROGRESS_TOKENS, Some("true"))], execute_source_list(input.clone())) + .await + .expect("another valid empty page starts a budget only when enabled"); + assert_eq!(budget_page.output.key_count, Some(0)); + assert_eq!(budget_page.output.is_truncated, Some(true)); + let next = budget_page.output.next_continuation_token.expect("framed v2 cursor"); + let token = decode_wire_token(&next); + assert!(token.framed); + assert_eq!(token.v, 2); + assert_eq!(token.no_progress, Some(1)); + input.continuation_token = Some(next); + + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, None::<&str>), + (ENV_LIST_FRAMED_TOKENS, Some("false")), + ], + async { + let second = execute_source_list(input.clone()) + .await + .expect("reader-only node reaches source data"); + assert_eq!(second.output.key_count, Some(1)); + assert_eq!(second.output.is_truncated, Some(true)); + assert_eq!(second.output.contents.expect("source object")[0].key.as_deref(), Some("0-source")); + let next = second.output.next_continuation_token.expect("remaining local listing"); + let token = decode_wire_token(&next); + assert!(token.framed, "resetting the budget must not downgrade the framing"); + assert_eq!(token.v, 1); + assert_eq!(token.no_progress, None); + assert!(token.source_done); + input.continuation_token = Some(next); + + OnDemandMigrationSys::get().remove(&input.bucket); + for (index, key) in expected_local.iter().enumerate() { + let page = execute_source_list(input.clone()).await.expect("local-only continuation"); + assert!(!page.headers.contains_key("x-rustfs-on-demand-migration-list")); + assert_eq!(page.output.key_count, Some(1)); + let keys: Vec<_> = page + .output + .contents + .expect("one local object") + .into_iter() + .map(|object| object.key.expect("local key")) + .collect(); + assert_eq!( + keys, + vec![key.to_string()], + "no duplicate or omitted key after disabling list-through" + ); + let truncated = index + 1 < expected_local.len(); + assert_eq!(page.output.is_truncated, Some(truncated)); + if truncated { + let next = page.output.next_continuation_token.expect("local side still has keys"); + let token = decode_wire_token(&next); + assert!(token.framed); + assert_eq!(token.v, 1); + assert_eq!(token.no_progress, None); + assert!(token.local.as_deref().expect("local marker").starts_with(*key)); + assert_eq!(token.last_key.as_deref(), Some(*key)); + input.continuation_token = Some(next); + } else { + assert!(page.output.next_continuation_token.is_none()); + } + } + }, + ) + .await; + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("finite source server must finish") + .expect("source server must not panic"); + assert_eq!( + requests.len(), + 5, + "format changes and local-only continuation perform no additional source I/O" + ); + assert!(!requests[0].contains("continuation-token=")); + for (request, cursor) in requests[1..].iter().zip(["A", "B", "C", "D"]) { + assert!(request.contains(&format!("continuation-token={cursor}")), "{request}"); + } + }, + ) + .await; + }); + } + + #[test] + #[serial_test::serial] + fn list_through_zero_sized_framed_request_preserves_its_budget() { + run_large_stack_test("list-through-framed-zero-size", || async { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, None), (ENV_LIST_FRAMED_TOKENS, None), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), ("HTTPS_PROXY", None), ("ALL_PROXY", None), + ("http_proxy", None), ("https_proxy", None), ("all_proxy", None), + ("NO_PROXY", Some("*")), ("no_proxy", Some("*")), + ], + async { + let (endpoint, server) = scripted_list_source(vec![ + source_xml(Some("B"), true, None), source_xml(Some("C"), true, None), + ]).await; + let (_state_guard, mut input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await; + let wire = concat!("\0odm-list:", r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":null,"no_progress":15}"#); + input.continuation_token = Some(base64_simd::STANDARD.encode_to_string(wire.as_bytes())); + input.max_keys = Some(0); + let zero = execute_source_list(input.clone()).await.expect("zero-sized request does not spend the budget"); + assert_eq!(zero.output.key_count, Some(0)); + assert_eq!(zero.output.is_truncated, Some(true)); + let next = zero.output.next_continuation_token.expect("unconsumed source"); + let token = decode_wire_token(&next); + assert!(token.framed); + assert_eq!(token.v, 2); + assert_eq!(token.no_progress, Some(15)); + assert_eq!(token.source.as_deref(), Some("A")); + assert_eq!(next, input.continuation_token.as_ref().expect("original cursor").as_str()); + let state = OnDemandMigrationSys::get().state(&input.bucket).expect("source state"); + assert_eq!(state.stats().snapshot(state.breaker().state()).source_latency.count, 0, "zero-sized request must not fetch the source"); + input.continuation_token = Some(next); + input.max_keys = Some(2); + assert_source_policy_result(execute_source_list(input).await, SourceErrorPolicy::Propagate); + let requests = tokio::time::timeout(Duration::from_secs(5), server).await + .expect("finite source server must finish").expect("source server must not panic"); + assert_eq!(requests.len(), 2, "only the resumed nonzero request fetches the source"); + assert_eq!(state.stats().snapshot(state.breaker().state()).source_latency.count, 2); + for (request, cursor) in requests.iter().zip(["A", "B"]) { + assert!(request.contains(&format!("continuation-token={cursor}")), "{request}"); + } + }, + ).await; + }); + } + + #[test] + #[serial_test::serial] + fn zero_sized_merged_cursors_preserve_each_side_and_wire_format() { + run_large_stack_test("list-through-zero-side-matrix", || async { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, Some("true")), + (ENV_LIST_FRAMED_TOKENS, Some("true")), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), ("HTTPS_PROXY", None), ("ALL_PROXY", None), + ("http_proxy", None), ("https_proxy", None), ("all_proxy", None), + ("NO_PROXY", Some("*")), ("no_proxy", Some("*")), + ], + async { + for framed in [false, true] { + for (local_done, source_done) in [(false, true), (true, false), (false, false), (true, true)] { + let (endpoint, server, stop) = list_source(std::iter::repeat(source_xml(Some("unexpected"), true, None))).await; + let (_state_guard, mut input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await; + let json = format!(r#"{{"t":"odm-list","v":2,"local":"local-marker","local_done":{local_done},"source":"source-marker","source_done":{source_done},"last_key":null,"no_progress":15}}"#); + let wire = if framed { format!("\0odm-list:{json}") } else { json }; + let original = base64_simd::STANDARD.encode_to_string(wire.as_bytes()); + input.continuation_token = Some(original.clone()); + input.max_keys = Some(0); + let output = execute_source_list(input).await.expect("zero page remains local").output; + let has_more = !local_done || !source_done; + assert_eq!(output.key_count, Some(0)); + assert_eq!(output.is_truncated, Some(has_more), "framed={framed}, local_done={local_done}, source_done={source_done}"); + assert_eq!(output.next_continuation_token.as_deref(), has_more.then_some(original.as_str())); + if let Some(next) = output.next_continuation_token { + let token = decode_wire_token(&next); + assert_eq!(token.framed, framed); + assert_eq!(token.v, 2); + assert_eq!(token.no_progress, Some(15)); + assert_eq!(token.local_done, local_done); + assert_eq!(token.source_done, source_done); + assert_eq!(token.local.as_deref(), Some("local-marker")); + assert_eq!(token.source.as_deref(), Some("source-marker")); + } + stop.cancel(); + let requests = tokio::time::timeout(Duration::from_secs(5), server).await + .expect("unused source must finish").expect("source server must not panic"); + assert!(requests.is_empty(), "zero-sized request must not access the source: {requests:?}"); + } + } + }, + ).await; + }); + } + fn assert_source_policy_result(result: S3Result>, policy: SourceErrorPolicy) { match policy { SourceErrorPolicy::Propagate => { diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index a596a5c70..f06ecf4b9 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -2783,8 +2783,21 @@ impl DefaultBucketUsecase { } else { (None, None) }; - let (object_infos, degraded) = match source_state { - Some(state) => { + let (object_infos, degraded) = match (source_state, merged_token.as_ref()) { + (None, Some(token)) if params.max_keys == 0 => { + // No source was consulted, so retain every unconsumed side and + // the original wire format without spending its progress budget. + let is_truncated = !token.local_done || !token.source_done; + ( + StorageListObjectsV2Info { + is_truncated, + next_continuation_token: params.decoded_continuation_token.clone().filter(|_| is_truncated), + ..Default::default() + }, + false, + ) + } + (Some(state), _) => { let outcome = list_through::merged_list_objects_v2( &store, &state, @@ -2797,12 +2810,12 @@ impl DefaultBucketUsecase { .await?; (outcome.info, outcome.degraded) } - None => { + (None, _) => { let cursor = list_through::local_cursor(params.decoded_continuation_token.as_deref(), merged_token.as_ref()); match cursor { list_through::LocalListCursor::Exhausted => (StorageListObjectsV2Info::default(), false), list_through::LocalListCursor::Token(token) => { - let infos = store + let mut infos = store .list_objects_v2( &bucket, ¶ms.prefix, @@ -2815,6 +2828,7 @@ impl DefaultBucketUsecase { ) .await .map_err(ApiError::from)?; + list_through::preserve_framed_local_cursor(&mut infos, merged_token.as_ref()); (infos, false) } } diff --git a/rustfs/src/on_demand_migration/azure.rs b/rustfs/src/on_demand_migration/azure.rs index ff2e90cb0..f589ad1cf 100644 --- a/rustfs/src/on_demand_migration/azure.rs +++ b/rustfs/src/on_demand_migration/azure.rs @@ -163,6 +163,32 @@ impl AzureSourceBackend { Ok(request) } + /// A missing blob is distinct from a missing container or version. Only + /// object reads may use BlobNotFound as positive evidence of absence. + async fn send_object_request(&self, request: reqwest::Request) -> Result { + let is_head = request.method() == Method::HEAD; + let versioned = request + .url() + .query_pairs() + .any(|(name, _)| name.eq_ignore_ascii_case("versionid") || name.eq_ignore_ascii_case("snapshot")); + let response = self.http.execute(request).await?; + if response.status() == http::StatusCode::NOT_FOUND && !versioned { + match header(response.headers(), HEADER_ERROR_CODE) { + Some("BlobNotFound") => return Err(SourceError::NotFound), + None | Some("ResourceNotFound") if is_head => { + // HEAD may omit an error code. One successful container + // probe proves key absence; a failed probe keeps its error. + // These are two independently timed requests, not one deadline. + drop(response); + self.probe().await?; + return Err(SourceError::NotFound); + } + _ => {} + } + } + NativeHttp::check_response(response, Some(HEADER_ERROR_CODE)) + } + /// Shared mapping for Get Blob and Get Blob Properties. fn head_from_response(headers: &HeaderMap) -> Result { // A customer-provided key means the service holds ciphertext it cannot @@ -189,7 +215,7 @@ impl AzureSourceBackend { impl SourceBackend for AzureSourceBackend { async fn head(&self, key: &str) -> Result { let request = self.request(Method::HEAD, self.blob_url(key)?, HeaderMap::new())?; - let response = self.http.send(request, HEADER_ERROR_CODE).await?; + let response = self.send_object_request(request).await?; Self::head_from_response(response.headers()) } @@ -202,7 +228,7 @@ impl SourceBackend for AzureSourceBackend { ); } let request = self.request(Method::GET, self.blob_url(key)?, headers)?; - let response = self.http.send(request, HEADER_ERROR_CODE).await?; + let response = self.send_object_request(request).await?; let head = Self::head_from_response(response.headers())?; let content_range = header(response.headers(), "content-range").map(str::to_string); Ok(SourceGet { @@ -240,7 +266,7 @@ impl SourceBackend for AzureSourceBackend { } let request = self.request(Method::GET, url, HeaderMap::new())?; - let response = self.http.send(request, HEADER_ERROR_CODE).await?; + let response = self.http.send(request, Some(HEADER_ERROR_CODE)).await?; let body = read_text(response, MAX_XML_BYTES).await?; let listing = parse_list_blobs(&body)?; @@ -256,7 +282,7 @@ impl SourceBackend for AzureSourceBackend { let mut url = self.blob_url(key)?; url.query_pairs_mut().append_pair("comp", "tags"); let request = self.request(Method::GET, url, HeaderMap::new())?; - let response = self.http.send(request, HEADER_ERROR_CODE).await?; + let response = self.http.send(request, Some(HEADER_ERROR_CODE)).await?; let body = read_text(response, MAX_XML_BYTES).await?; parse_blob_tags(&body) } @@ -265,7 +291,7 @@ impl SourceBackend for AzureSourceBackend { let mut url = self.container_url()?; url.query_pairs_mut().append_pair("restype", "container"); let request = self.request(Method::HEAD, url, HeaderMap::new())?; - self.http.send(request, HEADER_ERROR_CODE).await?; + self.http.send(request, Some(HEADER_ERROR_CODE)).await?; Ok(()) } } @@ -343,9 +369,9 @@ struct AzureListing { #[derive(Default)] struct BlobEntry { - name: String, + name: Option, etag: Option, - size: u64, + size: Option, last_modified: Option, access_tier: Option, } @@ -358,6 +384,7 @@ fn parse_list_blobs(xml: &str) -> Result { let mut next_marker = None; let mut blob: Option = None; let mut in_blob_prefix = false; + let mut blob_prefix: Option = None; // Open container elements. quick-xml reports a truncated document as a // plain end of input, so a non-zero depth at EOF is the only signal that // the page was cut short and must not be read as a complete listing. @@ -367,6 +394,9 @@ fn parse_list_blobs(xml: &str) -> Result { match reader.read_event() { Ok(Event::Start(start)) => { let name = local_name(start.name().as_ref()); + if matches!(name.as_str(), "blob" | "blobprefix") && (blob.is_some() || in_blob_prefix) { + return Err(SourceError::Other("source listing entries must not be nested".to_string())); + } match name.as_str() { "blob" => { depth += 1; @@ -385,27 +415,35 @@ fn parse_list_blobs(xml: &str) -> Result { } else { text }; - apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix); + apply_list_field(&name, text, &mut blob, &mut blob_prefix, &mut next_marker, in_blob_prefix)?; } } } Ok(Event::Empty(empty)) => { let name = local_name(empty.name().as_ref()); + if matches!(name.as_str(), "blob" | "blobprefix") { + return Err(SourceError::Other("source listing entry has no name".to_string())); + } let text = if name == "name" { decode_list_name(&empty, String::new())? } else { String::new() }; - apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix); + apply_list_field(&name, text, &mut blob, &mut blob_prefix, &mut next_marker, in_blob_prefix)?; } Ok(Event::End(end)) => match local_name(end.name().as_ref()).as_str() { "blob" => { depth = depth.saturating_sub(1); if let Some(entry) = blob.take() { objects.push(SourceObject { - key: entry.name, + key: entry + .name + .filter(|name| !name.is_empty()) + .ok_or_else(|| SourceError::Other("source listing object has no name".to_string()))?, etag: entry.etag, - size: entry.size, + size: entry + .size + .ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?, last_modified: entry.last_modified, storage_class: entry.access_tier, // Azure ETags carry no part count; the listing @@ -417,6 +455,12 @@ fn parse_list_blobs(xml: &str) -> Result { "blobprefix" => { depth = depth.saturating_sub(1); in_blob_prefix = false; + prefixes.push( + blob_prefix + .take() + .filter(|name| !name.is_empty()) + .ok_or_else(|| SourceError::Other("source listing prefix has no name".to_string()))?, + ); } "properties" | "blobs" | "enumerationresults" => depth = depth.saturating_sub(1), _ => {} @@ -478,16 +522,22 @@ fn apply_list_field( name: &str, text: String, blob: &mut Option, - prefixes: &mut Vec, + blob_prefix: &mut Option, next_marker: &mut Option, in_blob_prefix: bool, -) { +) -> Result<(), SourceError> { match name { "name" => { if in_blob_prefix { - prefixes.push(text); + if blob_prefix.is_some() { + return Err(SourceError::Other("source listing prefix has duplicate names".to_string())); + } + *blob_prefix = Some(text); } else if let Some(entry) = blob.as_mut() { - entry.name = text; + if entry.name.is_some() { + return Err(SourceError::Other("source listing object has duplicate names".to_string())); + } + entry.name = Some(text); } } "nextmarker" => *next_marker = Some(text), @@ -498,7 +548,14 @@ fn apply_list_field( } "content-length" => { if let Some(entry) = blob.as_mut() { - entry.size = text.trim().parse().unwrap_or(0); + if entry.size.is_some() { + return Err(SourceError::Other("source listing object has duplicate sizes".to_string())); + } + entry.size = Some( + text.trim() + .parse() + .map_err(|_| SourceError::Other("source listing object has no valid size".to_string()))?, + ); } } "last-modified" => { @@ -513,6 +570,7 @@ fn apply_list_field( } _ => {} } + Ok(()) } /// Parses a `Get Blob Tags` response. @@ -599,7 +657,7 @@ mod tests { use super::*; use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract}; use crate::on_demand_migration::source_client::SourceError; - use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server}; + use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, assert_requests, scripted_server}; const LIST_PAGE: &str = r#" @@ -769,6 +827,168 @@ mod tests { assert!(parse_blob_tags("").is_err(), "a truncated tag set must fail"); } + #[tokio::test] + async fn native_listing_rejects_missing_or_invalid_required_object_fields() { + for entry in [ + "", + "1", + "1", + "broken", + "broken", + "broken-1", + "broken18446744073709551616", + "brokennot-a-size", + "", + "", + "", + ] { + // Reject the entire page even if a valid object precedes the bad + // entry, so callers cannot expose partial data or advance its cursor. + let body = format!( + "valid1{entry}next" + ); + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await; + let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])) + .list(&SourceListRequest { + prefix: Some("dir/"), + delimiter: Some("/"), + continuation_token: Some("opaque+/="), + max_keys: 2, + ..Default::default() + }) + .await + .expect_err("malformed object must reject the complete native page"); + assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}"); + assert!(!err.is_retryable()); + assert_requests( + &recorded, + &[( + "GET", + "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&marker=opaque%2B%2F%3D&maxresults=2", + )], + ); + } + } + + #[tokio::test] + async fn native_listing_rejects_duplicate_fields_and_nested_entries() { + for entry in [ + "ab1", + "b1", + "a12", + "a/b/", + "b/", + "a1b2", + "a1b/", + "a/b2", + "a/b/", + ] { + let body = format!( + "valid0{entry}next" + ); + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await; + let result = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])) + .list(&SourceListRequest { + delimiter: Some("/"), + continuation_token: Some("opaque+/="), + max_keys: 2, + ..Default::default() + }) + .await; + let err = result.expect_err("ambiguous entries must reject the entire page and its cursor"); + assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}"); + assert!(!err.is_retryable(), "{entry}: {err:?}"); + assert_requests( + &recorded, + &[( + "GET", + "/legacy?restype=container&comp=list&delimiter=%2F&marker=opaque%2B%2F%3D&maxresults=2", + )], + ); + } + } + + #[tokio::test] + async fn native_listing_preserves_zero_size_unicode_prefixes_and_opaque_cursors() { + let body = "目录/空 & file0目录/子/opaque+/="; + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await; + let page = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])) + .list(&SourceListRequest { + max_keys: 2, + ..Default::default() + }) + .await + .expect("valid native page"); + assert_eq!(page.objects.len(), 1); + assert_eq!(page.objects[0].key, "目录/空 & file"); + assert_eq!(page.objects[0].size, 0); + assert_eq!(page.common_prefixes, ["目录/子/"]); + assert!(page.is_truncated); + assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/=")); + assert_requests(&recorded, &[("GET", "/legacy?restype=container&comp=list&maxresults=2")]); + } + + #[tokio::test] + async fn encoded_listing_preserves_required_field_and_entry_validation() { + for (entry, expected_error) in [ + ( + r#"a%2Fb"#, + "source listing object has no valid size", + ), + ( + r#"a%2Fb-1"#, + "source listing object has no valid size", + ), + ( + r#"a%2Fba/b1"#, + "source listing object has duplicate names", + ), + ( + r#"a%2Fb12"#, + "source listing object has duplicate sizes", + ), + ( + r#"a%2Fa/"#, + "source listing prefix has duplicate names", + ), + (r#""#, "source listing prefix has no name"), + ( + r#"a%2Fb1c%2Fd2"#, + "source listing entries must not be nested", + ), + ( + r#"a%2Fb%2F"#, + "source listing entries must not be nested", + ), + ] { + let body = format!( + r#"valid%252F0{entry}opaque%2B+marker"# + ); + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await; + let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])) + .list(&SourceListRequest { + delimiter: Some("/"), + continuation_token: Some("opaque%2B+marker"), + max_keys: 2, + ..Default::default() + }) + .await + .expect_err("encoded names cannot bypass whole-page validation"); + assert!(!err.is_retryable(), "{entry}: {err:?}"); + let SourceError::Other(message) = err else { + panic!("wrong error class for {entry}: {err:?}"); + }; + assert_eq!(message, expected_error, "{entry}"); + assert_requests( + &recorded, + &[( + "GET", + "/legacy?restype=container&comp=list&delimiter=%2F&marker=opaque%252B%2Bmarker&maxresults=2", + )], + ); + } + } + #[test] fn blob_tags_parse_into_the_shared_tag_map() { let tags = parse_blob_tags(TAGS).expect("tags should parse"); @@ -1251,6 +1471,184 @@ mod tests { ] } + #[tokio::test] + async fn object_not_found_requires_provider_evidence_or_one_successful_head_probe() { + for method in [Method::HEAD, Method::GET] { + for (status, code, expected) in [ + (404, Some("BlobNotFound"), "not_found"), + (403, Some("BlobNotFound"), "access_denied"), + (404, Some("ContainerNotFound"), "other"), + (404, Some("BlobVersionNotFound"), "other"), + (404, Some("UnrecognizedError"), "other"), + (404, None, if method == Method::HEAD { "not_found" } else { "other" }), + (404, Some("ResourceNotFound"), if method == Method::HEAD { "not_found" } else { "other" }), + ] { + let probes = method == Method::HEAD && status == 404 && matches!(code, None | Some("ResourceNotFound")); + let headers = code + .map(|value| vec![(HEADER_ERROR_CODE, value.to_string())]) + .unwrap_or_default(); + let mut responses = vec![ScriptedResponse::new(status, headers, "untrusted-error-body".to_string())]; + if probes { + responses.push(ScriptedResponse::new(200, Vec::new(), String::new())); + } + let (endpoint, recorded) = scripted_server(responses).await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + let result = if method == Method::HEAD { + backend.head("missing").await.map(|_| ()) + } else { + backend.get("missing", None).await.map(|_| ()) + }; + let err = result.expect_err("object error must remain an error"); + assert_eq!(err.class_label(), expected, "{method} {status} {code:?}: {err:?}"); + assert!(!err.is_retryable(), "{err:?}"); + assert!(!err.to_string().contains("untrusted-error-body")); + let mut requests = vec![(method.as_str(), "/legacy/missing")]; + if probes { + requests.push(("HEAD", "/legacy?restype=container")); + } + assert_requests(&recorded, &requests); + } + } + } + + #[tokio::test] + async fn s3_not_found_alias_never_proves_native_object_absence() { + for selector in [None, Some("versionid"), Some("snapshot")] { + for operation in ["head", "get", "list", "tags", "probe"] { + if selector.is_some() && !matches!(operation, "head" | "get") { + continue; + } + for (status, expected, retryable) in [ + (403, "access_denied", false), + (404, "other", false), + (416, "other", false), + (500, "server_error", true), + ] { + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new( + status, + vec![(HEADER_ERROR_CODE, "NoSuchKey".to_string())], + "untrusted-error-body".to_string(), + )]) + .await; + let credential = selector.map_or_else( + || Credential::SharedKey(vec![7_u8; 32]), + |selector| Credential::Sas(vec![(selector.to_string(), "old-version".to_string())]), + ); + let backend = backend(&endpoint, credential); + let result = match operation { + "head" => backend.head("missing").await.map(|_| ()), + "get" => backend.get("missing", None).await.map(|_| ()), + "list" => backend.list(&SourceListRequest::default()).await.map(|_| ()), + "tags" => backend.tagging("missing").await.map(|_| ()), + "probe" => backend.probe().await, + _ => unreachable!(), + }; + let err = result.expect_err("an S3 error alias is not Azure absence evidence"); + assert_eq!(err.class_label(), expected, "{operation} {selector:?} HTTP {status}: {err:?}"); + assert_eq!(err.is_retryable(), retryable, "{operation} {selector:?} HTTP {status}: {err:?}"); + if status == 500 { + assert!(matches!(err, SourceError::ServerError(500))); + } + assert!(!err.to_string().contains("untrusted-error-body")); + let (method, mut target) = match operation { + "head" => ("HEAD", "/legacy/missing".to_string()), + "get" => ("GET", "/legacy/missing".to_string()), + "list" => ("GET", "/legacy?restype=container&comp=list".to_string()), + "tags" => ("GET", "/legacy/missing?comp=tags".to_string()), + "probe" => ("HEAD", "/legacy?restype=container".to_string()), + _ => unreachable!(), + }; + if let Some(selector) = selector { + target.push_str(&format!("?{selector}=old-version")); + } + assert_requests(&recorded, &[(method, target.as_str())]); + } + } + } + } + + #[tokio::test] + async fn ambiguous_head_preserves_the_container_probe_failure() { + for (status, expected, retryable) in [ + (403, "access_denied", false), + (404, "other", false), + (429, "throttled", true), + (500, "server_error", true), + (503, "throttled", true), + ] { + let (endpoint, recorded) = scripted_server(vec![ + ScriptedResponse::new(404, Vec::new(), String::new()), + // A BlobNotFound header on a container request cannot prove + // that the object is missing, regardless of this status. + ScriptedResponse::new(status, vec![(HEADER_ERROR_CODE, "BlobNotFound".to_string())], String::new()), + ]) + .await; + let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])) + .head("missing") + .await + .expect_err("failed probe must not become object absence"); + assert_eq!(err.class_label(), expected, "probe {status}: {err:?}"); + assert_eq!(err.is_retryable(), retryable, "probe {status}: {err:?}"); + if status == 500 { + assert!(matches!(err, SourceError::ServerError(500))); + } + assert_requests(&recorded, &[("HEAD", "/legacy/missing"), ("HEAD", "/legacy?restype=container")]); + } + } + + #[tokio::test] + async fn version_and_snapshot_absence_are_not_missing_current_blobs() { + for selector in ["versionid", "snapshot"] { + for code in [None, Some("BlobNotFound"), Some("ResourceNotFound")] { + for method in [Method::HEAD, Method::GET] { + let headers = code + .map(|value| vec![(HEADER_ERROR_CODE, value.to_string())]) + .unwrap_or_default(); + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(404, headers, String::new())]).await; + let backend = backend(&endpoint, Credential::Sas(vec![(selector.to_string(), "old-version".to_string())])); + let result = if method == Method::HEAD { + backend.head("object").await.map(|_| ()) + } else { + backend.get("object", None).await.map(|_| ()) + }; + let err = result.expect_err("missing selected version must remain a source error"); + assert!(matches!(err, SourceError::Other(_)), "{method} {selector} {code:?}: {err:?}"); + assert_requests(&recorded, &[(method.as_str(), &format!("/legacy/object?{selector}=old-version"))]); + } + } + } + } + + #[tokio::test] + async fn blob_not_found_header_is_not_object_absence_for_list_or_tags() { + for tags in [false, true] { + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new( + 404, + vec![(HEADER_ERROR_CODE, "BlobNotFound".to_string())], + String::new(), + )]) + .await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + let result = if tags { + backend.tagging("missing").await.map(|_| ()) + } else { + backend.list(&SourceListRequest::default()).await.map(|_| ()) + }; + assert!(matches!(result, Err(SourceError::Other(_))), "tags={tags}: {result:?}"); + assert_requests( + &recorded, + &[( + "GET", + if tags { + "/legacy/missing?comp=tags" + } else { + "/legacy?restype=container&comp=list" + }, + )], + ); + } + } + #[tokio::test] async fn azure_backend_satisfies_the_shared_backend_contract() { let mut ranged = contract_blob_headers(); @@ -1258,7 +1656,7 @@ mod tests { // A HEAD reports the object size with no body, exactly as Azure does. let mut head_only = contract_blob_headers(); head_only.push(("Content-Length", "5".to_string())); - let (endpoint, _) = scripted_server(vec![ + let (endpoint, recorded) = scripted_server(vec![ ScriptedResponse::new(200, head_only, String::new()), ScriptedResponse::new(200, contract_blob_headers(), "hello".to_string()), ScriptedResponse::new(206, ranged, "ell".to_string()), @@ -1288,6 +1686,23 @@ mod tests { }, ) .await; + assert_requests( + &recorded, + &[ + ("HEAD", "/legacy/dir/a.txt"), + ("GET", "/legacy/dir/a.txt"), + ("GET", "/legacy/dir/a.txt"), + ("GET", "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&maxresults=2"), + ( + "GET", + "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&marker=cursor-1&maxresults=2", + ), + ("GET", "/legacy/dir/a.txt?comp=tags"), + ("HEAD", "/legacy?restype=container"), + ("HEAD", "/legacy/missing"), + ("HEAD", "/legacy/secret"), + ], + ); } #[tokio::test] diff --git a/rustfs/src/on_demand_migration/config.rs b/rustfs/src/on_demand_migration/config.rs index 28b67819c..6f8f28e66 100644 --- a/rustfs/src/on_demand_migration/config.rs +++ b/rustfs/src/on_demand_migration/config.rs @@ -106,12 +106,12 @@ pub struct SourceConfig { pub tls: TlsConfig, /// Required for [`Provider::Azure`] and rejected for every other /// provider. - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub azure: Option, /// Required for [`Provider::GcsNative`] and rejected for every other /// provider. [`Provider::Gcs`] keeps using `credentials` because it /// speaks the S3 interoperability API. - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub gcs: Option, } @@ -808,6 +808,10 @@ impl EndpointKey { mod tests { use super::*; + mod before_native_sources { + include!("../../fixtures/on_demand_migration/source_config_e2a.rs"); + } + const FULL_JSON: &str = r#"{ "version": 1, "enabled": true, @@ -878,6 +882,32 @@ mod tests { assert_eq!(minimal.policy.source_timeout.first_byte_ms, 15_000); } + #[test] + fn s3_config_writes_remain_readable_by_the_strict_pre_native_reader() { + // FULL_JSON is the complete config fixture already present in e2a921bc. + for provider in ["s3", "aws", "minio", "rustfs", "r2", "gcs"] { + let mut old_wire: serde_json::Value = serde_json::from_str(FULL_JSON).expect("historical config fixture"); + old_wire["source"]["provider"] = provider.into(); + let config = OnDemandMigrationConfig::from_json(&serde_json::to_vec(&old_wire).expect("historical wire")) + .expect("current reader accepts the historical source"); + let wire = config.to_json().expect("persist current config"); + let actual: serde_json::Value = serde_json::from_slice(&wire).expect("persisted config JSON"); + let old_source: before_native_sources::SourceConfig = serde_json::from_value(actual["source"].clone()) + .expect("an existing S3 source must remain readable by the strict e2a source consumer"); + assert_eq!(serde_json::to_value(old_source).expect("old reader wire"), old_wire["source"]); + assert_eq!(actual, old_wire, "provider={provider}: no existing config field or value may change"); + + for field in ["azure", "gcs"] { + let mut rejected = old_wire["source"].clone(); + rejected[field] = serde_json::Value::Null; + assert!( + serde_json::from_value::(rejected).is_err(), + "the frozen old reader must reject {field}, even when null" + ); + } + } + } + #[test] fn unknown_fields_are_rejected_at_every_level() { for (label, json) in [ @@ -1080,6 +1110,19 @@ mod tests { for cfg in [azure_cfg(), gcs_native_cfg()] { let json = cfg.to_json().expect("config must serialize"); assert_eq!(OnDemandMigrationConfig::from_json(&json).expect("config must parse"), cfg); + let wire: serde_json::Value = serde_json::from_slice(&json).expect("native config JSON"); + let (present, absent, expected) = match cfg.source.provider { + Provider::Azure => ("azure", "gcs", serde_json::to_value(&cfg.source.azure).expect("Azure block")), + Provider::GcsNative => ("gcs", "azure", serde_json::to_value(&cfg.source.gcs).expect("GCS block")), + _ => unreachable!("native fixture"), + }; + assert!(expected.is_object(), "native credentials must be present"); + assert_eq!(wire["source"][present], expected); + assert!(wire["source"].get(absent).is_none()); + assert!( + serde_json::from_value::(wire["source"].clone()).is_err(), + "native providers still require upgraded readers" + ); } // The wire labels are part of the admin contract. assert!( diff --git a/rustfs/src/on_demand_migration/gcs.rs b/rustfs/src/on_demand_migration/gcs.rs index e5696748d..9adfffadb 100644 --- a/rustfs/src/on_demand_migration/gcs.rs +++ b/rustfs/src/on_demand_migration/gcs.rs @@ -55,10 +55,6 @@ use url::Url; /// Read-only object scope: this backend never writes to the source. const READ_ONLY_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_only"; const METADATA_PREFIX: &str = "x-goog-meta-"; -/// GCS reports its error code in the response body, not a header; the shared -/// transport takes a header name, so it is given one that never matches and -/// classification falls back to the status. -const NO_ERROR_CODE_HEADER: &str = "x-goog-unused-error-code"; /// One `objects.list` page is small; refuse an unbounded document. const MAX_JSON_BYTES: usize = 8 * 1024 * 1024; @@ -125,7 +121,7 @@ impl GcsNativeSourceBackend { } async fn send_object(&self, request: reqwest::Request) -> Result { - match self.http.send_object(request, NO_ERROR_CODE_HEADER).await { + match self.http.send_object(request, None).await { Err(SourceError::NotFound) => { // An XML object URL also returns 404 when its bucket is gone. // Reuse the read-only listing probe before caching a key miss. @@ -225,7 +221,7 @@ impl SourceBackend for GcsNativeSourceBackend { } let request = self.request(Method::GET, url, HeaderMap::new()).await?; - let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?; + let response = self.http.send(request, None).await?; let body = read_text(response, MAX_JSON_BYTES).await?; parse_objects_list(&body) } @@ -245,7 +241,7 @@ impl SourceBackend for GcsNativeSourceBackend { let mut url = self.objects_url()?; url.query_pairs_mut().append_pair("maxResults", "1"); let request = self.request(Method::GET, url, HeaderMap::new()).await?; - let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?; + let response = self.http.send(request, None).await?; read_text(response, MAX_JSON_BYTES) .await .and_then(|body| parse_objects_list(&body))?; @@ -284,28 +280,38 @@ struct ListedObject { fn parse_objects_list(body: &str) -> Result { let listing: ObjectsList = serde_json::from_str(body).map_err(|err| SourceError::Other(format!("source listing is not valid JSON: {err}")))?; + if listing.prefixes.iter().any(|prefix| prefix.is_empty()) { + return Err(SourceError::Other("source listing prefix has no name".to_string())); + } let next_continuation_token = listing.next_page_token.filter(|token| !token.is_empty()); let objects = listing .items .into_iter() .map(|item| { + if item.name.is_empty() { + return Err(SourceError::Other("source listing object has no name".to_string())); + } + let size = item + .size + .and_then(|size| size.parse::().ok()) + .ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?; let etag = item .md5_hash .as_deref() .and_then(base64_md5_to_hex) .or_else(|| item.etag.map(|etag| etag.trim_matches('"').to_string())) .filter(|etag| !etag.is_empty()); - SourceObject { + Ok(SourceObject { key: item.name, etag, - size: item.size.and_then(|size| size.parse().ok()).unwrap_or(0), + size, last_modified: item.updated.as_deref().and_then(parse_http_timestamp), storage_class: item.storage_class, // GCS never encodes a part count in a digest or an ETag. is_multipart_etag: false, - } + }) }) - .collect(); + .collect::>()?; Ok(SourcePage { objects, @@ -319,7 +325,7 @@ fn parse_objects_list(body: &str) -> Result { mod tests { use super::*; use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract}; - use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server}; + use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, assert_requests, scripted_server}; use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder; const LIST_PAGE_ONE: &str = r#"{ @@ -573,6 +579,197 @@ mod tests { } } + #[tokio::test] + async fn native_listing_rejects_missing_or_invalid_required_object_fields() { + for entry in [ + r#"{"size":"1"}"#, + r#"{"name":"","size":"1"}"#, + r#"{"name":"broken"}"#, + r#"{"name":"broken","size":null}"#, + r#"{"name":"broken","size":""}"#, + r#"{"name":"broken","size":"-1"}"#, + r#"{"name":"broken","size":"18446744073709551616"}"#, + r#"{"name":"broken","size":"not-a-size"}"#, + r#"{"name":"broken","size":1}"#, + ] { + let body = format!(r#"{{"items":[{{"name":"valid","size":"1"}},{entry}],"nextPageToken":"next"}}"#); + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await; + let err = backend(&endpoint) + .list(&SourceListRequest { + prefix: Some("dir/"), + delimiter: Some("/"), + continuation_token: Some("opaque+/="), + max_keys: 2, + ..Default::default() + }) + .await + .expect_err("malformed object must reject the complete native page"); + assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}"); + assert!(!err.is_retryable()); + assert_requests( + &recorded, + &[( + "GET", + "/storage/v1/b/legacy/o?prefix=dir%2F&delimiter=%2F&pageToken=opaque%2B%2F%3D&maxResults=2", + )], + ); + } + } + + #[tokio::test] + async fn native_listing_rejects_empty_prefix_entries() { + for body in [ + r#"{"items":[{"name":"valid","size":"1"}],"prefixes":[""],"nextPageToken":"next"}"#, + r#"{"prefixes":[""],"nextPageToken":"next"}"#, + r#"{"prefixes":["目录/子/",""],"nextPageToken":"next"}"#, + ] { + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await; + let result = backend(&endpoint) + .list(&SourceListRequest { + delimiter: Some("/"), + continuation_token: Some("opaque+/="), + max_keys: 2, + ..Default::default() + }) + .await; + let err = result.expect_err("an empty prefix must reject the entire page and its cursor"); + assert!(matches!(err, SourceError::Other(_)), "{body}: {err:?}"); + assert!(!err.is_retryable()); + assert_requests( + &recorded, + &[("GET", "/storage/v1/b/legacy/o?delimiter=%2F&pageToken=opaque%2B%2F%3D&maxResults=2")], + ); + } + + let body = r#"{"prefixes":["目录/子/"],"nextPageToken":"opaque+/="}"#; + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await; + let page = backend(&endpoint) + .list(&SourceListRequest { + delimiter: Some("/"), + max_keys: 1, + ..Default::default() + }) + .await + .expect("a valid prefix-only page must remain usable"); + assert!(page.objects.is_empty()); + assert_eq!(page.common_prefixes, ["目录/子/"]); + assert!(page.is_truncated); + assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/=")); + assert_requests(&recorded, &[("GET", "/storage/v1/b/legacy/o?delimiter=%2F&maxResults=1")]); + } + + #[tokio::test] + async fn native_listing_preserves_zero_size_unicode_prefixes_and_opaque_cursors() { + let body = r#"{"items":[{"name":"目录/空 & file","size":"0"}],"prefixes":["目录/子/"],"nextPageToken":"opaque+/="}"#; + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await; + let page = backend(&endpoint) + .list(&SourceListRequest { + max_keys: 2, + ..Default::default() + }) + .await + .expect("valid native page"); + assert_eq!(page.objects.len(), 1); + assert_eq!(page.objects[0].key, "目录/空 & file"); + assert_eq!(page.objects[0].size, 0); + assert_eq!(page.common_prefixes, ["目录/子/"]); + assert!(page.is_truncated); + assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/=")); + assert_requests(&recorded, &[("GET", "/storage/v1/b/legacy/o?maxResults=2")]); + } + + #[tokio::test] + async fn missing_object_head_requires_one_successful_bucket_probe() { + for (status, body, expected, retryable) in [ + (200, "{}", "not_found", false), + (403, "", "access_denied", false), + (404, "", "other", false), + (429, "", "throttled", true), + (500, "", "server_error", true), + (503, "", "throttled", true), + (200, "not JSON", "other", false), + ] { + let (endpoint, recorded) = scripted_server(vec![ + ScriptedResponse::new(404, Vec::new(), String::new()), + ScriptedResponse::new(status, Vec::new(), body.to_string()), + ]) + .await; + let err = backend(&endpoint).head("missing").await.expect_err("missing HEAD must fail"); + assert_eq!(err.class_label(), expected, "probe {status} {body:?}: {err:?}"); + assert_eq!(err.is_retryable(), retryable, "probe {status} {body:?}: {err:?}"); + if status == 500 { + assert!(matches!(err, SourceError::ServerError(500))); + } + assert_requests(&recorded, &[("HEAD", "/legacy/missing"), ("GET", "/storage/v1/b/legacy/o?maxResults=1")]); + } + } + + #[tokio::test] + async fn denied_object_reads_do_not_probe_or_become_object_absence() { + for method in [Method::HEAD, Method::GET] { + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new( + 403, + vec![("x-goog-unused-error-code", "NoSuchKey".to_string())], + "untrusted-error-body".to_string(), + )]) + .await; + let backend = backend(&endpoint); + let result = if method == Method::HEAD { + backend.head("missing").await.map(|_| ()) + } else { + backend.get("missing", None).await.map(|_| ()) + }; + let err = result.expect_err("denied object read must remain a failure"); + assert_eq!(err.class_label(), "access_denied"); + assert!(!err.is_retryable()); + assert!(!err.to_string().contains("untrusted-error-body")); + assert_requests(&recorded, &[(method.as_str(), "/legacy/missing")]); + } + } + #[tokio::test] + async fn non_object_errors_ignore_untrusted_error_code_headers() { + for probe in [false, true] { + for (status, expected, retryable) in [(403, "access_denied", false), (500, "server_error", true)] { + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new( + status, + vec![("x-goog-unused-error-code", "NoSuchKey".to_string())], + "untrusted-error-body".to_string(), + )]) + .await; + let backend = backend(&endpoint); + let result = if probe { + backend.probe().await + } else { + backend + .list(&SourceListRequest { + max_keys: 2, + ..Default::default() + }) + .await + .map(|_| ()) + }; + let err = result.expect_err("a synthetic provider header cannot change the source status"); + assert_eq!(err.class_label(), expected, "probe={probe} status={status}: {err:?}"); + assert_eq!(err.is_retryable(), retryable); + assert!(!err.to_string().contains("untrusted-error-body")); + if status == 500 { + assert!(matches!(err, SourceError::ServerError(500))); + } + assert_requests( + &recorded, + &[( + "GET", + if probe { + "/storage/v1/b/legacy/o?maxResults=1" + } else { + "/storage/v1/b/legacy/o?maxResults=2" + }, + )], + ); + } + } + } + /// GCS states its error code in the response body, which this backend never /// reads, so every class must follow from the status alone. The classes are /// what the runtime acts on: only `NotFound` is negative-cached, and only a diff --git a/rustfs/src/on_demand_migration/list_through.rs b/rustfs/src/on_demand_migration/list_through.rs index 124d82d8f..3613eca03 100644 --- a/rustfs/src/on_demand_migration/list_through.rs +++ b/rustfs/src/on_demand_migration/list_through.rs @@ -94,13 +94,16 @@ pub struct MergePick { } /// The continuation-token envelope. Opaque to clients: it is serialized as -/// framed JSON and then base64-encoded by the same helper as a local marker. +/// JSON, optionally framed, then base64-encoded like a local marker. /// /// A `null` cursor with `done = false` means "list that side from the start"; /// `done = true` means the side is finished and must not be listed again. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ListThroughToken { + /// Transport framing observed by the decoder, never an envelope field. + #[serde(skip)] + pub framed: bool, /// Envelope marker, always [`LIST_THROUGH_TOKEN_TAG`]. pub t: String, pub v: u32, @@ -127,6 +130,7 @@ pub struct ListThroughToken { impl ListThroughToken { fn new(local: SideCursor, source: SideCursor, last_key: Option) -> Self { Self { + framed: false, t: LIST_THROUGH_TOKEN_TAG.to_string(), v: LIST_THROUGH_TOKEN_VERSION, local: local.token, @@ -141,7 +145,12 @@ impl ListThroughToken { pub fn encode(&self) -> String { // The envelope is built here from owned strings, so serialization // cannot fail; the fallback keeps the signature infallible. - format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default()) + let json = serde_json::to_string(self).unwrap_or_default(); + if self.framed { + format!("{LIST_THROUGH_TOKEN_PREFIX}{json}") + } else { + json + } } } @@ -165,16 +174,30 @@ pub enum ListThroughTokenError { /// Classifies an already base64-decoded continuation token. /// -/// Only a framed JSON object is read as a merged token; -/// anything else is a local marker, so a bucket that turns `list_through` off -/// keeps paginating with the tokens it handed out. A token that *is* an -/// envelope but was tampered with (unknown version, unknown field, truncated -/// JSON) is an error, never a silent fallback. +/// Framed envelopes and complete historical writer envelopes are merged tokens. +/// Partial JSON-shaped keys remain local markers. A key identical to a complete +/// historical envelope is inherently ambiguous and retains merged semantics. +/// Recognized envelopes share the same version, count and field validation. pub fn decode_continuation_token(decoded: &str) -> Result { - let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else { - return Ok(ListThroughCursor::Local(decoded.to_string())); + let (payload, framed) = match decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) { + Some(payload) => (payload, true), + None if decoded.starts_with('{') => (decoded, false), + None => return Ok(ListThroughCursor::Local(decoded.to_string())), }; - let value = serde_json::from_str::(payload).map_err(|_| ListThroughTokenError::Malformed)?; + let value = match serde_json::from_str::(payload) { + Ok(value) => value, + Err(_) if framed => return Err(ListThroughTokenError::Malformed), + Err(_) => return Ok(ListThroughCursor::Local(decoded.to_string())), + }; + // RUSTFS_COMPAT_TODO(odm-list-bare-envelope): old writers issued bare JSON. Remove after all supported readers understand framing and outstanding bare listings have drained or explicitly restarted. + if !framed + && (value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) + || ["v", "local", "local_done", "source", "source_done", "last_key"] + .iter() + .any(|field| value.get(field).is_none())) + { + return Ok(ListThroughCursor::Local(decoded.to_string())); + } if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) { return Err(ListThroughTokenError::Malformed); } @@ -198,7 +221,10 @@ pub fn decode_continuation_token(decoded: &str) -> Result return Err(ListThroughTokenError::Malformed), } serde_json::from_value::(value) - .map(|token| ListThroughCursor::Merged(Box::new(token))) + .map(|mut token| { + token.framed = framed; + ListThroughCursor::Merged(Box::new(token)) + }) .map_err(|_| ListThroughTokenError::Malformed) } @@ -643,6 +669,126 @@ impl Default for SourceListRateLimiter { } } +// Frozen framed-only codec from e1608fbd9ca934d157b5de46c80b4393f2dd3dd6. +// Keep its own DTO and constants: current-reader round trips cannot establish +// whether a deployed framed-only reader accepts the bytes we issue. +#[cfg(test)] +pub(crate) mod e160_framed_reader { + use serde::{Deserialize, Serialize}; + + /// The continuation-token version used by ordinary progressing pages. + pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1; + const LIST_THROUGH_PROGRESS_TOKEN_VERSION: u32 = 2; + + /// The sixteenth consecutive merged page without a key or new EOF fails. + /// This also bounds legitimate sparse listings; it is not a cycle detector. + pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16; + + /// Envelope marker. A bucket that is *not* merging hands out the local + /// listing's own marker, so the decoder needs a positive signal before it + /// treats an opaque token as a merged one. + const LIST_THROUGH_TOKEN_TAG: &str = "odm-list"; + // Object keys cannot contain NUL (bucket::utils::is_valid_object_prefix), + // so this framing cannot collide with a local key used as an opaque marker. + const LIST_THROUGH_TOKEN_PREFIX: &str = "\0odm-list:"; + + /// The continuation-token envelope. Opaque to clients: it is serialized as + /// framed JSON and then base64-encoded by the same helper as a local marker. + /// + /// A `null` cursor with `done = false` means "list that side from the start"; + /// `done = true` means the side is finished and must not be listed again. + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + pub struct ListThroughToken { + /// Envelope marker, always [`LIST_THROUGH_TOKEN_TAG`]. + pub t: String, + pub v: u32, + #[serde(default)] + pub local: Option, + #[serde(default)] + pub local_done: bool, + #[serde(default)] + pub source: Option, + #[serde(default)] + pub source_done: bool, + /// Last entry the previous page consumed. A side whose page was only + /// partially consumed is re-listed from the same cursor and everything at + /// or below this key is dropped, which is delimiter-safe: a rolled-up + /// common prefix compares as itself, never as its members. + #[serde(default)] + pub last_key: Option, + /// Consecutive empty truncated merged pages, present only in v2 tokens. + /// Ordinary v1 tokens retain their original serialized shape. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub no_progress: Option, + } + + impl ListThroughToken { + pub fn encode(&self) -> String { + // The envelope is built here from owned strings, so serialization + // cannot fail; the fallback keeps the signature infallible. + format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default()) + } + } + + /// What a decoded (base64-stripped) continuation token turned out to be. + #[derive(Clone, Debug, PartialEq, Eq)] + pub enum ListThroughCursor { + /// A plain local listing marker: the bucket was not merging when the token + /// was issued, or the client is paginating a non-merged listing. + Local(String), + Merged(Box), + } + + #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] + pub enum ListThroughTokenError { + #[error("continuation token version {0} is not supported")] + UnsupportedVersion(u32), + /// The message never echoes the token: it is client-controlled input. + #[error("continuation token is malformed")] + Malformed, + } + + /// Classifies an already base64-decoded continuation token. + /// + /// Only a framed JSON object is read as a merged token; + /// anything else is a local marker, so a bucket that turns `list_through` off + /// keeps paginating with the tokens it handed out. A token that *is* an + /// envelope but was tampered with (unknown version, unknown field, truncated + /// JSON) is an error, never a silent fallback. + pub fn decode_continuation_token(decoded: &str) -> Result { + let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else { + return Ok(ListThroughCursor::Local(decoded.to_string())); + }; + let value = serde_json::from_str::(payload).map_err(|_| ListThroughTokenError::Malformed)?; + if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) { + return Err(ListThroughTokenError::Malformed); + } + match value.get("v").and_then(serde_json::Value::as_u64) { + Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => { + // v1 readers reject this field even when it is null or zero. + if value.get("no_progress").is_some() { + return Err(ListThroughTokenError::Malformed); + } + } + Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => { + if !value + .get("no_progress") + .and_then(serde_json::Value::as_u64) + .is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count)) + { + return Err(ListThroughTokenError::Malformed); + } + } + Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)), + None => return Err(ListThroughTokenError::Malformed), + } + serde_json::from_value::(value) + .map(|token| ListThroughCursor::Merged(Box::new(token))) + .map_err(|_| ListThroughTokenError::Malformed) + } +} + #[cfg(test)] mod tests { use super::*; @@ -797,6 +943,7 @@ mod tests { #[test] fn a_degraded_page_keeps_the_source_cursor_for_the_next_one() { let resume = ListThroughToken { + framed: false, t: LIST_THROUGH_TOKEN_TAG.to_string(), v: LIST_THROUGH_TOKEN_VERSION, local: Some("local-1".to_string()), @@ -1033,7 +1180,7 @@ mod tests { #[test] fn token_round_trips_and_rejects_tampering() { - let token = ListThroughToken::new( + let mut token = ListThroughToken::new( SideCursor { token: Some("l".to_string()), done: false, @@ -1041,6 +1188,7 @@ mod tests { SideCursor { token: None, done: true }, Some("k".to_string()), ); + token.framed = true; let encoded = token.encode(); assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token)))); @@ -1089,35 +1237,148 @@ mod tests { #[test] fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() { - fn framed(payload: &str) -> String { - format!("{LIST_THROUGH_TOKEN_PREFIX}{payload}") - } - let token = progress_token(None, true, false); assert_eq!( token.encode(), - concat!( - "\0odm-list:", - r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"# - ) + r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"# ); - for count in 1..MAX_LIST_NO_PROGRESS_PAGES { - let token = progress_token(Some(count), true, false); - assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token)))); - } - for version in [1, 2] { - for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] { - let encoded = framed(&format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#)); + for framed in [false, true] { + let prefix = if framed { LIST_THROUGH_TOKEN_PREFIX } else { "" }; + for count in 1..MAX_LIST_NO_PROGRESS_PAGES { + let mut token = progress_token(Some(count), true, false); + token.framed = framed; + assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token)))); + } + // Bare recognition requires the complete shape emitted by old writers; + // partial JSON objects are also valid local keys. + for version in [1, 2] { + for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] { + let encoded = format!( + r#"{prefix}{{"t":"odm-list","v":{version},"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":{value}}}"# + ); + assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}"); + } + } + for encoded in [ + r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1}"#, + r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#, + r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1,"extra":true}"#, + r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1,"framed":true}"#, + ] { + let encoded = format!("{prefix}{encoded}"); assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}"); } + let bumped = format!("{prefix}{}", token.encode().replace("\"v\":1", "\"v\":9")); + assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(9))); } - for payload in [ - r#"{"t":"odm-list","v":1,"no_progress":1}"#, - r#"{"t":"odm-list","v":2}"#, - r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#, + } + + // Frozen decoder from 447f3c704, before framing was introduced. Keeping this + // independent of the current decoder catches a default-writer rollout break. + fn decode_before_framing(decoded: &str) -> Result { + if !decoded.starts_with('{') { + return Ok(ListThroughCursor::Local(decoded.to_string())); + } + let Ok(value) = serde_json::from_str::(decoded) else { + // Not JSON at all: an object key may legitimately start with '{'. + return Ok(ListThroughCursor::Local(decoded.to_string())); + }; + if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) { + return Ok(ListThroughCursor::Local(decoded.to_string())); + } + match value.get("v").and_then(serde_json::Value::as_u64) { + Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => { + // v1 readers reject this field even when it is null or zero. + if value.get("no_progress").is_some() { + return Err(ListThroughTokenError::Malformed); + } + } + Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => { + if !value + .get("no_progress") + .and_then(serde_json::Value::as_u64) + .is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count)) + { + return Err(ListThroughTokenError::Malformed); + } + } + Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)), + None => return Err(ListThroughTokenError::Malformed), + } + serde_json::from_value::(value) + .map(|token| ListThroughCursor::Merged(Box::new(token))) + .map_err(|_| ListThroughTokenError::Malformed) + } + + #[test] + fn historical_writer_fixtures_and_default_output_remain_readable() { + for (wire, version, count) in [ + ( + r#"{"t":"odm-list","v":1,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k"}"#, + 1, + None, + ), + ( + r#"{"t":"odm-list","v":2,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k","no_progress":15}"#, + 2, + Some(15), + ), ] { - let encoded = framed(payload); - assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}"); + let ListThroughCursor::Merged(mut token) = decode_continuation_token(wire).expect("historical issued token") else { + panic!("a historical cursor must not silently become a local marker, even if a key has identical JSON"); + }; + assert_eq!(token.local.as_deref(), Some("local-2")); + assert_eq!(token.source.as_deref(), Some("source-2")); + assert_eq!(token.last_key.as_deref(), Some("k")); + assert_eq!(token.v, version); + assert_eq!(token.no_progress, count); + assert!(!token.framed); + assert_eq!(token.encode(), wire, "bare output retains the historical bytes"); + assert_eq!(decode_before_framing(&token.encode()), Ok(ListThroughCursor::Merged(token.clone()))); + token.framed = true; + let framed = format!("\0odm-list:{wire}"); + assert_eq!(token.encode(), framed, "framing leaves the JSON payload unchanged"); + assert_eq!(decode_continuation_token(&framed), Ok(ListThroughCursor::Merged(token))); + } + } + + #[test] + fn frozen_e160_reader_distinguishes_framing_and_keeps_strict_budget_validation() { + use super::e160_framed_reader as old; + + for raw in [ + r#"{"t":"odm-list","v":1,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k"}"#, + r#"{"t":"odm-list","v":2,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k","no_progress":15}"#, + ] { + assert_eq!(old::decode_continuation_token(raw), Ok(old::ListThroughCursor::Local(raw.to_string()))); + let framed = format!("\0odm-list:{raw}"); + let old::ListThroughCursor::Merged(old_token) = old::decode_continuation_token(&framed).expect("old writer bytes") + else { + panic!("e160 recognizes its own frame"); + }; + assert_eq!(old_token.encode(), framed); + let ListThroughCursor::Merged(current) = decode_continuation_token(&framed).expect("dual reader") else { + panic!("dual readers preserve old framed chains"); + }; + assert_eq!(current.encode(), framed); + assert_eq!(current.local, old_token.local); + assert_eq!(current.local_done, old_token.local_done); + assert_eq!(current.source, old_token.source); + assert_eq!(current.source_done, old_token.source_done); + assert_eq!(current.last_key, old_token.last_key); + assert_eq!(current.v, old_token.v); + assert_eq!(current.no_progress, old_token.no_progress); + } + for count in ["null", "0", "16", "-1", "1.5", "\"1\"", "256"] { + let raw = format!( + "\0odm-list:{{\"t\":\"odm-list\",\"v\":2,\"local\":null,\"local_done\":true,\"source\":\"A\",\"source_done\":false,\"last_key\":null,\"no_progress\":{count}}}" + ); + assert_eq!( + old::decode_continuation_token(&raw), + Err(old::ListThroughTokenError::Malformed), + "{count}" + ); + assert_eq!(decode_continuation_token(&raw), Err(ListThroughTokenError::Malformed), "{count}"); } } diff --git a/rustfs/src/on_demand_migration/native_http.rs b/rustfs/src/on_demand_migration/native_http.rs index 519013de1..4c408fa98 100644 --- a/rustfs/src/on_demand_migration/native_http.rs +++ b/rustfs/src/on_demand_migration/native_http.rs @@ -99,6 +99,7 @@ impl NativeHttp { pub(super) fn for_test(endpoint: Url) -> Self { Self { client: reqwest::Client::builder() + .no_proxy() .redirect(reqwest::redirect::Policy::none()) .build() .expect("test http client should build"), @@ -130,13 +131,13 @@ impl NativeHttp { } /// Sends the request and returns the response only for a 2xx status. - /// Non-2xx statuses are classified from the status and the provider's own + /// Non-2xx statuses are classified from the status and an optional provider /// error-code header; response bodies are not read, so no provider message /// can smuggle credentials or markup into a log line. pub(super) async fn send( &self, request: reqwest::Request, - error_code_header: &str, + error_code_header: Option<&str>, ) -> Result { self.send_classified(request, error_code_header, false).await } @@ -145,7 +146,7 @@ impl NativeHttp { pub(super) async fn send_object( &self, request: reqwest::Request, - error_code_header: &str, + error_code_header: Option<&str>, ) -> Result { self.send_classified(request, error_code_header, true).await } @@ -153,26 +154,42 @@ impl NativeHttp { async fn send_classified( &self, request: reqwest::Request, - error_code_header: &str, + error_code_header: Option<&str>, not_found_on_404_without_code: bool, ) -> Result { - let response = self.client.execute(request).await.map_err(classify_transport_error)?; + let response = self.execute(request).await?; + let status = response.status(); + match Self::check_response(response, error_code_header) { + Err(SourceError::Other(_)) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound), + result => result, + } + } + + pub(super) async fn execute(&self, request: reqwest::Request) -> Result { + self.client.execute(request).await.map_err(classify_transport_error) + } + + pub(super) fn check_response( + response: reqwest::Response, + error_code_header: Option<&str>, + ) -> Result { let status = response.status(); if status.is_success() { return Ok(response); } - let code = response - .headers() - .get(error_code_header) + let code = error_code_header + .and_then(|header| response.headers().get(header)) .and_then(|value| value.to_str().ok()) .map(str::to_string); let message = match &code { Some(code) => format!("source returned HTTP {status} ({code})"), None => format!("source returned HTTP {status}"), }; - match classify_status(status.as_u16(), code.as_deref(), message) { - SourceError::Other(_) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound), - err => Err(err), + match classify_status(status.as_u16(), code.as_deref(), message.clone()) { + // Native object absence needs provider-specific evidence or a + // successful bucket probe, never an alias from the S3 classifier. + SourceError::NotFound => Err(classify_status(status.as_u16(), None, message)), + error => Err(error), } } } diff --git a/rustfs/src/on_demand_migration/source_client.rs b/rustfs/src/on_demand_migration/source_client.rs index 9f3e050c9..62adc5b4c 100644 --- a/rustfs/src/on_demand_migration/source_client.rs +++ b/rustfs/src/on_demand_migration/source_client.rs @@ -337,7 +337,7 @@ const THROTTLE_CODES: &[&str] = &[ "RequestThrottled", "ServerBusy", ]; -const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "BlobNotFound"]; +const NOT_FOUND_CODES: &[&str] = &["NoSuchKey"]; const ACCESS_DENIED_CODES: &[&str] = &[ "AccessDenied", "InvalidAccessKeyId", @@ -1813,10 +1813,11 @@ mod tests { /// The S3 backend behind the scripted connector, without the prefix-mapping /// client on top: the contract is a property of the backend itself. - async fn scripted_s3_backend(responses: Vec) -> S3SourceBackend { + async fn scripted_s3_backend(responses: Vec) -> (S3SourceBackend, Recorded) { let spec = spec(None); + let requests: Recorded = Arc::new(Mutex::new(Vec::new())); let connector = SharedHttpConnector::new(ScriptedConnector { - requests: Arc::new(Mutex::new(Vec::new())), + requests: Arc::clone(&requests), responses: Arc::new(Mutex::new(responses.into_iter().collect())), }); let http_client = http_client_fn(move |_settings, _components| connector.clone()); @@ -1826,17 +1827,20 @@ mod tests { .expect("test spec should build") .http_client(http_client) .interceptor(SourceProxyMarkerInterceptor::new()); - S3SourceBackend { - client: S3Client::from_conf(config.build()), - bucket: spec.bucket.clone(), - } + ( + S3SourceBackend { + client: S3Client::from_conf(config.build()), + bucket: spec.bucket.clone(), + }, + requests, + ) } #[tokio::test] async fn s3_backend_satisfies_the_shared_backend_contract() { let mut ranged = contract_object_headers(3); ranged.push(("content-range", "bytes 1-3/5".to_string())); - let backend = scripted_s3_backend(vec![ + let (backend, requests) = scripted_s3_backend(vec![ ok(contract_object_headers(5), ""), ok(contract_object_headers(5), "hello"), ok(ranged, "ell"), @@ -1845,6 +1849,7 @@ mod tests { ok(Vec::new(), CONTRACT_TAGGING), ok(Vec::new(), ""), status(404, ""), + // An object HEAD 404 requires the existing S3 bucket HEAD probe. ok(Vec::new(), ""), status(403, ACCESS_DENIED_BODY), ]) @@ -1859,6 +1864,32 @@ mod tests { }, ) .await; + let requests = recorded(&requests); + let actual: Vec<_> = requests + .iter() + .map(|request| { + ( + request.method.as_str(), + url::Url::parse(&request.uri).expect("recorded S3 URL").path().to_string(), + ) + }) + .collect(); + let expected = [ + ("HEAD", "/source-bucket/dir/a.txt"), + ("GET", "/source-bucket/dir/a.txt"), + ("GET", "/source-bucket/dir/a.txt"), + ("GET", "/source-bucket/"), + ("GET", "/source-bucket/"), + ("GET", "/source-bucket/dir/a.txt"), + ("HEAD", "/source-bucket/"), + ("HEAD", "/source-bucket/missing"), + ("HEAD", "/source-bucket/"), + ("HEAD", "/source-bucket/secret"), + ]; + assert_eq!(actual, expected.map(|(method, path)| (method, path.to_string()))); + for request in &requests { + assert_outbound_markers(request); + } } fn prefix_client(prefix: Option) -> SourceClient { diff --git a/rustfs/src/on_demand_migration/test_http_fixture.rs b/rustfs/src/on_demand_migration/test_http_fixture.rs index eb6dda014..cce876554 100644 --- a/rustfs/src/on_demand_migration/test_http_fixture.rs +++ b/rustfs/src/on_demand_migration/test_http_fixture.rs @@ -56,6 +56,16 @@ impl RecordedRequest { pub(super) type Recorder = Arc>>; +/// Checks the full request sequence, including the absence of extra probes. +pub(super) fn assert_requests(recorder: &Recorder, expected: &[(&str, &str)]) { + let recorded = recorder.lock().expect("recorder lock"); + let actual: Vec<_> = recorded + .iter() + .map(|request| (request.method.as_str(), request.target.as_str())) + .collect(); + assert_eq!(actual, expected, "unexpected native source request sequence"); +} + /// Binds a loopback listener that answers `responses` in order and returns its /// origin plus the recorder. The task ends once the script is exhausted. pub(super) async fn scripted_server(responses: Vec) -> (Url, Recorder) { From f17f31a3df6b541008fdac6e27c0113c2d7a0784 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 6 Sep 2026 14:13:09 +0800 Subject: [PATCH 10/20] feat(ilm): persist recovery controls for legacy tier journals (#7252) --- .../bucket/lifecycle/tier_delete_journal.rs | 192 ++++++++++++++++++ crates/ecstore/src/store/init.rs | 139 ++++++++++++- 2 files changed, 329 insertions(+), 2 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 2270bce3b..03b04a8a2 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -35,6 +35,10 @@ use crate::bucket::lifecycle::config_boundary; use crate::bucket::lifecycle::durable_namespace::{ TIER_DELETE_JOURNAL_NAMESPACE, TIER_DELETE_JOURNAL_V6_NAMESPACE, validate_durable_ilm_record, }; +use crate::bucket::lifecycle::recovery_control::{ + IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol, + load_recovery_control, observe_recovery_source, save_recovery_control_if_absent, +}; use crate::bucket::lifecycle::runtime_boundary; use crate::bucket::lifecycle::tier_sweeper::{ Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity, @@ -78,6 +82,13 @@ const TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY: usize = 32; const TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY: usize = 16; const TIER_DELETE_DISPATCH_CAS_CONCURRENCY: usize = 32; const TIER_DELETE_JOURNAL_VERSION: u8 = 2; +const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1"; +const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2"; +const TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-unknown"; +const TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS: &str = "tier_delete_journal_v1"; +const TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS: &str = "tier_delete_journal_v2"; +const TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS: &str = "tier_delete_journal_corrupt"; +const CORRUPT_TIER_DELETE_JOURNAL_IDENTITY: &str = "corrupt"; const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3; const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4; const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5; @@ -5509,6 +5520,125 @@ enum TierDeleteJournalEntryRecoveryOutcome { Failed, } +fn canonical_legacy_tier_delete_journal_identity(object_name: &str) -> Option<&str> { + let identity = object_name + .strip_prefix(TIER_DELETE_JOURNAL_LEGACY_PREFIX)? + .strip_suffix(".json")?; + (rustfs_utils::crypto::is_sha256_checksum(identity) + && !identity + .bytes() + .any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase())) + .then_some(identity) +} + +fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static str, &'static str)> { + match entry.persisted_version { + 1 => Some((TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS)), + TIER_DELETE_JOURNAL_VERSION => Some((TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS)), + _ => None, + } +} + +fn legacy_tier_delete_control_matches( + control: &IlmRecoveryControl, + identity: &IlmRecoveryControlIdentity, + generation: &crate::bucket::lifecycle::recovery_control::IlmRecoverySourceGeneration, + classification: IlmRecoveryClassification, + error_code: IlmRecoveryErrorCode, +) -> bool { + control.identity == *identity + && control.observed_source_generation == *generation + && control.classification == classification + && control.last_error_code == error_code + && control.owner.is_none() + && control.attempt_count == 0 + && control.consecutive_failure_count == 0 +} + +fn legacy_tier_delete_control_is_scheduler_fence(control: &IlmRecoveryControl, identity: &IlmRecoveryControlIdentity) -> bool { + control.identity == *identity && control.owner.is_none() && !control.classification.permits_automatic_attempt() +} + +async fn persist_legacy_tier_delete_recovery_control( + api: Arc, + object_name: &str, + observed_data: &[u8], + stable_operation_identity: String, + (source_schema, record_class): (&'static str, &'static str), + intended_classification: IlmRecoveryClassification, + intended_error_code: IlmRecoveryErrorCode, +) -> Result<()> { + let identity = IlmRecoveryControlIdentity { + protocol: IlmRecoveryProtocol::TierDeleteJournal, + canonical_source_path: object_name.to_string(), + stable_operation_identity, + record_class: record_class.to_string(), + }; + let control_id = identity.source_operation_digest().map_err(Error::other)?; + match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await { + Ok(observed) if legacy_tier_delete_control_is_scheduler_fence(&observed.control, &identity) => return Ok(()), + Ok(_) => return Err(Error::PreconditionFailed), + Err(Error::ConfigNotFound) => {} + Err(err) => return Err(err), + } + + let source = observe_recovery_source(api.clone(), object_name, source_schema).await?; + let exact_source = source.is_consistent() && source.canonical_data.as_deref() == Some(observed_data); + let (classification, error_code) = if exact_source { + (intended_classification, intended_error_code) + } else { + (IlmRecoveryClassification::Corrupt, IlmRecoveryErrorCode::SourceDivergent) + }; + let candidate = IlmRecoveryControl::new( + identity.clone(), + source.generation.clone(), + classification, + i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos()) + .map_err(|_| Error::other("tier delete journal recovery timestamp does not fit i64"))?, + error_code, + ) + .map_err(Error::other)?; + + match save_recovery_control_if_absent(api.clone(), &candidate).await { + Ok(()) | Err(Error::PreconditionFailed) => {} + Err(save_error) => match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await { + Ok(observed) + if legacy_tier_delete_control_matches( + &observed.control, + &identity, + &source.generation, + classification, + error_code, + ) => + { + return Ok(()); + } + Ok(_) | Err(_) => return Err(save_error), + }, + } + + let observed = load_recovery_control(api, IlmRecoveryProtocol::TierDeleteJournal, &control_id).await?; + if !legacy_tier_delete_control_matches(&observed.control, &identity, &source.generation, classification, error_code) { + return Err(Error::PreconditionFailed); + } + Ok(()) +} + +async fn retain_corrupt_legacy_tier_delete_journal(api: Arc, object_name: &str, data: &[u8]) -> Result<()> { + canonical_legacy_tier_delete_journal_identity(object_name) + .ok_or_else(|| Error::other("tier delete journal path is not canonical"))?; + persist_legacy_tier_delete_recovery_control( + api, + object_name, + data, + CORRUPT_TIER_DELETE_JOURNAL_IDENTITY.to_string(), + (TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS), + IlmRecoveryClassification::Corrupt, + IlmRecoveryErrorCode::SourceCorrupt, + ) + .await +} + async fn recover_tier_delete_journal_entry(api: Arc, object_name: String) -> TierDeleteJournalEntryRecoveryOutcome { let data = match config_boundary::read_config(api.clone(), &object_name).await { Ok(data) => data, @@ -5529,6 +5659,22 @@ async fn recover_tier_delete_journal_entry(api: Arc, object_name: Strin let je = match decode_tier_delete_journal_entry(&data) { Ok(je) => je, Err(err) => { + if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() { + return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await { + Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained, + Err(control_error) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + error = ?control_error, + "Failed to retain corrupt tier delete journal recovery control" + ); + TierDeleteJournalEntryRecoveryOutcome::Failed + } + }; + } warn!( event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, component = LOG_COMPONENT_ECSTORE, @@ -5542,6 +5688,22 @@ async fn recover_tier_delete_journal_entry(api: Arc, object_name: Strin }; if tier_delete_journal_object_name(&je) != object_name { + if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() { + return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await { + Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained, + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + error = ?err, + "Failed to retain mismatched tier delete journal recovery control" + ); + TierDeleteJournalEntryRecoveryOutcome::Failed + } + }; + } warn!( event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, component = LOG_COMPONENT_ECSTORE, @@ -5552,6 +5714,36 @@ async fn recover_tier_delete_journal_entry(api: Arc, object_name: Strin return TierDeleteJournalEntryRecoveryOutcome::Failed; } + if let Some((source_schema, record_class)) = legacy_tier_delete_recovery_descriptor(&je) { + let stable_operation_identity = canonical_legacy_tier_delete_journal_identity(&object_name) + .expect("decoded legacy journal path was validated against its canonical object name") + .to_string(); + return match persist_legacy_tier_delete_recovery_control( + api, + &object_name, + &data, + stable_operation_identity, + (source_schema, record_class), + IlmRecoveryClassification::RetainedAmbiguous, + IlmRecoveryErrorCode::RemoteVersionUnknown, + ) + .await + { + Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained, + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object_name, + error = ?err, + "Failed to retain legacy tier delete journal recovery control" + ); + TierDeleteJournalEntryRecoveryOutcome::Failed + } + }; + } + match api .durable_ilm_terminal_receipt_covers_active_source(&object_name, &data) .await diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 239cb20cc..86662f182 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -827,8 +827,8 @@ mod tests { }, recovery_control::{ IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, - IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, load_recovery_control, observe_recovery_source, - save_recovery_control_if_absent, + IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control, + observe_recovery_source, save_recovery_control_if_absent, }, tier_delete_journal::{ DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX, @@ -16760,6 +16760,141 @@ mod tests { } } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn legacy_tier_delete_journals_create_redacted_recovery_controls_without_remote_calls() { + let temp_dir = tempfile::tempdir().expect("create legacy journal recovery store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-tier-journal-recovery", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let tier_name = "LEGACY-RECOVERY"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("legacy recovery tier lease should resolve") + .backend_identity(); + let fixtures = [ + serde_json::json!({ + "version": 1, + "obj_name": "legacy/remote-v1", + "version_id": "opaque-v1", + "tier_name": tier_name, + }), + serde_json::json!({ + "version": 2, + "obj_name": "legacy/remote-v2", + "version_id": "opaque-v2", + "tier_name": tier_name, + "backend_identity": backend_identity, + }), + ]; + let mut journal_paths = Vec::new(); + for fixture in &fixtures { + let data = serde_json::to_vec(&fixture).expect("legacy journal fixture should encode"); + let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data) + .expect("legacy journal fixture should decode"); + let path = tier_delete_journal_object_name(&entry); + com::save_config(store.clone(), &path, data) + .await + .expect("legacy journal fixture should persist"); + journal_paths.push(path); + } + let corrupt_path = format!( + "{TIER_DELETE_JOURNAL_PREFIX}/{}.json", + rustfs_utils::crypto::hex_sha256(b"corrupt legacy tier journal", ToOwned::to_owned) + ); + com::save_config(store.clone(), &corrupt_path, b"{corrupt".to_vec()) + .await + .expect("corrupt legacy journal fixture should persist"); + + let (first, concurrent) = tokio::join!( + recover_tier_delete_journal_entries(store.clone(), 100, None), + recover_tier_delete_journal_entries(store.clone(), 100, None), + ); + for stats in [first, concurrent] { + let stats = stats.expect("concurrent legacy journal recovery scan should finish"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 0, 0)); + } + assert_eq!(tier_delete_journal_count(store.clone()).await, 3); + assert_eq!(backend.remove_count().await, 0, "legacy recovery must not call the remote tier"); + assert_eq!(backend.exact_remove_count(), 0, "legacy recovery must not issue exact remote DELETE"); + assert!(backend.op_log().await.is_empty(), "legacy recovery must not invoke any backend operation"); + + let mut first_controls = list_recovery_controls(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, None, 100, None) + .await + .expect("legacy recovery controls should be listable") + .records; + first_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id)); + assert_eq!(first_controls.len(), 3); + assert_eq!( + first_controls + .iter() + .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous) + .count(), + 2 + ); + assert_eq!( + first_controls + .iter() + .filter(|control| control.classification == IlmRecoveryClassification::Corrupt) + .count(), + 1 + ); + for view in &first_controls { + assert_eq!(view.protocol, IlmRecoveryProtocol::TierDeleteJournal); + assert_eq!(view.revision, 1); + assert_eq!(view.attempt_count, 0); + let encoded = serde_json::to_string(view).expect("recovery control view should encode"); + for secret in ["legacy/remote-v1", "legacy/remote-v2", "opaque-v1", "opaque-v2", tier_name] { + assert!(!encoded.contains(secret), "recovery control view must redact `{secret}`"); + } + let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &view.control_id) + .await + .expect("legacy recovery control should load"); + match view.source_schema.as_str() { + "rustfs-tier-delete-journal-v1" => { + assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v1"); + assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown); + } + "rustfs-tier-delete-journal-v2" => { + assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v2"); + assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown); + } + "rustfs-tier-delete-journal-unknown" => { + assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_corrupt"); + assert_eq!(view.last_error_code, IlmRecoveryErrorCode::SourceCorrupt); + } + schema => panic!("unexpected legacy recovery source schema: {schema}"), + } + } + + com::save_config( + store.clone(), + &journal_paths[0], + serde_json::to_vec_pretty(&fixtures[0]).expect("rewritten legacy journal fixture should encode"), + ) + .await + .expect("equivalent legacy journal rewrite should persist"); + let second = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("repeated legacy journal recovery scan should finish"); + assert_eq!((second.scanned, second.deleted, second.failed), (3, 0, 0)); + let mut second_controls = list_recovery_controls(store, IlmRecoveryProtocol::TierDeleteJournal, None, 100, None) + .await + .expect("repeated legacy recovery controls should remain listable") + .records; + second_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id)); + assert_eq!(second_controls, first_controls, "repeated scans must not reset durable controls"); + assert_eq!(backend.remove_count().await, 0, "repeated recovery must remain remote-call free"); + assert_eq!(backend.exact_remove_count(), 0); + assert!( + backend.op_log().await.is_empty(), + "repeated recovery must not invoke any backend operation" + ); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] From 92c17af8e37a0608c97ffad14ac051e4d0d10167 Mon Sep 17 00:00:00 2001 From: RustFS Date: Sun, 6 Sep 2026 14:13:21 +0800 Subject: [PATCH 11/20] ci(e2e): run distributed e2e on ubuntu-latest (#7253) --- .config/e2e-distributed-selection.txt | 2 +- .github/workflows/e2e-distributed.yml | 32 ++++++++++++++++++--------- docs/testing/distributed-e2e.md | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.config/e2e-distributed-selection.txt b/.config/e2e-distributed-selection.txt index 7291d5e03..d99f12976 100644 --- a/.config/e2e-distributed-selection.txt +++ b/.config/e2e-distributed-selection.txt @@ -1,2 +1,2 @@ -sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07 +sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2 sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07 diff --git a/.github/workflows/e2e-distributed.yml b/.github/workflows/e2e-distributed.yml index c3a5ca0e1..091d15c98 100644 --- a/.github/workflows/e2e-distributed.yml +++ b/.github/workflows/e2e-distributed.yml @@ -22,11 +22,13 @@ # Upgrade cases download the same pinned previous release as e2e-upgrade.yml. # # Isolated pool filesystems: expand/decommission/rebalance cases require -# independent `statfs` capacity. `sm-standard-4` is an ARC pod -# (`scripts/ci/check_runner_ephemerality.sh`) and usually has no -# `/dev/loop-control`, so `mount -o loop` fails with ENOENT ("mount failed: -# No such file or directory"). The prepare step therefore mounts four 1 GiB -# tmpfs instances and exports them as `RUSTFS_E2E_POOL_ROOTS`. +# independent `statfs` capacity. This job runs on GitHub-hosted +# `ubuntu-latest` because the self-hosted `sm-standard-4` ARC pods cannot +# create filesystems: `mount -o loop` fails with ENOENT (no +# `/dev/loop-control`), and `mount -t tmpfs` fails with "cannot mount tmpfs +# read-only" (no `CAP_SYS_ADMIN` in the initial namespace). The same reason +# `uring-integration` and `e2e-s3tests.yml` left that label. The prepare +# step mounts four 1 GiB tmpfs instances and exports `RUSTFS_E2E_POOL_ROOTS`. name: e2e-distributed @@ -76,7 +78,9 @@ concurrency: jobs: distributed: name: Distributed 4-node 4-disk e2e - runs-on: sm-standard-4 + # GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an + # ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only"). + runs-on: ubuntu-latest timeout-minutes: 180 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" @@ -97,7 +101,9 @@ jobs: uses: ./.github/actions/setup with: rust-version: stable - cache-shared-key: ci-e2e-distributed + # Dedicated key: ubuntu-latest and sm-standard-4 share runner.os, so + # a shared key would mix VM and ARC pod target/ artifacts. + cache-shared-key: ci-e2e-distributed-hosted cache-save-if: ${{ github.ref == 'refs/heads/main' }} install-build-packaging-tools: 'false' @@ -110,10 +116,14 @@ jobs: for pool in 0 1 2 3; do mountpoint="${mount_base}/pool-${pool}" mkdir -p "${mountpoint}" - # sm-standard-4 is an ARC pod without usable loop devices, so - # `mount -o loop` fails with ENOENT. Sized tmpfs still reports a - # distinct st_dev and independent 1G statfs capacity. - sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}" + # Sized tmpfs reports a distinct st_dev and independent 1G + # statfs capacity. Requires a VM runner (ubuntu-latest). + if ! sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"; then + echo "tmpfs mount failed on $(uname -a)" >&2 + findmnt || true + grep Cap /proc/self/status || true + exit 1 + fi sudo chmod 1777 "${mountpoint}" roots+=("${mountpoint}") done diff --git a/docs/testing/distributed-e2e.md b/docs/testing/distributed-e2e.md index 10d882f9b..5d997ddbf 100644 --- a/docs/testing/distributed-e2e.md +++ b/docs/testing/distributed-e2e.md @@ -17,7 +17,7 @@ A multi-pool layout in which any pool spans several localhost ports is not expre Data-movement cases fail closed. A decommission or rebalance test must observe a successful start response, an active state, a clean terminal state, non-zero movement counters, and post-operation object integrity. An unsupported response, HTTP 5xx, missing status fields, cleanup warning, or zero-progress terminal response fails the case; pre/post S3 availability alone is not evidence that movement ran. -The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated 1 GiB tmpfs filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use ext4 loop devices: the `sm-standard-4` ARC pods have no `/dev/loop-control`, so `mount -o loop` fails with `No such file or directory`. Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata. +The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job runs on GitHub-hosted `ubuntu-latest` and mounts four isolated 1 GiB tmpfs filesystems, then exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use the self-hosted `sm-standard-4` ARC pods: those cannot create filesystems (`mount -o loop` fails with `No such file or directory`, and `mount -t tmpfs` fails with `cannot mount tmpfs read-only`). Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata. The expansion fixture is an all-current-binary fleet, so it initializes pool metadata with the documented V3 write and fleet-confirmation gates. Decommission cases write their baseline objects, version history, and multipart data into pool 0 before adding pools 1–3, then retire pool 0. This makes a passing result evidence of user-data movement rather than merely an internal-metadata counter changing. From 51893abfbfc90b50aad61d9cde0cbf3e5ad0029b Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 6 Sep 2026 14:13:35 +0800 Subject: [PATCH 12/20] feat(heal): pace running admin work at safe boundaries (#7255) --- crates/heal/src/heal/erasure_healer.rs | 154 ++++++++++- crates/heal/src/heal/manager.rs | 8 +- crates/heal/src/heal/manager/scheduler.rs | 22 +- crates/heal/src/heal/manager/tests.rs | 2 + .../heal/manager/tests/running_mainline.rs | 249 ++++++++++++++++++ crates/heal/src/heal/mod.rs | 1 + crates/heal/src/heal/pacing.rs | 238 +++++++++++++++++ crates/heal/src/heal/task.rs | 56 ++-- crates/heal/src/heal/task/heal_bucket.rs | 3 + crates/heal/src/heal/task/heal_erasure_set.rs | 3 +- docs/operations/scanner-runtime-controls.md | 18 +- 11 files changed, 718 insertions(+), 36 deletions(-) create mode 100644 crates/heal/src/heal/manager/tests/running_mainline.rs create mode 100644 crates/heal/src/heal/pacing.rs diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index a6c288dfd..e3d4e9b53 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -108,6 +108,144 @@ pub struct ErasureSetHealer { target_endpoints: Arc<[String]>, replacement_task_id: Option, replacement_target_identities: Option>, + mainline_pacer: Option>, +} + +async fn acquire_page_permit( + semaphore: Arc, + pacer: Option<&super::pacing::MainlinePacer>, + cancel: &tokio_util::sync::CancellationToken, +) -> Result { + let acquire = || async { + tokio::select! { + biased; + _ = cancel.cancelled() => Err(Error::TaskCancelled), + permit = semaphore.clone().acquire_owned() => permit.map_err(|err| Error::other(format!("Failed to acquire page concurrency permit: {err}"))), + } + }; + let mut paid_pause = false; + loop { + let permit = acquire().await?; + if let Some(pacer) = pacer { + // Keep the real permit on the low-pressure path. Every acquisition + // gets a fresh decision, including a waiter that queued a second + // time. One completed pause is a bounded minimum-progress grant. + match pacer.admission_decision() { + super::pacing::PacingDecision::Wait(pressure) if !paid_pause => { + drop(permit); + paid_pause = pacer.wait_after_admission(cancel, pressure).await?; + continue; + } + _ => {} + } + } + return Ok(permit); + } +} + +#[cfg(test)] +mod mainline_pacing_tests { + use super::*; + use crate::heal::pacing::{MainlinePacer, TestPressure}; + use rustfs_concurrency::WorkloadClass; + use std::sync::atomic::Ordering; + use tokio_util::sync::CancellationToken; + + #[tokio::test(start_paused = true)] + async fn running_mainline_page_waiters_resample_after_capacity_and_release_permits() { + let semaphore = Arc::new(Semaphore::new(1)); + let occupied = semaphore + .clone() + .acquire_owned() + .await + .expect("existing object owns capacity"); + let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0)); + let pacer = Arc::new(MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250)).expect("pacer")); + let cancel = CancellationToken::new(); + let waiting = tokio::spawn({ + let semaphore = semaphore.clone(); + let pacer = pacer.clone(); + let cancel = cancel.clone(); + async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await } + }); + tokio::task::yield_now().await; + provider.active.store(100, Ordering::SeqCst); + drop(occupied); + provider.sampled.notified().await; + assert_eq!(semaphore.available_permits(), 1, "running pressure wait cannot retain the page permit"); + cancel.cancel(); + assert!(matches!(waiting.await.expect("page waiter"), Err(Error::TaskCancelled))); + assert_eq!(semaphore.available_permits(), 1); + let deadline = tokio::time::timeout( + Duration::from_millis(10), + acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()), + ) + .await; + assert!(deadline.is_err()); + assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits"); + let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new()) + .await + .expect("unpaced admission"); + assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap"); + drop(permit); + assert_eq!(semaphore.available_permits(), 1); + } + + #[tokio::test(start_paused = true)] + async fn running_mainline_two_page_waiters_check_pressure_at_final_admission() { + use std::task::Poll; + for raise_pressure in [false, true] { + let semaphore = Arc::new(Semaphore::new(1)); + let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters"); + let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0)); + let pause = Duration::from_millis(250); + let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause).expect("pacer"); + let cancel = CancellationToken::new(); + let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel)); + let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel)); + assert!(futures::poll!(first.as_mut()).is_pending()); + assert!(futures::poll!(second.as_mut()).is_pending()); + drop(occupied); + let first_ready = match futures::poll!(first.as_mut()) { + Poll::Ready(result) => Some(result.expect("first admission")), + Poll::Pending => None, + }; + assert!(futures::poll!(second.as_mut()).is_pending()); + let first_permit = match first_ready { + Some(permit) => permit, + None => tokio::time::timeout(Duration::from_millis(1), first) + .await + .expect("low-pressure waiters must not bounce capacity forever") + .expect("first permit"), + }; + // The first object owns real page capacity while its commit runs. + tokio::time::advance(Duration::from_millis(100)).await; + if raise_pressure { + provider.active.store(100, Ordering::SeqCst); + } + drop(first_permit); + let admitted = if raise_pressure { + assert!( + futures::poll!(second.as_mut()).is_pending(), + "a second acquisition cannot reuse the earlier low-pressure sample" + ); + assert_eq!(semaphore.available_permits(), 1, "pressure wait must release page capacity"); + tokio::time::advance(pause).await; + tokio::time::timeout(Duration::from_millis(1), second) + .await + .expect("sustained pressure must allow one unit after its bounded pause") + .expect("second permit") + } else { + tokio::time::timeout(Duration::from_millis(1), second) + .await + .expect("low pressure must make progress") + .expect("second permit") + }; + assert_eq!(semaphore.available_permits(), 0); + drop(admitted); + assert_eq!(semaphore.available_permits(), 1); + } + } } pub(crate) fn target_outcomes_complete(result: &HealResultItem, target_endpoints: &[String]) -> bool { @@ -219,9 +357,15 @@ impl ErasureSetHealer { target_endpoints: Vec::new().into(), replacement_task_id: None, replacement_target_identities: None, + mainline_pacer: None, } } + pub(crate) fn with_mainline_pacer(mut self, pacer: Option>) -> Self { + self.mainline_pacer = pacer; + self + } + pub(crate) fn with_replacement_targets( mut self, mut target_endpoints: Vec, @@ -856,6 +1000,9 @@ impl ErasureSetHealer { let include_lifecycle_object_info = lifecycle_expiry_context.is_some(); loop { + if let Some(pacer) = &self.mainline_pacer { + pacer.wait(&self.cancel_token).await?; + } self.verify_replacement_identity_fence("page scan").await?; // Get one page of object versions let (objects, next_token, is_truncated) = if use_disk_walk { @@ -1034,13 +1181,10 @@ impl ErasureSetHealer { let semaphore = semaphore.clone(); let target_endpoints = self.target_endpoints.clone(); let replacement_commit_evidence_required = self.replacement_task_id.is_some(); + let mainline_pacer = self.mainline_pacer.clone(); page_tasks.push(async move { - let permit = semaphore - .clone() - .acquire_owned() - .await - .map_err(|e| Error::other(format!("Failed to acquire page concurrency permit: {e}"))); + let permit = acquire_page_permit(semaphore, mainline_pacer.as_deref(), &cancel_token).await; let _permit = match permit { Ok(permit) => permit, diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index c90a18729..74ec6d546 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -602,13 +602,13 @@ pub struct HealConfig { pub set_bulkhead_enable: bool, /// Whether erasure-set page parallelism is enabled. pub page_parallel_enable: bool, - /// Whether foreground read pressure can delay best-effort heal task starts. + /// Whether foreground pressure delays best-effort starts and paces running admin work. pub mainline_throttle_enable: bool, - /// Foreground read permit utilization percentage that delays best-effort heal starts. + /// Foreground read utilization high watermark for start admission and admin pacing. pub mainline_read_utilization_high_percent: usize, - /// Foreground write utilization percentage that delays best-effort heal starts. + /// Foreground write utilization high watermark for start admission and admin pacing. pub mainline_write_utilization_high_percent: usize, - /// Delay before rechecking foreground pressure after delaying heal starts. + /// Start recheck interval; running admin pacing caps each holder's pause at one second. pub mainline_max_sleep: Duration, } diff --git a/crates/heal/src/heal/manager/scheduler.rs b/crates/heal/src/heal/manager/scheduler.rs index 704a7cbeb..5247c9ea7 100644 --- a/crates/heal/src/heal/manager/scheduler.rs +++ b/crates/heal/src/heal/manager/scheduler.rs @@ -175,11 +175,23 @@ impl HealManager { .unwrap_or_else(|poisoned| poisoned.into_inner()) .get(&request.id) .cloned(); - let task = Arc::new(HealTask::from_replacement_recovery_request( - request, - storage.clone(), - replacement_resume_endpoint, - )); + let mainline_pacer = if request.source == HealRequestSource::Admin && config.mainline_throttle_enable { + workload_provider.as_ref().and_then(|provider| { + crate::heal::pacing::MainlinePacer::new( + provider.clone(), + config.mainline_read_utilization_high_percent, + config.mainline_write_utilization_high_percent, + config.mainline_max_sleep, + ) + .map(Arc::new) + }) + } else { + None + }; + let task = Arc::new( + HealTask::from_replacement_recovery_request(request, storage.clone(), replacement_resume_endpoint) + .with_mainline_pacer(mainline_pacer), + ); let task_id = task.id.clone(); active_heals_guard.insert(task_id.clone(), task.clone()); publish_active_heal_count(&active_heals_guard); diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index 17feee1b5..364f594f0 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -26,6 +26,8 @@ use rustfs_madmin::heal_commands::HealResultItem; use std::sync::Mutex as StdMutex; use tempfile::TempDir; +mod running_mainline; + use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo}; #[tokio::test] diff --git a/crates/heal/src/heal/manager/tests/running_mainline.rs b/crates/heal/src/heal/manager/tests/running_mainline.rs new file mode 100644 index 000000000..4f21d7238 --- /dev/null +++ b/crates/heal/src/heal/manager/tests/running_mainline.rs @@ -0,0 +1,249 @@ +// Copyright 2026 RustFS Team +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crate::heal::storage::HealListItem; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use tokio::sync::Semaphore; + +#[derive(Default)] +struct PressureProbe { + active: AtomicUsize, + commit_open: AtomicBool, + high_sampled: Notify, +} + +impl WorkloadAdmissionSnapshotProvider for PressureProbe { + fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot { + assert!( + !self.commit_open.load(Ordering::SeqCst), + "pressure must not be sampled inside an object commit" + ); + let active = self.active.load(Ordering::SeqCst); + if active >= 80 { + self.high_sampled.notify_one(); + } + WorkloadAdmissionRegistrySnapshot::new(vec![ + WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Open).with_counts( + Some(active), + None, + Some(100), + ), + ]) + } +} + +struct RunningStorage { + provider: Arc, + namespace: Mutex<()>, + io: Arc, + first_started: Notify, + release_first: Notify, + first_finished: Notify, + second_finished: Notify, + started: AtomicUsize, + committed: AtomicUsize, +} + +#[async_trait::async_trait] +impl HealStorageAPI for RunningStorage { + async fn get_object_meta(&self, _: &str, _: &str) -> Result> { + Ok(None) + } + async fn ec_decode_rebuild(&self, _: &str, _: &str) -> Result> { + Ok(Vec::new()) + } + async fn get_bucket_info(&self, bucket: &str) -> Result> { + Ok(Some(BucketInfo { + name: bucket.into(), + ..Default::default() + })) + } + async fn list_buckets(&self) -> Result> { + Ok(Vec::new()) + } + async fn object_exists(&self, _: &str, _: &str) -> Result { + Ok(true) + } + async fn heal_bucket(&self, _: &str, _: &HealOpts) -> Result { + Ok(HealResultItem::default()) + } + async fn heal_format(&self, _: bool) -> Result<(HealResultItem, Option)> { + Ok((HealResultItem::default(), None)) + } + async fn get_disk_for_resume(&self, _: &str) -> Result { + Err(Error::other("no resume disk in bucket fixture")) + } + async fn list_objects_for_heal_page( + &self, + _: &str, + _: &str, + _: Option<&str>, + _: bool, + ) -> Result<(Vec, Option, bool)> { + Ok(( + ["a", "b"] + .into_iter() + .map(|name| HealListItem { + name: name.into(), + version_id: None, + mod_time_unix_nanos: None, + lifecycle_object_info: None, + is_delete_marker: false, + }) + .collect(), + None, + false, + )) + } + async fn heal_object(&self, _: &str, _: &str, _: Option<&str>, _: &HealOpts) -> Result<(HealResultItem, Option)> { + let permit = self.io.clone().acquire_owned().await.expect("fixture I/O permit"); + let namespace = self.namespace.lock().await; + self.provider.commit_open.store(true, Ordering::SeqCst); + let index = self.started.fetch_add(1, Ordering::SeqCst); + if index == 0 { + self.first_started.notify_one(); + self.release_first.notified().await; + } + self.committed.fetch_add(1, Ordering::SeqCst); + self.provider.commit_open.store(false, Ordering::SeqCst); + drop(namespace); + drop(permit); + if index == 0 { + self.first_finished.notify_one(); + } else { + self.second_finished.notify_one(); + } + Ok(( + HealResultItem { + object_size: 1, + ..Default::default() + }, + None, + )) + } +} + +async fn start_fixture( + provider_enabled: bool, + pacing_enabled: bool, + timeout: Duration, +) -> (HealManager, Arc, Arc, Arc) { + let provider = Arc::new(PressureProbe::default()); + let storage = Arc::new(RunningStorage { + provider: provider.clone(), + namespace: Mutex::new(()), + io: Arc::new(Semaphore::new(1)), + first_started: Notify::new(), + release_first: Notify::new(), + first_finished: Notify::new(), + second_finished: Notify::new(), + started: AtomicUsize::new(0), + committed: AtomicUsize::new(0), + }); + let manager = HealManager::new_with_workload_provider( + storage.clone(), + Some(HealConfig { + mainline_throttle_enable: pacing_enabled, + mainline_read_utilization_high_percent: 80, + mainline_write_utilization_high_percent: 80, + mainline_max_sleep: Duration::from_millis(250), + max_concurrent_heals: 1, + ..HealConfig::default() + }), + provider_enabled.then(|| provider.clone() as WorkloadSnapshotProviderRef), + ); + let mut request = bucket_request("running-mainline", HealPriority::High, HealRequestSource::Admin); + request.options.recursive = true; + request.options.timeout = Some(timeout); + let task_id = request.id.clone(); + manager.submit_heal_request(request).await.expect("queue admin heal"); + process_manager_queue_once(&manager).await; + storage.first_started.notified().await; + let task = manager + .active_heals + .lock() + .await + .get(&task_id) + .cloned() + .expect("running task"); + (manager, storage, provider, task) +} + +#[tokio::test(start_paused = true)] +async fn running_mainline_admin_resamples_after_commit_and_yields_without_io_guards() { + let (_manager, storage, provider, _task) = start_fixture(true, true, Duration::from_secs(60)).await; + provider.active.store(100, Ordering::SeqCst); + assert!(storage.provider.commit_open.load(Ordering::SeqCst)); + assert_eq!(storage.committed.load(Ordering::SeqCst), 0); + storage.release_first.notify_one(); + storage.first_finished.notified().await; + tokio::time::timeout(Duration::from_millis(1), provider.high_sampled.notified()) + .await + .expect("running admin heal must re-sample rising pressure before its next object"); + assert_eq!(storage.started.load(Ordering::SeqCst), 1); + assert_eq!( + storage.committed.load(Ordering::SeqCst), + 1, + "in-flight commit must finish despite pressure" + ); + assert_eq!(storage.io.available_permits(), 1, "pacing must release I/O permits"); + assert!(storage.namespace.try_lock().is_ok(), "pacing must not hold the namespace lock"); + tokio::time::advance(Duration::from_millis(250)).await; + storage.second_finished.notified().await; + assert_eq!( + storage.committed.load(Ordering::SeqCst), + 2, + "sustained pressure must still allow bounded maintenance progress" + ); +} + +#[tokio::test(start_paused = true)] +async fn running_mainline_missing_provider_or_disabled_pacing_preserves_progress() { + for (provider_enabled, pacing_enabled) in [(false, true), (true, false)] { + let (_manager, storage, provider, _task) = start_fixture(provider_enabled, pacing_enabled, Duration::from_secs(60)).await; + provider.active.store(100, Ordering::SeqCst); + let before = tokio::time::Instant::now(); + storage.release_first.notify_one(); + storage.second_finished.notified().await; + assert_eq!(storage.committed.load(Ordering::SeqCst), 2); + assert_eq!(tokio::time::Instant::now(), before); + assert_eq!(storage.io.available_permits(), 1); + } +} + +#[tokio::test(start_paused = true)] +async fn running_mainline_cancellation_and_deadline_leave_next_object_unstarted() { + for cancelled in [true, false] { + let (_manager, storage, provider, task) = start_fixture(true, true, Duration::from_millis(100)).await; + provider.active.store(100, Ordering::SeqCst); + storage.release_first.notify_one(); + provider.high_sampled.notified().await; + if cancelled { + task.cancel_token.cancel(); + } else { + tokio::time::advance(Duration::from_millis(100)).await; + } + tokio::time::timeout(Duration::from_secs(1), async { + while matches!(task.get_status().await, HealTaskStatus::Running) { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("pacing must not mask cancellation or timeout"); + assert_eq!(storage.started.load(Ordering::SeqCst), 1); + assert_eq!(storage.committed.load(Ordering::SeqCst), 1); + assert_eq!(storage.io.available_permits(), 1); + assert!(storage.namespace.try_lock().is_ok()); + let outcome = task.get_outcome().await; + assert_eq!(outcome.counters.processed, 1); + assert_eq!( + task.get_status().await, + if cancelled { + HealTaskStatus::Cancelled + } else { + HealTaskStatus::Timeout + } + ); + } +} diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index 5f17c8cd8..ce2058086 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -17,6 +17,7 @@ pub mod erasure_healer; pub mod manager; pub mod mrf_queue; pub mod outcome; +pub(crate) mod pacing; pub mod progress; pub(crate) mod replacement_readiness; pub mod resume; diff --git a/crates/heal/src/heal/pacing.rs b/crates/heal/src/heal/pacing.rs new file mode 100644 index 000000000..ff8e7bea7 --- /dev/null +++ b/crates/heal/src/heal/pacing.rs @@ -0,0 +1,238 @@ +// Copyright 2026 RustFS Team +// Licensed under the Apache License, Version 2.0. + +use crate::{Error, Result}; +use rustfs_concurrency::{ + WorkloadAdmissionSnapshotProvider, + workload::{ForegroundPressure, foreground_pressure}, +}; +use std::{sync::Arc, time::Duration}; +use tokio::{sync::Mutex, time::Instant}; +use tokio_util::sync::CancellationToken; + +#[derive(Default)] +struct PacingState { + throttled: bool, + low_since: Option, +} + +pub(crate) enum PacingDecision { + Ready, + Wait(Option), +} + +/// Cooperative pacing for one admin execution, not a storage admission permit. +pub(crate) struct MainlinePacer { + provider: Arc, + read_high: usize, + write_high: usize, + pause: Duration, + state: Mutex, +} + +impl MainlinePacer { + pub(crate) fn new( + provider: Arc, + read_high: usize, + write_high: usize, + pause: Duration, + ) -> Option { + if (read_high == 0 && write_high == 0) || pause.is_zero() { + return None; + } + Some(Self { + provider, + read_high: read_high.min(100), + write_high: write_high.min(100), + pause: pause.min(Duration::from_secs(1)), + state: Mutex::new(PacingState::default()), + }) + } + + /// Fresh, nonblocking decision while the caller owns actual page capacity. + /// A contended pacing latch is conservative, but never awaited here. + pub(crate) fn admission_decision(&self) -> PacingDecision { + let snapshot = self.provider.workload_admission_snapshot(); + let pressure = foreground_pressure(&snapshot, self.read_high, self.write_high); + if pressure.is_none() && self.state.try_lock().is_ok_and(|state| !state.throttled) { + PacingDecision::Ready + } else { + PacingDecision::Wait(pressure) + } + } + + /// Call only between storage operations, with no namespace lock or I/O + /// permit held. The pacing-only mutex serializes starts within this task; + /// each holder waits at most one pause so persistent pressure cannot stop + /// all maintenance progress. Cancellation also interrupts queued waiters. + pub(crate) async fn wait(&self, cancel: &CancellationToken) -> Result<()> { + self.wait_after_admission(cancel, None).await.map(|_| ()) + } + + /// Returns whether this unit paid a bounded pause. That grant permits one + /// unit even if pressure persists when page capacity becomes available. + pub(crate) async fn wait_after_admission( + &self, + cancel: &CancellationToken, + observed: Option, + ) -> Result { + let mut state = tokio::select! { + biased; + _ = cancel.cancelled() => return Err(Error::TaskCancelled), + state = self.state.lock() => state, + }; + if observed.is_some() { + state.throttled = true; + state.low_since = None; + } + let snapshot = self.provider.workload_admission_snapshot(); + let pressure = foreground_pressure(&snapshot, self.read_high, self.write_high); + if pressure.is_some() { + state.throttled = true; + state.low_since = None; + } else if state.throttled { + let low = |high: usize| if high == 0 { 0 } else { (high * 3 / 4).max(1) }; + if foreground_pressure(&snapshot, low(self.read_high), low(self.write_high)).is_none() { + let now = Instant::now(); + let since = state.low_since.get_or_insert(now); + if now.duration_since(*since) >= self.pause.saturating_mul(4) { + state.throttled = false; + state.low_since = None; + } + } else { + state.low_since = None; + } + } + if !state.throttled { + return Ok(false); + } + metrics::counter!( + "rustfs_heal_mainline_throttle_total", + "source" => "admin", + "result" => "delayed", + "reason" => pressure.or(observed).map_or("recovery_window", |pressure| pressure.reason()) + ) + .increment(1); + tokio::select! { + biased; + _ = cancel.cancelled() => Err(Error::TaskCancelled), + _ = tokio::time::sleep(self.pause) => Ok(true), + } + } +} + +#[cfg(test)] +pub(crate) struct TestPressure { + pub(crate) active: std::sync::atomic::AtomicUsize, + pub(crate) sampled: tokio::sync::Notify, + class: rustfs_concurrency::WorkloadClass, +} + +#[cfg(test)] +impl TestPressure { + pub(crate) fn new(class: rustfs_concurrency::WorkloadClass, active: usize) -> Self { + Self { + active: std::sync::atomic::AtomicUsize::new(active), + sampled: tokio::sync::Notify::new(), + class, + } + } +} + +#[cfg(test)] +impl WorkloadAdmissionSnapshotProvider for TestPressure { + fn workload_admission_snapshot(&self) -> rustfs_concurrency::WorkloadAdmissionRegistrySnapshot { + let active = self.active.load(std::sync::atomic::Ordering::SeqCst); + self.sampled.notify_one(); + rustfs_concurrency::WorkloadAdmissionRegistrySnapshot::new(vec![ + rustfs_concurrency::WorkloadAdmissionSnapshot::new(self.class, rustfs_concurrency::AdmissionState::Open).with_counts( + Some(active), + None, + Some(100), + ), + ]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_concurrency::WorkloadClass; + use std::sync::atomic::Ordering; + + #[tokio::test(start_paused = true)] + async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() { + for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] { + let provider = Arc::new(TestPressure::new(class, 0)); + let pause = Duration::from_millis(250); + let pacer = MainlinePacer::new( + provider.clone(), + if class == WorkloadClass::ForegroundRead { 40 } else { 0 }, + if class == WorkloadClass::ForegroundWrite { 40 } else { 0 }, + pause, + ) + .expect("enabled pacer"); + let cancel = CancellationToken::new(); + let now = Instant::now(); + pacer.wait(&cancel).await.expect("quiet work"); + assert_eq!(Instant::now(), now); + // Low watermark is 30 for the configured high watermark 40. + for utilization in [40, 29, 35, 29, 29, 29, 29] { + provider.active.store(utilization, Ordering::SeqCst); + let before = Instant::now(); + pacer.wait(&cancel).await.expect("bounded maintenance progress"); + assert_eq!(Instant::now() - before, pause); + } + let before = Instant::now(); + pacer.wait(&cancel).await.expect("stable low pressure restores unpaced work"); + assert_eq!(Instant::now(), before); + } + } + + #[tokio::test(start_paused = true)] + async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() { + let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100)); + assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1)).is_none()); + assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO).is_none()); + let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600)).expect("pacer"); + let before = Instant::now(); + pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause"); + assert_eq!(Instant::now() - before, Duration::from_secs(1)); + } + + #[tokio::test(start_paused = true)] + async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() { + let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100)); + let paced = Arc::new(MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1)).expect("pacer")); + let cancel_first = CancellationToken::new(); + let first = tokio::spawn({ + let paced = paced.clone(); + let cancel = cancel_first.clone(); + async move { paced.wait(&cancel).await } + }); + provider.sampled.notified().await; + let cancel_second = CancellationToken::new(); + let second = tokio::spawn({ + let paced = paced.clone(); + let cancel = cancel_second.clone(); + async move { paced.wait(&cancel).await } + }); + tokio::task::yield_now().await; + cancel_second.cancel(); + assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled))); + provider.active.store(0, Ordering::SeqCst); + let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1)).expect("independent task"); + let before = Instant::now(); + other_task + .wait(&CancellationToken::new()) + .await + .expect("another task has no inherited latch"); + assert_eq!(Instant::now(), before, "task/set pacing state must not be global"); + cancel_first.cancel(); + assert!(matches!(first.await.expect("sleeping waiter"), Err(Error::TaskCancelled))); + tokio::time::timeout(Duration::from_secs(2), paced.wait(&CancellationToken::new())) + .await + .expect("pacing lock released") + .expect("bounded work after cancellation"); + } +} diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index a1b09c3b7..f1707be24 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -446,6 +446,7 @@ pub struct HealTask { pub cancel_token: tokio_util::sync::CancellationToken, /// Storage layer interface pub storage: Arc, + mainline_pacer: Option>, } impl HealTask { @@ -493,6 +494,7 @@ impl HealTask { task_start_instant: Arc::new(RwLock::new(None)), cancel_token: tokio_util::sync::CancellationToken::new(), storage, + mainline_pacer: None, } } @@ -529,6 +531,18 @@ impl HealTask { task } + pub(crate) fn with_mainline_pacer(mut self, pacer: Option>) -> Self { + self.mainline_pacer = pacer; + self + } + + async fn pace_mainline(&self) -> Result<()> { + if let Some(pacer) = &self.mainline_pacer { + self.await_with_control(pacer.wait(&self.cancel_token)).await?; + } + Ok(()) + } + pub fn metric_type_label(&self) -> &'static str { self.heal_type.kind_label() } @@ -933,24 +947,32 @@ impl HealTask { }); self.emit_trace_task_state("started", Duration::ZERO, None); - let result = match &self.heal_type { - HealType::Cluster => self.heal_cluster().await, - HealType::Object { - bucket, - object, - version_id, - } => self.heal_object(bucket, object, version_id.as_deref()).await, - HealType::Bucket { bucket } => self.heal_bucket(bucket).await, - HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await, + let result = async { + if self.heal_type.is_per_object() { + self.pace_mainline().await?; + } + match &self.heal_type { + HealType::Cluster => self.heal_cluster().await, + HealType::Object { + bucket, + object, + version_id, + } => self.heal_object(bucket, object, version_id.as_deref()).await, + HealType::Bucket { bucket } => self.heal_bucket(bucket).await, + HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await, - HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await, - HealType::ECDecode { - bucket, - object, - version_id, - } => self.heal_ec_decode(bucket, object, version_id.as_deref()).await, - HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await, - }; + HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await, + HealType::ECDecode { + bucket, + object, + version_id, + } => self.heal_ec_decode(bucket, object, version_id.as_deref()).await, + HealType::ErasureSet { buckets, set_disk_id } => { + self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await + } + } + } + .await; #[cfg(test)] pause_outcome_finish(&self.id).await; diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index c6225c844..cb8d230b4 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -34,6 +34,7 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio impl HealTask { pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> { + self.pace_mainline().await?; debug!( target: "rustfs::heal::task", event = EVENT_HEAL_BUCKET_STAGE, @@ -308,6 +309,7 @@ impl HealTask { self.check_control_flags().await?; let mut listing_attempt = 0; let (objects, next_token, is_truncated) = loop { + self.pace_mainline().await?; let page = if let Some(set_disk_id) = set_disk_id.as_deref() { self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk( set_disk_id, @@ -362,6 +364,7 @@ impl HealTask { let mut retry = Vec::with_capacity(pending.len()); for item in pending { self.check_control_flags().await?; + self.pace_mainline().await?; let mut telemetry_unknown = false; let object = item.name.as_str(); let identity = diff --git a/crates/heal/src/heal/task/heal_erasure_set.rs b/crates/heal/src/heal/task/heal_erasure_set.rs index cc1d638ba..446cb07d7 100644 --- a/crates/heal/src/heal/task/heal_erasure_set.rs +++ b/crates/heal/src/heal/task/heal_erasure_set.rs @@ -422,7 +422,8 @@ impl HealTask { self.source, ) .with_replacement_targets(self.heal_endpoints.clone(), is_auto_replacement.then(|| self.id.clone())) - .with_replacement_identity_fence(replacement_target_identities.clone()); + .with_replacement_identity_fence(replacement_target_identities.clone()) + .with_mainline_pacer(self.mainline_pacer.clone()); { let mut progress = self.progress.write().await; diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 31fe6124c..55e40cc31 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -280,10 +280,10 @@ Heal knobs are environment-only and read by `HealConfig::default` (`crates/heal/ | `RUSTFS_HEAL_SET_BULKHEAD_ENABLE` | `true` (`DEFAULT_HEAL_SET_BULKHEAD_ENABLE`) | Per-set bulkhead scheduling. | | `RUSTFS_HEAL_PAGE_PARALLEL_ENABLE` | `true` (`DEFAULT_HEAL_PAGE_PARALLEL_ENABLE`) | Page-level parallel object healing during erasure-set repair. | | `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY` | `8` (`DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY`) | Concurrent object heals within one erasure-set page. Forced to `1` when page parallelism is off, for `Deep` scan mode, and for `AutoHeal`-sourced requests (`ErasureSetHealer::effective_heal_page_object_concurrency_for_source`). | -| `RUSTFS_HEAL_MAINLINE_THROTTLE_ENABLE` | `true` (`DEFAULT_HEAL_MAINLINE_THROTTLE_ENABLE`) | Pause best-effort heal task starts while foreground I/O is saturated. | -| `RUSTFS_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT`, capped at 100) | Foreground read-permit utilization at which heal starts pause. | -| `RUSTFS_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT`, capped at 100) | Foreground write utilization at which heal starts pause. | -| `RUSTFS_HEAL_MAINLINE_MAX_SLEEP_MS` | `250` (`DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS`) | Recheck delay after deferring heal starts for foreground pressure. | +| `RUSTFS_HEAL_MAINLINE_THROTTLE_ENABLE` | `true` (`DEFAULT_HEAL_MAINLINE_THROTTLE_ENABLE`) | Defer best-effort starts and cooperatively pace running admin heal at safe work boundaries. | +| `RUSTFS_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT`, capped at 100) | Read-utilization high watermark for start admission and running admin pacing; zero disables this class. | +| `RUSTFS_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT`, capped at 100) | Write-utilization high watermark for start admission and running admin pacing; zero disables this class. | +| `RUSTFS_HEAL_MAINLINE_MAX_SLEEP_MS` | `250` (`DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS`) | Start recheck interval; running admin waits cap each pacing-gate holder at 1000 ms. Zero disables running pacing. | | `RUSTFS_HEAL_OVERLAP_POLICY` | `merge` (`DEFAULT_HEAL_OVERLAP_POLICY`) | `merge` dedups an admin heal start that overlaps a running or queued heal; `minio_error` returns a typed already-running / overlapping-paths rejection like madmin. | | `RUSTFS_HEAL_MRF_ENABLE` | `true` (`DEFAULT_HEAL_MRF_ENABLE`) | MRF intent pipeline: error paths deliver repair intents to the heal runtime and unconsumed intents replay from the durable journal after restart. | | `RUSTFS_HEAL_MRF_QUEUE_SIZE` | `100000` (`DEFAULT_HEAL_MRF_QUEUE_SIZE`) | MRF in-memory queue capacity. | @@ -291,6 +291,16 @@ Heal knobs are environment-only and read by `HealConfig::default` (`crates/heal/ | `RUSTFS_HEAL_MRF_REPLAY_BATCH` | `256` (`DEFAULT_HEAL_MRF_REPLAY_BATCH`) | Intents per replay push round. | | `RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS` | `3600` (`DEFAULT_HEAL_DANGLING_DELETE_GRACE_SECS`, `crates/ecstore/src/set_disk/core/io_primitives.rs`) | A recently modified object is never deleted as dangling inside this window; `0` disables the grace window. | +### Running admin heal pacing + +The manager passes its existing workload provider and a configuration snapshot into each admin execution. Bucket/prefix listing and object boundaries resample foreground pressure; erasure-set page workers also resample after earlier work releases page capacity. `High`, `Urgent`, and `force_start` do not exempt ordinary admin execution from this runtime pacing. The existing start-time bypass and overlap-control meanings are unchanged. + +Each execution has its own pacing latch, with no new global manager or cross-set pacing lock. The low watermark for each enabled class is `max(1, floor(high * 3 / 4))`: the default high watermark 80 therefore recovers below 60. High pressure latches pacing, and intermediate pressure resets the recovery window. Unpaced starts resume after sampled pressure remains below the low watermarks for four pause intervals, normally one second. While pressure persists, a pacing-gate holder waits only one interval, at most one second, then permits maintenance to continue. Concurrent page waiters serialize through this task-local gate; queue waiting still counts against the existing task execution timeout. + +The pacing gate holds neither namespace locks nor I/O/page permits while sleeping. At final page admission, each real permit acquisition gets a fresh, nonblocking pressure decision. Low-pressure work keeps that permit; only a unit that needs a pause releases capacity to wait. A unit that has completed one bounded pause may proceed despite persistent pressure, which supplies minimum maintenance progress without an endless acquire/pause loop. Existing object operations and commit tails are not interrupted because pressure rose. Cancellation and deadlines remain interruptible, and disabling pacing cannot bypass the global, per-set or page-concurrency hard caps. The existing `RUSTFS_HEAL_MAINLINE_THROTTLE_ENABLE=false` setting is the operational opt-out for newly created executions; no additional request override is introduced. + +A missing provider, zero pause, or both class thresholds set to zero preserves unpaced execution. Missing counts follow the existing shared pressure interpreter; they are observations, not health, quorum or resource-ownership proof. The current provider exposes node-level workload classes, so this does not claim independent per-set foreground measurements or a hard global resource budget. Runtime waits increment `rustfs_heal_mainline_throttle_total` with `source=admin`, `result=delayed`, and a foreground-pressure or `recovery_window` reason. Real p99/throughput protection requires the separate W20 fixed-load ABBA measurements. + ## Deliberate non-parity with MinIO These differences from MinIO are design decisions, recorded so they are not re-filed as gaps. From 395ba797fc4016696d792431bef71faba09af456 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 6 Sep 2026 14:13:47 +0800 Subject: [PATCH 13/20] fix(admin): bound peer probe retries to one round deadline (#7257) --- crates/config/README.md | 8 ++ crates/config/src/constants/internode.rs | 11 ++ .../src/cluster/rpc/peer_rest_client.rs | 15 ++- .../ecstore/src/services/notification_sys.rs | 119 ++++++++++++++++-- docs/operations/admin-peer-probe-timeout.md | 27 ++++ 5 files changed, 166 insertions(+), 14 deletions(-) create mode 100644 docs/operations/admin-peer-probe-timeout.md diff --git a/crates/config/README.md b/crates/config/README.md index 2313a41ec..b5db390c6 100644 --- a/crates/config/README.md +++ b/crates/config/README.md @@ -172,6 +172,14 @@ Drive timeout profile preset: - Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback. - Then the profile-derived default (`default` or `high_latency`). +## Admin peer probe timeout + +- `RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS` + - total per-peer budget for the `server_info`/`storage_info` admin probe round; `server_info` may reconnect once and `storage_info` remains a single attempt. + - default is `10` seconds, preserving the previous two-attempt worst-case budget. + - values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`. + - the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout. + ## Startup filesystem boundary policy - `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary. diff --git a/crates/config/src/constants/internode.rs b/crates/config/src/constants/internode.rs index 4a7910632..9fb78ebe2 100644 --- a/crates/config/src/constants/internode.rs +++ b/crates/config/src/constants/internode.rs @@ -39,6 +39,15 @@ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 20; pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS"; pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 30; +/// Total budget for one admin peer probe round, including any reconnect retry. +/// +/// This is intentionally separate from the transport-level RPC timeout: admin +/// probes may retry once, but the retry must consume the same round budget. +pub const ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS: &str = "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS"; +pub const DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 10; +pub const MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 60; +const _: () = assert!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS <= MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS); + // ── Client-side internode gRPC channel tuning (P0) ── // These mirror the server-side HTTP/2 transport tuning in `rustfs/src/server/http.rs` // on the *client* `tonic` `Endpoint` used for internode control-plane RPCs. Prior to @@ -312,6 +321,7 @@ mod tests { assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5); assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 20); assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 30); + assert_eq!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS, 10); assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy"); } @@ -412,6 +422,7 @@ mod tests { "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS" ); assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS"); + assert_eq!(ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS"); assert_eq!(ENV_INTERNODE_HTTP_TUNING_PROFILE, "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE"); assert_eq!(ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST, "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST"); assert_eq!(ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS, "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS"); diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 9d57c4647..c6a15a6b9 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -68,7 +68,10 @@ use std::{ }, time::SystemTime, }; -use tokio::{net::TcpStream, time::Duration}; +use tokio::{ + net::TcpStream, + time::{Duration, timeout}, +}; use tonic::Request; use tonic::service::interceptor::InterceptedService; use tracing::{debug, info, warn}; @@ -874,6 +877,16 @@ impl PeerRestClient { self.offline.store(false, Ordering::Release); } + /// Prepare a retry without allowing connection-cache cleanup to extend the + /// caller's probe deadline. The offline gate is cleared even when eviction + /// times out so a cancelled cleanup cannot strand the peer in fast-fail + /// mode; a later request can perform a fresh eviction if needed. + pub async fn prepare_retry_with_timeout(&self, timeout_duration: Duration) -> bool { + let evicted = timeout(timeout_duration, self.evict_connection()).await.is_ok(); + self.offline.store(false, Ordering::Release); + evicted + } + /// Whether this failure means the peer is unreachable, so it should be /// gated offline and its connection evicted. /// diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index 04b84a18d..d1955a397 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -72,6 +72,29 @@ const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4; /// service must not advertise this version until the conditional writer from /// rustfs/backlog#684 is available. const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5; + +fn resolve_admin_peer_probe_timeout_secs(configured: Option) -> u64 { + configured + .filter(|seconds| *seconds > 0) + .unwrap_or(rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS) + .min(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS) +} + +fn admin_peer_probe_timeout() -> Duration { + let configured = rustfs_utils::get_env_opt_u64_with_aliases(rustfs_config::ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, &[]); + let seconds = resolve_admin_peer_probe_timeout_secs(configured); + Duration::from_secs(seconds) +} + +fn remaining_admin_peer_probe_timeout(deadline: Instant) -> Option { + remaining_admin_peer_probe_timeout_at(deadline, Instant::now()) +} + +fn remaining_admin_peer_probe_timeout_at(deadline: Instant, now: Instant) -> Option { + let remaining = deadline.saturating_duration_since(now); + (!remaining.is_zero()).then_some(remaining) +} + type CrossPoolFencePolicyResult = Result>; fn cross_pool_fence_policy_results( @@ -1538,7 +1561,7 @@ impl NotificationSys { { let mut futures = Vec::with_capacity(self.peer_clients.len()); let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into()); - let peer_timeout = Duration::from_secs(5); + let peer_timeout = admin_peer_probe_timeout(); for (idx, client) in self.peer_clients.iter().enumerate() { let endpoints = endpoints.clone(); @@ -1546,7 +1569,9 @@ impl NotificationSys { futures.push(async move { if let Some(client) = client { let host = client.host.to_string(); - match timeout(peer_timeout, client.local_storage_info()).await { + let deadline = Instant::now() + peer_timeout; + let probe_timeout = remaining_admin_peer_probe_timeout(deadline).unwrap_or_default(); + match timeout(probe_timeout, client.local_storage_info()).await { Ok(Ok(mut info)) => { normalize_and_cache_peer_storage_info(cache, &host, &mut info); Some(info) @@ -1557,7 +1582,6 @@ impl NotificationSys { } Err(_) => { warn!("peer {} storage_info timed out after {:?}", host, peer_timeout); - client.evict_connection().await; handle_peer_failure(cache, &host, &endpoints) } } @@ -1583,7 +1607,7 @@ impl NotificationSys { pub async fn server_info(&self) -> Vec { let mut futures = Vec::with_capacity(self.peer_clients.len()); let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into()); - let peer_timeout = Duration::from_secs(5); + let peer_timeout = admin_peer_probe_timeout(); for (idx, client) in self.peer_clients.iter().enumerate() { let host = self @@ -1600,12 +1624,23 @@ impl NotificationSys { }; }; + let deadline = Instant::now() + peer_timeout; + let Some(first_timeout) = remaining_admin_peer_probe_timeout(deadline) else { + let health = peer_disk_health_with_deadline(&host, deadline).await; + return PeerServerInfoProbe { + host, + result: Err(PeerServerInfoProbeFailure::Rpc { health }), + }; + }; + // First attempt. A single evicted or half-open internode channel // is enough to fail one probe and, before retrying, would drop - // the member to unknown/offline for this whole snapshot. So on any - // first-attempt failure we evict the channel and re-dial once - // before falling back (rustfs/backlog#1049, P1-B). - match timeout(peer_timeout, client.server_info()).await { + // the member to unknown/offline for this whole snapshot. On a + // quick failure we evict the channel and re-dial once before + // falling back (rustfs/backlog#1049, P1-B). A slow attempt + // consumes the round budget and therefore does not trigger a + // second full wait or an asynchronous eviction side effect. + match timeout(first_timeout, client.server_info()).await { Ok(Ok(info)) => { return PeerServerInfoProbe { host, result: Ok(info) }; } @@ -1619,14 +1654,37 @@ impl NotificationSys { // `evict_connection` would leave that gate up and the retry would // fast-fail with "temporarily offline" instead of reconnecting // (rustfs/backlog#1049 P1-B). - client.prepare_retry().await; + let Some(retry_budget) = remaining_admin_peer_probe_timeout(deadline) else { + let health = peer_disk_health_with_deadline(&host, deadline).await; + return PeerServerInfoProbe { + host, + result: Err(PeerServerInfoProbeFailure::Rpc { health }), + }; + }; + // Bound connection-cache cleanup too. The helper clears the offline gate even + // when eviction itself times out, so cancellation cannot strand this peer in + // fast-fail mode. + if !client.prepare_retry_with_timeout(retry_budget).await { + let health = peer_disk_health_with_deadline(&host, deadline).await; + return PeerServerInfoProbe { + host, + result: Err(PeerServerInfoProbeFailure::Rpc { health }), + }; + } // Second and final attempt on the fresh channel. - match timeout(peer_timeout, client.server_info()).await { + let Some(retry_timeout) = remaining_admin_peer_probe_timeout(deadline) else { + let health = peer_disk_health_with_deadline(&host, deadline).await; + return PeerServerInfoProbe { + host, + result: Err(PeerServerInfoProbeFailure::Rpc { health }), + }; + }; + match timeout(retry_timeout, client.server_info()).await { Ok(Ok(info)) => PeerServerInfoProbe { host, result: Ok(info) }, Ok(Err(err)) => { warn!("peer {host} server_info failed after retry: {err}"); - let health = peer_disk_health(&host).await; + let health = peer_disk_health_with_deadline(&host, deadline).await; PeerServerInfoProbe { host, result: Err(PeerServerInfoProbeFailure::Rpc { health }), @@ -1634,8 +1692,7 @@ impl NotificationSys { } Err(_) => { warn!("peer {host} server_info timed out after retry ({peer_timeout:?})"); - client.evict_connection().await; - let health = peer_disk_health(&host).await; + let health = peer_disk_health_with_deadline(&host, deadline).await; PeerServerInfoProbe { host, result: Err(PeerServerInfoProbeFailure::Rpc { health }), @@ -3023,6 +3080,11 @@ async fn peer_disk_health(host: &str) -> Option { } } +async fn peer_disk_health_with_deadline(host: &str, deadline: Instant) -> Option { + let remaining = remaining_admin_peer_probe_timeout(deadline)?; + timeout(remaining, peer_disk_health(host)).await.ok().flatten() +} + /// Handle a peer failure for server_info: return cached data if available, or /// classify the member as `unknown` / `degraded` / `offline` depending on how /// many consecutive probes have failed and whether the peer's drives are still @@ -4017,6 +4079,37 @@ mod tests { } } + #[test] + fn admin_peer_probe_timeout_rejects_zero_and_caps_large_values() { + assert_eq!( + resolve_admin_peer_probe_timeout_secs(None), + rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS + ); + assert_eq!( + resolve_admin_peer_probe_timeout_secs(Some(0)), + rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS + ); + assert_eq!( + resolve_admin_peer_probe_timeout_secs(Some(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS + 1)), + rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS + ); + assert_eq!(resolve_admin_peer_probe_timeout_secs(Some(7)), 7); + } + + #[tokio::test] + async fn admin_peer_probe_health_fallback_respects_expired_deadline() { + let deadline = Instant::now(); + assert!(peer_disk_health_with_deadline("peer-1", deadline).await.is_none()); + } + + #[test] + fn admin_peer_probe_deadline_is_shared_across_attempts() { + let start = Instant::now(); + let deadline = start + Duration::from_secs(10); + assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(6)).is_some()); + assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(10)).is_none()); + } + #[tokio::test] async fn call_peer_with_timeout_returns_value_when_fast() { let result = call_peer_with_timeout( diff --git a/docs/operations/admin-peer-probe-timeout.md b/docs/operations/admin-peer-probe-timeout.md new file mode 100644 index 000000000..b7b0e0d8b --- /dev/null +++ b/docs/operations/admin-peer-probe-timeout.md @@ -0,0 +1,27 @@ +# Admin peer probe timeout + +RustFS admin server information and storage information aggregate read-only +state from remote peers. A peer may answer the RPC while its local disk +diagnostic is still recovering after a restart or outage, so these probes use a +bounded per-peer round budget. + +## Configuration + +| Environment variable | Default | Accepted range | Behavior | +| --- | ---: | ---: | --- | +| `RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS` | `10` seconds | `1..=60` seconds | Total budget for one peer probe round; `server_info` may reconnect once, while `storage_info` remains a single attempt. | + +`0` and invalid values fall back to the default. Values above `60` are clamped +to `60`. The timeout is read by the node aggregating the admin response; it is +not a wire or mixed-version protocol setting. + +Any retry shares the same per-peer deadline. A fast transport failure can still +trigger the existing reconnect retry, but a slow first attempt consumes the +remaining budget and cannot add another full timeout. Configure this value +with margin below any external health-check deadline (for example, a +keepalived script timeout); the default preserves the previous two-attempt +worst-case budget and may need to be lowered for a tighter watchdog. + +This setting does not change `RUSTFS_INTERNODE_RPC_TIMEOUT_SECS` or the drive +health policy. A disk probe timeout can still update drive health according to +`RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION`. From 1dddf357cda5ba5072068d8532895665936fb231 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 6 Sep 2026 14:14:07 +0800 Subject: [PATCH 14/20] test(scanner): add bounded cache cost microprofile (#7261) --- crates/scanner/src/data_usage_define/tests.rs | 2 + .../src/data_usage_define/tests/cache_cost.rs | 346 ++++++++++++++++++ docs/testing/README.md | 2 + docs/testing/scanner-cache-cost.md | 36 ++ 4 files changed, 386 insertions(+) create mode 100644 crates/scanner/src/data_usage_define/tests/cache_cost.rs create mode 100644 docs/testing/scanner-cache-cost.md diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index 7f6dfd30d..2f8c98723 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -29,6 +29,8 @@ use tokio::sync::Mutex; const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([3; 32]); +mod cache_cost; + #[test] fn scoped_scan_coverage_metadata_preserves_map_compatibility() { #[derive(serde::Deserialize)] diff --git a/crates/scanner/src/data_usage_define/tests/cache_cost.rs b/crates/scanner/src/data_usage_define/tests/cache_cost.rs new file mode 100644 index 000000000..cc1990bbf --- /dev/null +++ b/crates/scanner/src/data_usage_define/tests/cache_cost.rs @@ -0,0 +1,346 @@ +// Copyright 2026 RustFS Team +// Licensed under the Apache License, Version 2.0. + +use super::*; +use std::hint::black_box; +use std::sync::atomic::AtomicU64; +use std::time::Instant as WallInstant; + +const MAX_WIRE_BYTES: u64 = 32 * 1024 * 1024; +const CACHE_NAME: &str = "bucket/cache-cost.bin"; + +/// Two bounded memory slots model revision preconditions and count the bytes +/// consumed by the real save entry point, not disk writes or fsync latency. +#[derive(Debug, Default)] +struct CountingStore { + slots: Mutex<[(u64, Vec); 2]>, + puts: AtomicU64, + bytes: AtomicU64, + ingest_ns: AtomicU64, +} + +impl CountingStore { + fn slot(object: &str) -> usize { + let main = path_join_buf(&[BUCKET_META_PREFIX, CACHE_NAME]); + if object == main { + 0 + } else { + assert_eq!(object, format!("{main}.bkp"), "only two fixture cache paths are permitted"); + 1 + } + } + + fn reset_counts(&self) { + self.puts.store(0, Ordering::Relaxed); + self.bytes.store(0, Ordering::Relaxed); + self.ingest_ns.store(0, Ordering::Relaxed); + } +} + +#[async_trait::async_trait] +impl ObjectIO for CountingStore { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = ScannerGetObjectReader; + type PutObjectReader = ScannerPutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _headers: Self::HeaderMap, + _options: &Self::ObjectOptions, + ) -> StorageResult { + // The real loader may probe the legacy metadata bucket on a miss. + if bucket != RUSTFS_META_BUCKET { + return Err(Error::FileNotFound); + } + let slots = self.slots.lock().await; + let (revision, bytes) = &slots[Self::slot(object)]; + if *revision == 0 { + return Err(Error::FileNotFound); + } + Ok(CacheReadStore::reader(CacheReadBody::Bytes(bytes.clone()), &revision.to_string())) + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut Self::PutObjectReader, + options: &Self::ObjectOptions, + ) -> StorageResult { + assert_eq!(bucket, RUSTFS_META_BUCKET); + let started = WallInstant::now(); + let mut bytes = Vec::new(); + (&mut data.stream).take(MAX_WIRE_BYTES + 1).read_to_end(&mut bytes).await?; + assert!(u64::try_from(bytes.len()).expect("wire length") <= MAX_WIRE_BYTES); + let mut slots = self.slots.lock().await; + let (revision, stored) = &mut slots[Self::slot(object)]; + let preconditions = options.http_preconditions.as_ref().expect("profile saves must use CAS"); + let expected = revision.to_string(); + if (*revision == 0 && preconditions.if_none_match_value() != Some("*")) + || (*revision != 0 && preconditions.if_match_value() != Some(expected.as_str())) + { + return Err(Error::PreconditionFailed); + } + self.bytes + .fetch_add(u64::try_from(bytes.len()).expect("save length"), Ordering::Relaxed); + self.puts.fetch_add(1, Ordering::Relaxed); + *stored = bytes; + *revision += 1; + self.ingest_ns.fetch_add(elapsed_ns(started), Ordering::Relaxed); + Ok(ObjectInfo { + etag: Some(revision.to_string()), + ..Default::default() + }) + } +} + +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for CountingStore { + async fn delete_config_object( + &self, + _bucket: &str, + _object: &str, + _options: crate::ScannerObjectOptions, + ) -> crate::EcstoreResult { + Err(Error::NotImplemented) + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) + } +} + +fn elapsed_ns(started: WallInstant) -> u64 { + u64::try_from(started.elapsed().as_nanos()).expect("bounded profile duration") +} + +fn fixture(objects: usize) -> DataUsageCache { + assert!((1..=16384).contains(&objects)); + let mut cache = DataUsageCache::default(); + cache.info.name = "bucket".to_string(); + cache.info.snapshot_complete = true; + cache.replace("bucket", "", DataUsageEntry::default()); + for index in 0..objects { + cache.replace( + &format!("bucket/object-{index:05}"), + "bucket", + DataUsageEntry { + objects: 1, + versions: 2, + size: 4096, + ..Default::default() + }, + ); + } + cache +} + +fn canonical_cache_value(mut value: Value) -> Value { + for entry in value["cache"].as_object_mut().expect("cache entry map").values_mut() { + let children = entry["children"].as_array_mut().expect("entry children set"); + // Sort only the set representation. Do not deduplicate or reorder + // histograms and other arrays whose element positions carry meaning. + children.sort_unstable_by(|left, right| { + left.as_str() + .expect("child key string") + .cmp(right.as_str().expect("child key string")) + }); + } + value +} + +fn same_cache(actual: &DataUsageCache, expected: &DataUsageCache) { + assert_eq!( + canonical_cache_value(serde_json::to_value(actual).expect("actual cache structure")), + canonical_cache_value(serde_json::to_value(expected).expect("expected cache structure")), + "every cache field and map entry must be retained" + ); +} + +#[test] +fn cache_cost_comparison_preserves_set_and_ordered_field_semantics() { + let forward = fixture(2); + let mut reverse = forward.clone(); + let children = &mut reverse.cache.get_mut(&hash_path("bucket").key()).expect("root").children; + children.clear(); + for index in (0..2).rev() { + children.insert(hash_path(&format!("bucket/object-{index:05}")).key()); + } + same_cache(&forward, &reverse); + + let original = serde_json::json!({"cache": {"root": {"children": ["a", "b"], "size": 1, "histogram": [1, 2]}}}); + let mut reordered = original.clone(); + reordered["cache"]["root"]["children"] = serde_json::json!(["b", "a"]); + assert_eq!(canonical_cache_value(original.clone()), canonical_cache_value(reordered)); + for children in [serde_json::json!(["a"]), serde_json::json!(["a", "b", "b"])] { + let mut changed = original.clone(); + changed["cache"]["root"]["children"] = children; + assert_ne!(canonical_cache_value(original.clone()), canonical_cache_value(changed)); + } + for (field, replacement) in [("size", serde_json::json!(2)), ("histogram", serde_json::json!([2, 1]))] { + let mut changed = original.clone(); + changed["cache"]["root"][field] = replacement; + assert_ne!(canonical_cache_value(original.clone()), canonical_cache_value(changed)); + } +} + +fn quantiles(mut samples: Vec) -> Value { + assert!(!samples.is_empty() && samples.len() <= 5); + samples.sort_unstable(); + serde_json::json!({"p50_ns": samples[samples.len() / 2], "max_ns": samples[samples.len() - 1]}) +} + +async fn profile_case(objects: usize, scenario: &str, samples: usize) { + let baseline = fixture(objects); + let mut cache = baseline.clone(); + let dirty = match scenario { + "unchanged" => 0, + "small_dirty" => (objects / 100).max(1), + "all_dirty" => objects, + _ => panic!("unknown fixed scenario"), + }; + let mut changed_entry_wire_bytes = 0; + for index in 0..dirty { + let entry = cache + .cache + .get_mut(&hash_path(&format!("bucket/object-{index:05}")).key()) + .expect("dirty leaf"); + entry.size += 1; + entry.versions += 1; + changed_entry_wire_bytes += rmp_serde::to_vec(entry).expect("changed entry wire bytes").len(); + } + if scenario == "small_dirty" { + cache.info.snapshot_complete = false; + cache.info.scan_resume_after = Some("bucket/object-00000".to_string()); + } + let expected_wire = cache.marshal_msg().expect("fixture encoding"); + assert!(u64::try_from(expected_wire.len()).expect("fixture bytes") <= MAX_WIRE_BYTES); + let store = Arc::new(CountingStore::default()); + let mut loaded = DataUsageCache::default(); + let initial = loaded + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("initial revisions"); + baseline + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0) + .await + .expect("baseline save"); + + let mut clone_ns = Vec::new(); + let mut copy_ns = Vec::new(); + let mut flatten_ns = Vec::new(); + let mut encode_ns = Vec::new(); + let mut save_ns = Vec::new(); + let mut ingest_ns = Vec::new(); + for _ in 0..samples { + let started = WallInstant::now(); + let cloned = black_box(cache.clone()); + clone_ns.push(elapsed_ns(started)); + same_cache(&cloned, &cache); + drop(cloned); + + let mut copied = DataUsageCache { + info: cache.info.clone(), + ..Default::default() + }; + let started = WallInstant::now(); + copied.copy_with_children(black_box(&cache), &hash_path("bucket"), &None); + copy_ns.push(elapsed_ns(started)); + same_cache(&copied, &cache); + drop(copied); + + let started = WallInstant::now(); + let aggregate = black_box(cache.checked_flatten("bucket").expect("valid fixture tree")); + flatten_ns.push(elapsed_ns(started)); + assert_eq!( + (aggregate.objects, aggregate.versions, aggregate.size), + (objects, objects * 2 + dirty, objects * 4096 + dirty) + ); + + let started = WallInstant::now(); + let encoded = black_box(cache.marshal_msg().expect("measured encoding")); + encode_ns.push(elapsed_ns(started)); + assert_eq!(encoded, expected_wire); + same_cache(&DataUsageCache::unmarshal(&encoded).expect("measured wire reload"), &cache); + + let revisions = loaded + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("current revisions"); + store.reset_counts(); + let started = WallInstant::now(); + cache + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect("measured save"); + save_ns.push(elapsed_ns(started)); + ingest_ns.push(store.ingest_ns.load(Ordering::Relaxed)); + assert_eq!(store.puts.load(Ordering::Relaxed), 2, "main and backup writes must both occur"); + assert_eq!( + store.bytes.load(Ordering::Relaxed), + u64::try_from(expected_wire.len() * 2).expect("two saved bodies") + ); + loaded + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("saved cache reload"); + same_cache(&loaded, &cache); + } + + let before_rejected = store.slots.lock().await[0].1.clone(); + let mut conflicting = cache.clone(); + conflicting.info.next_cycle += 1; + assert!(matches!( + conflicting + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0) + .await, + Err(Error::PreconditionFailed) + )); + assert_eq!( + store.slots.lock().await[0].1, + before_rejected, + "stale CAS must not replace the retained checkpoint" + ); + println!( + "CACHE_COST {}", + serde_json::json!({ + "schema": 1, "scenario": scenario, "objects": objects, "dirty_objects": dirty, "samples": samples, + "build": { + "debug_assertions": cfg!(debug_assertions), + "test_opt_level_override": option_env!("CARGO_PROFILE_TEST_OPT_LEVEL"), + "dev_opt_level_override": option_env!("CARGO_PROFILE_DEV_OPT_LEVEL"), + "rustflags_visible_to_rustc": option_env!("RUSTFLAGS"), + "encoded_rustflags_visible_to_rustc": option_env!("CARGO_ENCODED_RUSTFLAGS"), + "source_revision": option_env!("RUSTFS_CACHE_COST_SOURCE"), + "source_tree": option_env!("RUSTFS_CACHE_COST_TREE"), + }, + "retained_cache_entries": cache.cache.len(), "cache_wire_bytes": expected_wire.len(), + "changed_entry_wire_bytes": changed_entry_wire_bytes, "save_body_bytes_per_sample": expected_wire.len() * 2, + "snapshot_complete": cache.info.snapshot_complete, + "clone": quantiles(clone_ns), "copy_with_children": quantiles(copy_ns), "checked_flatten": quantiles(flatten_ns), + "encode": quantiles(encode_ns), "save_inclusive": quantiles(save_ns), "memory_backend_ingest": quantiles(ingest_ns), + }) + ); +} + +#[tokio::test] +async fn cache_cost_profile_preserves_checkpoint_and_counts() { + let profile = match std::env::var("RUSTFS_CACHE_COST_PROFILE") { + Err(std::env::VarError::NotPresent) => false, + Ok(value) if value == "1" => true, + _ => panic!("RUSTFS_CACHE_COST_PROFILE must be absent or 1"), + }; + let (sizes, samples): (&[usize], usize) = if profile { (&[1024, 4096, 16384], 5) } else { (&[64], 1) }; + for &objects in sizes { + for scenario in ["unchanged", "small_dirty", "all_dirty"] { + profile_case(objects, scenario, samples).await; + } + } +} diff --git a/docs/testing/README.md b/docs/testing/README.md index 55dc6a4e0..138f5b874 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -23,6 +23,8 @@ Every script named above is indexed with status and wiring in [`scripts/README.m The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation. +The [scanner cache cost profile](scanner-cache-cost.md) separates clone, subtree copy, encoding, and counted save costs without changing production cache behavior. + ## Naming conventions ### Reserved test-name substrings (migration gate) diff --git a/docs/testing/scanner-cache-cost.md b/docs/testing/scanner-cache-cost.md new file mode 100644 index 000000000..7bc285c60 --- /dev/null +++ b/docs/testing/scanner-cache-cost.md @@ -0,0 +1,36 @@ +# Scanner Cache Cost Profile + +The `cache_cost_profile_preserves_checkpoint_and_counts` test isolates the real cache operations used by the scanner: full clone, `copy_with_children`, checked flattening, MessagePack encoding, and `save_with_revisions_for_epoch`. It does not run a namespace walker or the scanner scheduler. An unchanged-cache save is deliberately requested to measure its cost, not to claim that production always saves cold buckets. + +```sh +cargo test -p rustfs-scanner --lib cache_cost_profile -- --list +RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib cache_cost_profile -- --nocapture +env -u RUSTFLAGS -u CARGO_ENCODED_RUSTFLAGS \ + CARGO_PROFILE_TEST_OPT_LEVEL=0 CARGO_PROFILE_DEV_OPT_LEVEL=0 \ + RUSTFS_CACHE_COST_SOURCE="$(git rev-parse HEAD)" \ + RUSTFS_CACHE_COST_TREE="$(git rev-parse HEAD^{tree})" \ + RUST_MIN_STACK=4194304 RUSTFS_CACHE_COST_PROFILE=1 \ + cargo test -p rustfs-scanner --lib cache_cost_profile -- --nocapture +``` + +The default positive control has 64 object entries and one sample for each of unchanged, small-dirty, and all-dirty caches. Explicit profiling uses 1,024, 4,096, and 16,384 object entries, each with five samples in all three scenarios. Small-dirty updates one percent of leaves, with a minimum of one; all-dirty updates every leaf. Each synthetic object initially accounts for two versions and 4,096 logical bytes. These are cache metadata fixtures, not uploaded S3 bodies. Small-dirty snapshots also carry a partial flag and resume marker. No test is ignored, and wall-time thresholds do not determine correctness. + +Every measured result is checked outside its timing interval: clone and subtree copy retain every field and entry; flattening yields exact object/version/byte counts; encoding reloads the same structure; saves write both main and backup and reload the same checkpoint. A stale revision with conflicting content must fail without replacing the preceding main cache. This checks the fixture's revision contract, not distributed CAS or publication-authority behavior. + +`CACHE_COST` JSON rows contain: + +The explicit profile command is an unoptimized Cargo test/debug run (`opt-level=0`), not a release build. Run from a clean worktree and retain the command, source SHA/tree, compiler version and relevant Cargo configuration with the raw rows. Each row records compile-time assertion mode, visible optimization/flag overrides and supplied source identifiers. Null build fields mean unrecorded, not inferred defaults; these fields alone do not discover every Cargo configuration source. Debug phase ratios are not production hotspot evidence and cannot justify a runtime optimization or close the performance task. No release rebuild is required for this bounded diagnostic. + +| Field | Meaning | +|---|---| +| `clone`, `copy_with_children`, `checked_flatten`, `encode` | Phase wall-clock p50 and maximum nanoseconds; setup, validation, and disposal are excluded. | +| `save_inclusive` | Actual save entry-point wall time, including its own encoding, buffer copies, admission checks, and both backend calls. This overlaps the independently measured encode operation. | +| `memory_backend_ingest` | Sum of time inside the two counted in-memory backend puts, including stream consumption and revision checking. It is part of `save_inclusive`, not an additional cost. | +| `cache_wire_bytes` | Full snapshot's actual MessagePack size; not heap allocation, cloned bytes, or retained S3 payload bytes. | +| `changed_entry_wire_bytes` | Sum of serialized changed leaf entries, excluding keys, ancestors and metadata; a diagnostic denominator, not a durable-progress proof. Zero in the unchanged scenario. | +| `save_body_bytes_per_sample` | Bytes consumed by both successful main/backup put streams. It does not include network framing, erasure shards or retries. | +| `retained_cache_entries` | Structurally verified cache entries including the root, not newly proven namespace coverage. | + +The fixture has two memory slots capped at 32 MiB each, at most 16,384 leaves, at most five samples per case, and nine profile rows. Oversized wire data and unknown configuration fail. No sample history grows with runtime and no permanent service starts. Profile runs must be exclusive of builds and other benchmarks; otherwise label the measurements exploratory/noisy. The default debug build is a diagnostic, not release throughput evidence. Repeated identical phases can benefit from warm allocator and CPU caches; the test does not establish absence of quadratic growth or bounded production RSS. + +Use the existing [scanner ABBA harness](../../scripts/scanner_abba.py) and [benchmark runbook](../operations/scanner-benchmark-runbook.md) for deployment comparisons. This microprofile does not supply deployment ABBA, a flamegraph, allocation attribution, syscall/fsync latency, remote RPC, erasure persistence, process-crash recovery, or a performance improvement. Only measured evidence can justify a separately reviewed runtime optimization; serialization, partial/complete proof, and persistence boundaries remain unchanged here. From 0137d1406465ccb912649de2dbdb93d4a15961bf Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 14:48:41 +0800 Subject: [PATCH 15/20] test(odm): refresh verified Linux E2E selection (#7271) --- .config/e2e-full-selection.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index 6a44dee52..a4d6c6cae 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ sha256-darwin=53b05ac745905809d3828c6994bdd8ecf9d20b2b61a8a9d80fe15eb62f932193 -sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b +sha256-linux=7c892afa4b9d1591b46bd79c976b647109a277284fddb3b98edced4b0297eda2 From 27d593fa351c59f2934223e13b20c49cfbc89810 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 15:22:50 +0800 Subject: [PATCH 16/20] test(odm): bypass loopback proxies in native list fixtures (#7276) --- rustfs/src/app/bucket_list_through.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs index b0f276e32..7fecdfce1 100644 --- a/rustfs/src/app/bucket_list_through.rs +++ b/rustfs/src/app/bucket_list_through.rs @@ -1010,10 +1010,11 @@ mod tests { #[serial_test::serial] fn native_list_through_malformed_fields_follow_both_source_policies() { run_large_stack_test("native-list-through-fields", || async { + // The proxy matcher treats IP literals separately from the `*` domain wildcard. temp_env::async_with_vars( [("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), ("HTTP_PROXY", None), ("HTTPS_PROXY", None), ("ALL_PROXY", None), ("http_proxy", None), ("https_proxy", None), ("all_proxy", None), - ("NO_PROXY", Some("*")), ("no_proxy", Some("*"))], + ("NO_PROXY", Some("127.0.0.1,localhost,::1")), ("no_proxy", Some("127.0.0.1,localhost,::1"))], async { #[cfg(feature = "gcs")] let service_account = native_test_service_account(); @@ -1068,6 +1069,7 @@ mod tests { #[serial_test::serial] fn native_list_through_preserves_valid_empty_pages_and_zero_size_objects() { run_large_stack_test("native-list-through-valid", || async { + // The proxy matcher treats IP literals separately from the `*` domain wildcard. temp_env::async_with_vars( [ ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), @@ -1077,8 +1079,8 @@ mod tests { ("http_proxy", None), ("https_proxy", None), ("all_proxy", None), - ("NO_PROXY", Some("*")), - ("no_proxy", Some("*")), + ("NO_PROXY", Some("127.0.0.1,localhost,::1")), + ("no_proxy", Some("127.0.0.1,localhost,::1")), ], async { #[cfg(feature = "gcs")] From c99efd947721a7db87fb0b9b4ee3599e8269eba0 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 15:23:30 +0800 Subject: [PATCH 17/20] test(admin): box remote target repair scenarios (#7277) --- rustfs/src/admin/handlers/replication.rs | 287 ++++++++++++----------- 1 file changed, 148 insertions(+), 139 deletions(-) diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 9c0e485c0..e4ca6ec9d 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -3146,64 +3146,67 @@ mod target_repair_tests { #[tokio::test] #[serial_test::serial] async fn repair_existing_cached_target_persists_readable_targets_and_lists() { - temp_env::async_with_vars(TARGET_REPAIR_ENV, async { - let (_temp, env) = test_env().await; - let server = RemoteTargetServer::start().await; - let mut target = server.target(); - target.arn = "arn:rustfs:replication:us-east-1:cached:remote".to_string(); - let targets = BucketTargets { - targets: vec![target.clone()], - }; - metadata_sys::update(BUCKET, BUCKET_TARGETS_FILE, serde_json::to_vec(&targets).expect("encode cached target")) - .await - .expect("seed cached target"); - seed_unreadable(&env).await; - assert_eq!( - BucketTargetSys::get().get_remote_arn(BUCKET, Some(&target), "").await, - (target.arn.clone(), true) - ); - assert!( - BucketTargetSys::get() - .get_remote_target_client(BUCKET, &target.arn) + temp_env::async_with_vars( + TARGET_REPAIR_ENV, + Box::pin(async { + let (_temp, env) = test_env().await; + let server = RemoteTargetServer::start().await; + let mut target = server.target(); + target.arn = "arn:rustfs:replication:us-east-1:cached:remote".to_string(); + let targets = BucketTargets { + targets: vec![target.clone()], + }; + metadata_sys::update(BUCKET, BUCKET_TARGETS_FILE, serde_json::to_vec(&targets).expect("encode cached target")) .await - .is_some() - ); + .expect("seed cached target"); + seed_unreadable(&env).await; + assert_eq!( + BucketTargetSys::get().get_remote_arn(BUCKET, Some(&target), "").await, + (target.arn.clone(), true) + ); + assert!( + BucketTargetSys::get() + .get_remote_target_client(BUCKET, &target.arn) + .await + .is_some() + ); - let arn = repair(&target, "replace-unreadable=true").await.expect("repair must commit"); - assert_ne!(arn, target.arn); - assert!( - BucketTargetSys::get() - .get_remote_target_client(BUCKET, &target.arn) + let arn = repair(&target, "replace-unreadable=true").await.expect("repair must commit"); + assert_ne!(arn, target.arn); + assert!( + BucketTargetSys::get() + .get_remote_target_client(BUCKET, &target.arn) + .await + .is_none() + ); + assert!(BucketTargetSys::get().get_remote_target_client(BUCKET, &arn).await.is_some()); + let persisted = metadata_sys::get_config_from_disk(BUCKET) .await - .is_none() - ); - assert!(BucketTargetSys::get().get_remote_target_client(BUCKET, &arn).await.is_some()); - let persisted = metadata_sys::get_config_from_disk(BUCKET) - .await - .expect("read repaired metadata"); - assert!(!persisted.bucket_targets_unreadable()); - let targets = persisted.bucket_target_config.expect("decode persisted repair"); - assert_eq!(targets.targets.len(), 1); - assert_eq!(targets.targets[0].arn, arn); - assert_eq!( - targets.targets[0] - .credentials - .as_ref() - .expect("persist credentials") - .secret_key, - "remote-secret" - ); - let list = ListRemoteTargetHandler {} - .call(request(Method::GET, "", Vec::new()), Params::new()) - .await - .expect("list repaired targets"); - assert_eq!(list.output.0, StatusCode::OK); - let listed: serde_json::Value = - serde_json::from_slice(&list.output.1.collect().await.expect("collect target list").to_bytes()) - .expect("decode list"); - assert_eq!(listed.as_array().expect("targets list").len(), 1); - assert_eq!(listed[0]["arn"], arn); - }) + .expect("read repaired metadata"); + assert!(!persisted.bucket_targets_unreadable()); + let targets = persisted.bucket_target_config.expect("decode persisted repair"); + assert_eq!(targets.targets.len(), 1); + assert_eq!(targets.targets[0].arn, arn); + assert_eq!( + targets.targets[0] + .credentials + .as_ref() + .expect("persist credentials") + .secret_key, + "remote-secret" + ); + let list = ListRemoteTargetHandler {} + .call(request(Method::GET, "", Vec::new()), Params::new()) + .await + .expect("list repaired targets"); + assert_eq!(list.output.0, StatusCode::OK); + let listed: serde_json::Value = + serde_json::from_slice(&list.output.1.collect().await.expect("collect target list").to_bytes()) + .expect("decode list"); + assert_eq!(listed.as_array().expect("targets list").len(), 1); + assert_eq!(listed[0]["arn"], arn); + }), + ) .await; } @@ -3260,51 +3263,54 @@ mod target_repair_tests { #[tokio::test] #[serial_test::serial] async fn repair_with_stale_unreadable_cache_preserves_another_committed_repair() { - temp_env::async_with_vars(TARGET_REPAIR_ENV, async { - let (_temp, env) = test_env().await; - let first = RemoteTargetServer::start().await; - let second = RemoteTargetServer::start().await; - seed_unreadable(&env).await; - let first_arn = repair(&first.target(), "replace-unreadable=true") - .await - .expect("commit first repair"); - // Model another node which still retains the original unreadable - // verdict when it begins its repair after this commit. - BucketTargetSys::get().mark_targets_unreadable(BUCKET).await; - let second_arn = repair(&second.target(), "replace-unreadable=true") - .await - .expect("merge second repair"); - let persisted = metadata_sys::get_config_from_disk(BUCKET).await.expect("read both repairs"); - let targets = persisted.bucket_target_config.expect("decode both repairs"); - assert_eq!(targets.targets.len(), 2); - assert!(targets.targets.iter().any(|target| target.arn == first_arn)); - assert!(targets.targets.iter().any(|target| target.arn == second_arn)); - assert_eq!( - BucketTargetSys::get() - .list_bucket_targets(BUCKET) + temp_env::async_with_vars( + TARGET_REPAIR_ENV, + Box::pin(async { + let (_temp, env) = test_env().await; + let first = RemoteTargetServer::start().await; + let second = RemoteTargetServer::start().await; + seed_unreadable(&env).await; + let first_arn = repair(&first.target(), "replace-unreadable=true") .await - .expect("published repair") - .targets - .len(), - 2 - ); - assert_eq!( - repair(&first.target(), "replace-unreadable=true") + .expect("commit first repair"); + // Model another node which still retains the original unreadable + // verdict when it begins its repair after this commit. + BucketTargetSys::get().mark_targets_unreadable(BUCKET).await; + let second_arn = repair(&second.target(), "replace-unreadable=true") .await - .expect("idempotent repair"), - first_arn - ); - assert_eq!( - metadata_sys::get_config_from_disk(BUCKET) - .await - .expect("read repeated repair") - .bucket_target_config - .expect("decode repeated repair") - .targets - .len(), - 2 - ); - }) + .expect("merge second repair"); + let persisted = metadata_sys::get_config_from_disk(BUCKET).await.expect("read both repairs"); + let targets = persisted.bucket_target_config.expect("decode both repairs"); + assert_eq!(targets.targets.len(), 2); + assert!(targets.targets.iter().any(|target| target.arn == first_arn)); + assert!(targets.targets.iter().any(|target| target.arn == second_arn)); + assert_eq!( + BucketTargetSys::get() + .list_bucket_targets(BUCKET) + .await + .expect("published repair") + .targets + .len(), + 2 + ); + assert_eq!( + repair(&first.target(), "replace-unreadable=true") + .await + .expect("idempotent repair"), + first_arn + ); + assert_eq!( + metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read repeated repair") + .bucket_target_config + .expect("decode repeated repair") + .targets + .len(), + 2 + ); + }), + ) .await; } @@ -3408,48 +3414,51 @@ mod target_repair_tests { #[tokio::test] #[serial_test::serial] async fn failed_repair_transaction_never_reports_a_successful_replacement() { - temp_env::async_with_vars(TARGET_REPAIR_ENV, async { - let (_temp, env) = test_env().await; - let server = RemoteTargetServer::start().await; - seed_unreadable(&env).await; - let target = server.target(); - BucketTargetSys::get() - .validate_target(BUCKET, &target) - .await - .expect("remote validation must succeed before injecting the metadata failure"); - let file = metadata_sys::get_config_from_disk(BUCKET) - .await - .expect("read source metadata") - .save_file_path(); - // Keep the source versioning and unreadable-target caches intact, - // but make the transaction's fresh disk load fail. - let corrupt = b"invalid metadata envelope".to_vec(); - env.put_object_bytes(".rustfs.sys", &file, corrupt.clone()).await; - assert!(metadata_sys::get_config_from_disk(BUCKET).await.is_err()); - let log = tempfile::NamedTempFile::new().expect("create captured log"); - let writer = log.reopen().expect("open captured log writer"); - let subscriber = tracing_subscriber::fmt() - .with_ansi(false) - .without_time() - .with_writer(writer) - .finish(); - let error = repair(&target, "replace-unreadable=true") - .with_subscriber(subscriber) - .await - .expect_err("repair must fail on the unreadable metadata envelope"); - assert_eq!(error.code(), &S3ErrorCode::InternalError); - let lines = std::fs::read_to_string(log.path()).expect("read captured log"); - assert!( - !lines.contains("unreadable_targets_replaced"), - "a failed transaction must not claim success: {lines}" - ); - assert_eq!( - crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + temp_env::async_with_vars( + TARGET_REPAIR_ENV, + Box::pin(async { + let (_temp, env) = test_env().await; + let server = RemoteTargetServer::start().await; + seed_unreadable(&env).await; + let target = server.target(); + BucketTargetSys::get() + .validate_target(BUCKET, &target) .await - .expect("read failed repair bytes"), - corrupt - ); - }) + .expect("remote validation must succeed before injecting the metadata failure"); + let file = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read source metadata") + .save_file_path(); + // Keep the source versioning and unreadable-target caches intact, + // but make the transaction's fresh disk load fail. + let corrupt = b"invalid metadata envelope".to_vec(); + env.put_object_bytes(".rustfs.sys", &file, corrupt.clone()).await; + assert!(metadata_sys::get_config_from_disk(BUCKET).await.is_err()); + let log = tempfile::NamedTempFile::new().expect("create captured log"); + let writer = log.reopen().expect("open captured log writer"); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_writer(writer) + .finish(); + let error = repair(&target, "replace-unreadable=true") + .with_subscriber(subscriber) + .await + .expect_err("repair must fail on the unreadable metadata envelope"); + assert_eq!(error.code(), &S3ErrorCode::InternalError); + let lines = std::fs::read_to_string(log.path()).expect("read captured log"); + assert!( + !lines.contains("unreadable_targets_replaced"), + "a failed transaction must not claim success: {lines}" + ); + assert_eq!( + crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read failed repair bytes"), + corrupt + ); + }), + ) .await; } From a40ec8a6f3e919ef95b213228f7ea9a6e3adc550 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 15:24:59 +0800 Subject: [PATCH 18/20] fix(odm): preserve progress when list-through is disabled (#7278) * test(odm): cover disabled list-through continuation progress * test(odm): cover literal cache tags in disabled list cursors * fix(odm): preserve progress when list-through is disabled --- docs/operations/on-demand-migration.md | 2 + rustfs/src/app/bucket_list_through.rs | 512 ++++++++++++++++++------- rustfs/src/app/bucket_usecase.rs | 45 +-- 3 files changed, 385 insertions(+), 174 deletions(-) diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md index 06d066fd4..137f42248 100644 --- a/docs/operations/on-demand-migration.md +++ b/docs/operations/on-demand-migration.md @@ -28,6 +28,8 @@ The two rollout switches have different defaults. Unset or invalid boolean value - `RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false` and allows a v1 listing to first issue a v2 token after an empty truncated merged page. Existing v2 tokens keep their budget even on reader-only nodes. Consuming an object/common prefix or reaching a new EOF resets the budget to v1 without changing the chain's framing. - `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS` defaults to `true`, preserving the framed output of the #7187 / `e1608fbd9` generation. It allows a bare/new merged listing to first issue a NUL-prefixed JSON envelope inside the existing base64 encoding. Existing framed chains stay framed even with this switch off, including a reset to v1 and local continuation after list-through is disabled. With framing issuance off, new bare v1 output keeps its historical bytes; ordinary local listings remain unchanged. This switch does not enable the v2 budget. +When list-through or the global module is disabled, existing merged continuations retain their last emitted key and original bare/framed format, filter already returned objects and common prefixes, and make no source requests. If the local-only response remains truncated, its continuation can resume both sides after re-enabling list-through. + Choose the upgrade path from the binaries currently serving LIST requests, including every load-balancer route. This build reads complete historical bare envelopes and framed v1/v2 envelopes with the same strict version/count validation. There is no single writer format understood by both bare-only and framed-only readers. A bare-v1-only binary rejects bare v2 with `400 InvalidArgument`; a bare-only reader mistakes framed input for a local marker, while a framed-only reader mistakes bare input for one. These framing mismatches can restart a merged scan and lose its budget without returning an error. For example, with local keys `b,d` and source keys `a,c`, a new node issuing bare after `a` followed by an `e1608fbd9` reader can return `a` again. That old reader then emits framing, so the symptom need not be an infinite loop. - **Upgrading from the framed-only #7187 / `e1608fbd9` generation:** before deploying the first new node, set `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS=true` in the new nodes' deployment environment, or leave it unset to use this build's `true` default. Remove any previous explicit `false` override. Existing framed-only nodes ignore this new variable and already emit framing. Keep framing enabled while any of those nodes serves continuations. While list-through remains active, new and existing framed v1/v2 chains then retain their cursors and any active budget in both directions. Only after all serving nodes are dual readers may you choose `false`; existing framed chains still stay framed, while newly started bare chains must stay on dual readers. diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs index 7fecdfce1..40713b952 100644 --- a/rustfs/src/app/bucket_list_through.rs +++ b/rustfs/src/app/bucket_list_through.rs @@ -33,10 +33,9 @@ use super::storage_api::bucket_usecase::s3_api::bucket::ListObjectsV2Params; use crate::app::object::shared::{odm_source_unavailable_error, odm_state_error_class}; use crate::error::ApiError; use crate::on_demand_migration::{ - BucketOdmState, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, - ListThroughToken, ListThroughTokenError, MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, - SourceError, SourceErrorPolicy, SourceListPlan, SourceListRequest, SourceObject, SourcePage, decode_continuation_token, - source_list_plan, + BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, + MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan, + SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan, }; use futures::StreamExt; use rustfs_utils::http::{SUFFIX_SOURCE_PROXY_REQUEST, get_header}; @@ -61,15 +60,6 @@ const ENV_LIST_FRAMED_TOKENS: &str = "RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOK /// source-only keys for a shadowing delete marker. const DELETE_MARKER_PROBE_CONCURRENCY: usize = 32; -/// Where the local side of a listing resumes. -pub(crate) enum LocalListCursor { - /// Continuation token for the local store, `None` for the first page. - Token(Option), - /// The local side of a merged listing was already exhausted, so a listing - /// that no longer merges has nothing left to return. - Exhausted, -} - /// Reads the (already base64-decoded) continuation token. /// /// This runs whether or not the bucket merges: a token handed out while @@ -83,47 +73,6 @@ pub(crate) fn decode_list_cursor(decoded: Option<&str>) -> S3Result, merged: Option<&ListThroughToken>) -> LocalListCursor { - match merged { - Some(token) if token.local_done => LocalListCursor::Exhausted, - Some(token) => LocalListCursor::Token(token.local.clone()), - None => LocalListCursor::Token(decoded.map(str::to_string)), - } -} - -/// A framed chain keeps its envelope when the bucket stops consulting source. -pub(crate) fn preserve_framed_local_cursor(info: &mut ListObjectsV2Info, previous: Option<&ListThroughToken>) { - let Some(previous) = previous.filter(|token| token.framed) else { - return; - }; - let Some(next) = info.next_continuation_token.take() else { - return; - }; - let mut token = previous.clone(); - token.local = Some(next); - token.local_done = false; - if let Some(last_key) = info - .objects - .iter() - .map(|object| object.name.as_str()) - .chain(info.prefixes.iter().map(String::as_str)) - .max() - { - token.last_key = Some( - token - .last_key - .as_deref() - .map_or(last_key, |previous| previous.max(last_key)) - .to_string(), - ); - token.v = LIST_THROUGH_TOKEN_VERSION; - token.no_progress = None; - } - info.next_continuation_token = Some(token.encode()); -} - fn invalid_continuation_token(err: &ListThroughTokenError) -> S3Error { debug!(error = %err, "rejected an on-demand migration list continuation token"); S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid continuation token".to_string()) @@ -240,7 +189,10 @@ fn source_page_entries(bucket: &str, page: SourcePage) -> Vec { interleave(objects, page.common_prefixes) } -/// Runs one merged `ListObjectsV2` page. +/// Runs one merged `ListObjectsV2` page. With no source state, continues only +/// the local side of an existing merged chain. The original local page cursor +/// is reread and raw names are filtered against the last emitted key here; +/// sending that key to storage as a marker could interpret literal cache tags. /// /// Cost: at most two listings per side per request — the first page of each /// side, plus one refill when the previous page had already consumed most of @@ -249,30 +201,34 @@ fn source_page_entries(bucket: &str, page: SourcePage) -> Vec { /// source listings. pub(crate) async fn merged_list_objects_v2( store: &Arc, - state: &Arc, + state: Option<&Arc>, bucket: &str, params: &ListObjectsV2Params, fetch_owner: bool, incl_deleted: bool, token: Option<&ListThroughToken>, ) -> S3Result { - let policy = &state.config().policy; let max_keys = usize::try_from(params.max_keys).unwrap_or(0); let mut merger = ListThroughMerger::new(max_keys, token); let mut buffers: [Vec>; 2] = [Vec::new(), Vec::new()]; let mut degraded = false; - let plan = source_list_plan(¶ms.prefix, state.config().filter.prefix.as_deref(), params.delimiter.as_deref()); + let plan = state.map_or(SourceListPlan::Skip, |state| { + source_list_plan(¶ms.prefix, state.config().filter.prefix.as_deref(), params.delimiter.as_deref()) + }); let mut client: Option> = None; - match &plan { - // Nothing the source holds can appear under this prefix; that is a - // filter decision, not a degradation. - SourceListPlan::Skip => merger.disable_source(), - _ => match state.client() { - Ok(ready) if state.breaker().allow_request() => client = Some(Arc::clone(ready)), - Ok(_) => degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "breaker_open")?, - Err(error) => degrade_or_fail(&mut merger, &mut degraded, policy.source_error, odm_state_error_class(error))?, - }, + match (state, &plan) { + // A disabled or excluded source is not a degradation. The merger + // retains its unconsumed source cursor for a later enabled request. + (None, _) | (_, SourceListPlan::Skip) => merger.disable_source(), + (Some(state), _) => { + let policy = &state.config().policy; + match state.client() { + Ok(ready) if state.breaker().allow_request() => client = Some(Arc::clone(ready)), + Ok(_) => degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "breaker_open")?, + Err(error) => degrade_or_fail(&mut merger, &mut degraded, policy.source_error, odm_state_error_class(error))?, + } + } } while let Some(fetch) = merger.next_fetch() { @@ -295,6 +251,8 @@ pub(crate) async fn merged_list_objects_v2( (interleave(objects, info.prefixes), info.is_truncated, info.next_continuation_token) } MergeSide::Source => { + let state = state.expect("the source side is disabled without bucket state"); + let policy = &state.config().policy; let client = client.as_ref().expect("the source side is disabled without a client"); match fetch_source_page(state, client, bucket, params, &plan, fetch.token.as_deref()).await { Ok(page) => page, @@ -314,6 +272,7 @@ pub(crate) async fn merged_list_objects_v2( if let Err(error) = merger.push_page(fetch.side, keys, is_truncated, next_token) { match fetch.side { MergeSide::Source => { + let policy = &state.expect("only an enabled source can return a page").config().policy; degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?; continue; } @@ -324,10 +283,15 @@ pub(crate) async fn merged_list_objects_v2( } let issue_progress_tokens = rustfs_utils::get_env_bool(ENV_LIST_PROGRESS_TOKENS, false); - let framed = token.is_some_and(|token| token.framed) || rustfs_utils::get_env_bool(ENV_LIST_FRAMED_TOKENS, true); + let framed = + token.is_some_and(|token| token.framed) || (state.is_some() && rustfs_utils::get_env_bool(ENV_LIST_FRAMED_TOKENS, true)); let outcome = match merger.finish(issue_progress_tokens) { Ok(outcome) => outcome, Err(ListPageError::NoProgress(MergeSide::Source)) => { + let policy = &state + .expect("a disabled source cannot exhaust the progress budget") + .config() + .policy; degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?; merger .finish(issue_progress_tokens) @@ -353,7 +317,10 @@ pub(crate) async fn merged_list_objects_v2( } } - if !source_only_keys.is_empty() && policy.respect_local_delete_marker && bucket_keeps_delete_markers(bucket).await { + if !source_only_keys.is_empty() + && state.is_some_and(|state| state.config().policy.respect_local_delete_marker) + && bucket_keeps_delete_markers(bucket).await + { let shadowed = local_delete_markers(store, bucket, &source_only_keys).await; objects.retain(|object| !shadowed.contains(&object.name)); } @@ -586,14 +553,18 @@ mod tests { let encoded = resume.encode(); let decoded = decode_list_cursor(Some(&encoded)).expect("a valid envelope decodes"); assert_eq!(decoded.as_ref(), Some(&resume)); - assert!(matches!( - local_cursor(Some(&encoded), decoded.as_ref()), - LocalListCursor::Token(Some(local)) if local == "local-2" - )); + let mut merger = ListThroughMerger::new(2, decoded.as_ref()); + merger.disable_source(); + let fetch = merger.next_fetch().expect("unconsumed local page"); + assert_eq!(fetch.side, MergeSide::Local); + assert_eq!(fetch.token.as_deref(), Some("local-2")); let encoded = token(None, true).encode(); let decoded = decode_list_cursor(Some(&encoded)).expect("a valid envelope decodes"); - assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted)); + let mut merger = ListThroughMerger::new(2, decoded.as_ref()); + merger.disable_source(); + assert!(merger.next_fetch().is_none()); + assert!(!merger.finish(false).expect("local EOF").is_truncated); } #[test] @@ -606,31 +577,44 @@ mod tests { let encoded = resume.encode(); let decoded = decode_list_cursor(Some(&encoded)).expect("a v2 envelope decodes"); assert_eq!(decoded.as_ref(), Some(&resume)); - assert!(matches!( - local_cursor(Some(&encoded), decoded.as_ref()), - LocalListCursor::Token(Some(local)) if local == "local-2" - )); + let mut merger = ListThroughMerger::new(2, decoded.as_ref()); + merger.disable_source(); + let fetch = merger.next_fetch().expect("unconsumed local page"); + assert_eq!(fetch.side, MergeSide::Local); + assert_eq!(fetch.token.as_deref(), Some("local-2")); resume.local_done = true; let encoded = resume.encode(); let decoded = decode_list_cursor(Some(&encoded)).expect("v2 with local EOF decodes"); - assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted)); + let mut merger = ListThroughMerger::new(2, decoded.as_ref()); + merger.disable_source(); + assert!(merger.next_fetch().is_none()); + assert!(!merger.finish(false).expect("local EOF").is_truncated); } } #[test] fn a_plain_local_token_is_passed_through_and_a_tampered_one_is_rejected() { + use crate::app::storage_api::bucket_usecase::s3_api::bucket::parse_list_objects_v2_params; + let json_key = r#"{"t":"odm-list","v":1,"local_done":true}"#; assert!(decode_list_cursor(Some(json_key)).expect("valid local key").is_none()); - assert!(matches!(local_cursor(Some(json_key), None), LocalListCursor::Token(Some(local)) if local == json_key)); assert!( decode_list_cursor(Some("photos/a.jpg")) .expect("plain markers decode") .is_none() ); - assert!(matches!( - local_cursor(Some("photos/a.jpg"), None), - LocalListCursor::Token(Some(local)) if local == "photos/a.jpg" - )); + + for marker in [json_key, "photos/a.jpg"] { + let params = parse_list_objects_v2_params( + None, + None, + Some(2), + Some(base64_simd::STANDARD.encode_to_string(marker.as_bytes())), + None, + ) + .expect("plain local listing parameters"); + assert_eq!(params.decoded_continuation_token.as_deref(), Some(marker)); + } let tampered = token(Some("local-2"), false).encode().replace("\"v\":1", "\"v\":9"); let err = decode_list_cursor(Some(&tampered)).expect_err("a bumped version is rejected"); @@ -1783,7 +1767,7 @@ mod tests { Duration::from_secs(10), merged_list_objects_v2( &store, - &state, + Some(&state), &input.bucket, ¶ms, input.fetch_owner.unwrap_or_default(), @@ -2116,71 +2100,303 @@ mod tests { .expect("merged continuation token") } + #[derive(Clone, Copy)] + enum DisabledListScenario { + FirstLocalPage, + PartiallyConsumedLocalPage, + CacheTag(&'static str), + CommonPrefixes, + Reenable, + } + + async fn assert_disabled_list_progress(scenario: DisabledListScenario, framed: bool, disable_module: bool) { + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + let (local_keys, source_names, expected_first, expected_disabled) = match scenario { + DisabledListScenario::FirstLocalPage => (vec!["a", "c"], ["b", "d"], ["a", "b"], vec!["c", "z-local"]), + DisabledListScenario::PartiallyConsumedLocalPage => { + (vec!["a", "b", "c", "e"], ["d", "f"], ["a", "b"], vec!["e", "z-local"]) + } + DisabledListScenario::CacheTag(key) => (vec!["a", "b!", "b0", "c"], [key, "f"], ["a", "b!"], vec!["c", "z-local"]), + DisabledListScenario::CommonPrefixes => (vec!["p/a/1", "p/c/1"], ["p/b/", "p/d/"], ["p/a/", "p/b/"], vec!["p/c/"]), + DisabledListScenario::Reenable => (vec!["a", "c", "e"], ["b", "f"], ["a", "b"], vec!["c", "e"]), + }; + let prefixes = matches!(scenario, DisabledListScenario::CommonPrefixes); + let entries = source_names + .iter() + .map(|name| { + if prefixes { + format!("{name}") + } else { + format!("{name}1") + } + }) + .collect::(); + let body = format!( + "false{entries}" + ); + let source_disabled = Arc::new(AtomicBool::new(false)); + let source_requests = Arc::new(AtomicUsize::new(0)); + let forbidden = Arc::clone(&source_disabled); + let count = Arc::clone(&source_requests); + let (endpoint, server, stop) = list_source_with_response(std::iter::repeat(body), move |_, body| { + assert!(!forbidden.load(Ordering::SeqCst), "disabled listing must not call the source"); + count.fetch_add(1, Ordering::SeqCst); + body + }) + .await; + let (_state_guard, mut input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await; + let store = shared_gating_ecstore().await; + for key in local_keys { + store + .put_object( + &input.bucket, + key, + &mut StoragePutObjReader::from_vec(vec![1]), + &StorageObjectOptions::default(), + ) + .await + .expect("seed disabled-list local keys"); + } + if prefixes { + input.prefix = Some("p/".to_string()); + input.delimiter = Some("/".to_string()); + } + input.start_after = Some(if prefixes { "p/0" } else { "0" }.to_string()); + let page_names = |page: &ListObjectsV2Output| { + let mut names = page + .contents + .iter() + .flatten() + .map(|object| object.key.clone().expect("listed object key")) + .chain( + page.common_prefixes + .iter() + .flatten() + .map(|prefix| prefix.prefix.clone().expect("listed common prefix")), + ) + .collect::>(); + names.sort(); + names + }; + let first = execute_source_list(input.clone()).await.expect("first merged page").output; + assert_eq!(page_names(&first), expected_first); + assert_eq!(first.key_count, Some(2)); + assert_eq!(first.is_truncated, Some(true)); + let mut cursor = first.next_continuation_token.expect("merged cursor"); + let mut token = decode_wire_token(&cursor); + assert_eq!(token.framed, framed); + let second_page = match scenario { + DisabledListScenario::PartiallyConsumedLocalPage => Some((["c", "d"], ["c", "e"])), + DisabledListScenario::CacheTag(key) => Some((["b0", key], ["b0", "c"])), + _ => None, + }; + if let Some((expected_second, expected_replay)) = second_page { + let first_local = token.local.clone().expect("first local page was fully consumed"); + assert!( + first_local.starts_with(&format!("{}[rustfs_cache:", expected_first[1])), + "expected a real opaque cache cursor: {first_local}" + ); + input.continuation_token = Some(cursor); + let second = execute_source_list(input.clone()).await.expect("second merged page").output; + assert_eq!(page_names(&second), expected_second); + cursor = second.next_continuation_token.expect("partially consumed local page"); + token = decode_wire_token(&cursor); + assert_eq!( + token.local.as_deref(), + Some(first_local.as_str()), + "keep the partially consumed local page" + ); + assert_eq!(token.last_key.as_deref(), Some(expected_second[1])); + // ECStore prioritizes its opaque continuation over StartAfter. A + // fix that merely passes both would still replay c from this page. + let token_wins = Arc::clone(&store) + .list_objects_v2( + &input.bucket, + "", + Some(first_local), + None, + 2, + false, + Some(expected_second[1].to_string()), + false, + ) + .await + .expect("verify the real storage continuation contract"); + assert_eq!( + token_wins + .objects + .iter() + .map(|object| object.name.as_str()) + .collect::>(), + expected_replay + ); + } else { + assert_eq!(token.local, None, "the first local page remains partially consumed"); + assert_eq!(token.last_key.as_deref(), Some(expected_first[1])); + } + assert!(!token.local_done); + assert!(!token.source_done); + let requests_before_disable = source_requests.load(Ordering::SeqCst); + assert_eq!(requests_before_disable, if second_page.is_some() { 2 } else { 1 }); + let sys = OnDemandMigrationSys::get(); + let installed = sys.state(&input.bucket).expect("installed source state"); + let saved_config = installed.config().clone(); + source_disabled.store(true, Ordering::SeqCst); + if disable_module { + sys.set_module_enabled(false); + } else { + let mut disabled_config = saved_config.clone(); + disabled_config.policy.list_through = false; + sys.apply_for_incarnation(&input.bucket, installed.incarnation_id(), Some(&disabled_config)) + .await; + } + input.continuation_token = Some(cursor); + let disabled = execute_source_list(input.clone()).await.expect("local-only continuation"); + assert!(!disabled.headers.contains_key("x-rustfs-on-demand-migration-list")); + assert_eq!( + page_names(&disabled.output), + expected_disabled, + "framed={framed}, module={disable_module}" + ); + assert_eq!( + disabled.output.key_count, + Some(i32::try_from(expected_disabled.len()).expect("page size")) + ); + assert_eq!(disabled.output.prefix.as_deref(), Some(if prefixes { "p/" } else { "" })); + assert_eq!(disabled.output.delimiter, input.delimiter); + assert_eq!(disabled.output.start_after, input.start_after, "echo the client's original StartAfter"); + assert_eq!(source_requests.load(Ordering::SeqCst), requests_before_disable); + if matches!(scenario, DisabledListScenario::Reenable) { + assert_eq!(disabled.output.is_truncated, Some(true)); + let next = disabled.output.next_continuation_token.expect("local side still has z-local"); + let next_token = decode_wire_token(&next); + assert_eq!(next_token.framed, framed, "retain the chain's existing wire format"); + assert_eq!(next_token.source, token.source, "disabled requests do not consume source pages"); + assert_eq!(next_token.last_key.as_deref(), Some("e")); + source_disabled.store(false, Ordering::SeqCst); + if disable_module { + sys.set_module_enabled(true); + } else { + sys.apply_for_incarnation(&input.bucket, installed.incarnation_id(), Some(&saved_config)) + .await; + } + input.continuation_token = Some(next); + let resumed = execute_source_list(input).await.expect("reenabled continuation").output; + assert_eq!(page_names(&resumed), ["f", "z-local"], "both sides continue beyond the local-only page"); + assert_eq!(resumed.is_truncated, Some(false)); + assert!(resumed.next_continuation_token.is_none()); + assert_eq!(source_requests.load(Ordering::SeqCst), requests_before_disable + 1); + } else { + assert_eq!(disabled.output.is_truncated, Some(false)); + assert!(disabled.output.next_continuation_token.is_none()); + } + stop.cancel(); + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("source server must finish") + .expect("source access must respect the disabled phase"); + assert_eq!(requests.len(), source_requests.load(Ordering::SeqCst)); + for request in requests { + assert!( + !request.contains("continuation-token="), + "the partially consumed source page is reread: {request}" + ); + if prefixes { + assert!(request.contains("prefix=p%2F") && request.contains("delimiter=%2F"), "{request}"); + } + } + } + + async fn disabled_list_progress_matrix(scenario: DisabledListScenario) { + for framed in [false, true] { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, Some("true")), + (ENV_LIST_FRAMED_TOKENS, Some(if framed { "true" } else { "false" })), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + Box::pin(async move { + for disable_module in [false, true] { + assert_disabled_list_progress(scenario, framed, disable_module).await; + } + }), + ) + .await; + } + } + #[test] - fn framed_local_continuations_preserve_json_markers_and_zero_sized_budgets() { + #[serial_test::serial] + fn list_through_disabled_resumes_partially_consumed_first_local_page() { + run_large_stack_test("list-disabled-first", || { + disabled_list_progress_matrix(DisabledListScenario::FirstLocalPage) + }); + } + + #[test] + #[serial_test::serial] + fn list_through_disabled_resumes_partially_consumed_opaque_local_page() { + run_large_stack_test("list-disabled-opaque", || { + disabled_list_progress_matrix(DisabledListScenario::PartiallyConsumedLocalPage) + }); + } + + #[test] + #[serial_test::serial] + fn list_through_disabled_treats_source_cache_tag_names_as_literal_keys() { + run_large_stack_test("list-disabled-literal-cache-tag", || async { + for key in ["b[rustfs_cache:v1,return:]", "b[rustfs_cache:v2,return:]"] { + disabled_list_progress_matrix(DisabledListScenario::CacheTag(key)).await; + } + }); + } + + #[test] + #[serial_test::serial] + fn list_through_disabled_resumes_common_prefixes_without_repeating() { + run_large_stack_test("list-disabled-prefixes", || { + disabled_list_progress_matrix(DisabledListScenario::CommonPrefixes) + }); + } + + #[test] + #[serial_test::serial] + fn list_through_disabled_then_reenabled_retains_progress_on_both_sides() { + run_large_stack_test("list-disabled-reenable", || disabled_list_progress_matrix(DisabledListScenario::Reenable)); + } + + #[test] + fn local_merger_preserves_json_markers_and_resets_budget_on_progress() { let json_key = r#"{"t":"odm-list","v":1,"local_done":true}"#; - let mut resume = token(Some("local-2"), false); - resume.framed = true; - resume.v = 2; - resume.no_progress = Some(15); - let mut page = ListObjectsV2Info { - is_truncated: true, - next_continuation_token: Some(json_key.to_string()), - objects: vec![info(json_key)], - ..Default::default() - }; - preserve_framed_local_cursor(&mut page, Some(&resume)); - let raw = page.next_continuation_token.expect("local continuation"); - assert!(raw.starts_with("\0odm-list:")); - let decoded = decode_list_cursor(Some(&raw)) - .expect("framed local continuation") - .expect("envelope"); - assert!(decoded.framed); - assert_eq!( - decoded.local.as_deref(), - Some(json_key), - "the local marker is embedded without another encoding" - ); - assert_eq!(decoded.source, resume.source); - assert_eq!(decoded.last_key.as_deref(), Some(json_key)); - assert_eq!(decoded.v, 1); - assert_eq!(decoded.no_progress, None); - assert!(matches!(local_cursor(Some(&raw), Some(&decoded)), LocalListCursor::Token(Some(local)) if local == json_key)); - - let mut prefix_page = ListObjectsV2Info { - is_truncated: true, - next_continuation_token: Some("photos/".to_string()), - prefixes: vec!["photos/".to_string()], - ..Default::default() - }; - preserve_framed_local_cursor(&mut prefix_page, Some(&resume)); - let prefix = decode_list_cursor(prefix_page.next_continuation_token.as_deref()) - .expect("prefix continuation") - .expect("framed prefix envelope"); - assert!(prefix.framed); - assert_eq!(prefix.last_key.as_deref(), Some("photos/")); - assert_eq!(prefix.v, 1); - assert_eq!(prefix.no_progress, None); - - let mut zero = ListObjectsV2Info { - is_truncated: true, - next_continuation_token: resume.local.clone(), - ..Default::default() - }; - preserve_framed_local_cursor(&mut zero, Some(&resume)); - assert_eq!( - decode_list_cursor(zero.next_continuation_token.as_deref()).expect("zero-sized continuation"), - Some(resume.clone()) - ); - - resume.framed = false; - let mut ordinary = ListObjectsV2Info { - is_truncated: true, - next_continuation_token: Some(json_key.to_string()), - ..Default::default() - }; - preserve_framed_local_cursor(&mut ordinary, Some(&resume)); - assert_eq!(ordinary.next_continuation_token.as_deref(), Some(json_key)); + for key in [ListEntryKey::object(json_key), ListEntryKey::prefix("photos/")] { + let mut resume = token(Some("local-2"), false); + resume.v = 2; + resume.no_progress = Some(15); + let mut merger = ListThroughMerger::new(1, Some(&resume)); + merger.disable_source(); + merger + .push_page(MergeSide::Local, vec![key.clone()], true, Some(key.name.clone())) + .expect("valid local page"); + let outcome = merger.finish(false).expect("local progress resets the budget"); + assert_eq!(outcome.picks.len(), 1); + assert_eq!(outcome.picks[0].side, MergeSide::Local); + let next = outcome.next_token.expect("local continuation"); + assert_eq!(next.local.as_deref(), Some(key.name.as_str()), "the local marker is embedded unchanged"); + assert_eq!(next.source, resume.source); + assert_eq!(next.source_done, resume.source_done); + assert_eq!(next.last_key.as_deref(), Some(key.name.as_str())); + assert_eq!(next.v, 1); + assert_eq!(next.no_progress, None); + } } #[test] diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index f06ecf4b9..1cb5a8be9 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -2797,42 +2797,35 @@ impl DefaultBucketUsecase { false, ) } - (Some(state), _) => { + (None, None) => { + let infos = store + .list_objects_v2( + &bucket, + ¶ms.prefix, + params.decoded_continuation_token.clone(), + params.delimiter.clone(), + params.max_keys, + fetch_owner.unwrap_or_default(), + params.start_after_for_query.clone(), + incl_deleted, + ) + .await + .map_err(ApiError::from)?; + (infos, false) + } + (state, token) => { let outcome = list_through::merged_list_objects_v2( &store, - &state, + state.as_ref(), &bucket, ¶ms, fetch_owner.unwrap_or_default(), incl_deleted, - merged_token.as_ref(), + token, ) .await?; (outcome.info, outcome.degraded) } - (None, _) => { - let cursor = list_through::local_cursor(params.decoded_continuation_token.as_deref(), merged_token.as_ref()); - match cursor { - list_through::LocalListCursor::Exhausted => (StorageListObjectsV2Info::default(), false), - list_through::LocalListCursor::Token(token) => { - let mut infos = store - .list_objects_v2( - &bucket, - ¶ms.prefix, - token, - params.delimiter.clone(), - params.max_keys, - fetch_owner.unwrap_or_default(), - params.start_after_for_query.clone(), - incl_deleted, - ) - .await - .map_err(ApiError::from)?; - list_through::preserve_framed_local_cursor(&mut infos, merged_token.as_ref()); - (infos, false) - } - } - } }; let output = build_list_objects_v2_output( From b92392a04a234d9efa8e4aedd46052b7e844d22c Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 6 Sep 2026 15:38:05 +0800 Subject: [PATCH 19/20] ci: give target repair tests larger stack (#7272) --- .config/nextest.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 17c5dc7a5..814feb384 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.default.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::.*|app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.default.scripts]] @@ -217,7 +217,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.ci.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::.*|app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.ci.scripts]] From d7b7d1835ff94a014f45f1709e90b9d9301f3928 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 15:43:13 +0800 Subject: [PATCH 20/20] test(admin): validate repair futures with the default stack (#7280) --- .config/nextest.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 814feb384..17c5dc7a5 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.default.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::.*|app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.default.scripts]] @@ -217,7 +217,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.ci.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::.*|app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.ci.scripts]]