From 123967e729c74903a7f3d6dcc0a388736acd4f6c Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 13:02:23 +0800 Subject: [PATCH 01/40] fix(ecstore): fail closed on an unreadable bucket-targets blob (#7172) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ecstore): correct sealed-credential test helper parameter type The helper took a HashMap that nothing imports, so the ecstore test target did not compile. * fix(ecstore): fail closed on an unreadable bucket-targets blob An undecodable bucket-targets.json was replaced by an empty BucketTargets, so every replication target of that bucket disappeared, replication stopped, and no caller saw an error. A missing secretKey alone triggers it, because Credentials has no struct-level serde(default). parse_all_configs now retains the failure instead: the raw bytes stay and the typed field stays None, which BucketMetadata::bucket_targets_unreadable reads as "exists but cannot be read" — the same distinction the fabricated marker draws for bucket metadata as a whole. One corrupt sub-config still never fails the metadata load, so an unreadable bucket cannot take down its neighbours or the node. BucketTargetSys records such buckets and answers every targets query with the new BucketRemoteTargetsUnreadable, leaving any snapshot from an earlier readable load in place so in-flight replication is not torn down. The replication heal queue reports Missed rather than scheduling against an empty target set, and the admin listing surfaces the fault instead of an empty list. Refs: rustfs/backlog#2282 * fix(ecstore): report corrupt permissive bucket configs as invalid Audit of the remaining parse_all_configs branches. Policy, versioning, object lock and replication already fail closed at their accessors; encryption, public access block and quota did not, and for those three "absent" is exactly the state that grants something — plaintext storage, anonymous access, unbounded capacity. They now report a stored-but-undecodable payload as invalid rather than as ConfigNotFound, matching the guard the versioning and object-lock accessors already use. The quota enforcement path already refused such a payload; only the metadata read path was misreporting it. The branches left degrading, and the concrete reason each is safe, are recorded in the table on parse_all_configs. Refs: rustfs/backlog#2282 --- .../ecstore/src/bucket/bucket_target_sys.rs | 109 ++++++++--- crates/ecstore/src/bucket/metadata.rs | 170 +++++++++++++++++- crates/ecstore/src/bucket/metadata_sys.rs | 165 ++++++++++++++++- .../bucket/replication/replication_pool.rs | 19 +- .../replication_target_boundary.rs | 3 +- rustfs/src/admin/handlers/replication.rs | 12 +- 6 files changed, 438 insertions(+), 40 deletions(-) diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index dca41cf72..db4e80f29 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -59,7 +59,7 @@ use rustfs_utils::http::{ insert_header, }; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::error::Error; use std::fmt; use std::str::FromStr as _; @@ -376,6 +376,11 @@ pub struct BucketTargetSys { /// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`. ssec_passthrough_map: Arc>>, pub targets_map: Arc>>>, + /// Buckets whose persisted `bucket-targets.json` exists but cannot be + /// decoded (rustfs/backlog#2282). Written under the bucket's update mutex + /// alongside `targets_map`, and read before it so an unreadable + /// configuration surfaces as a typed error instead of an empty target set. + unreadable_targets: Arc>>, pub h_mutex: Arc>>, target_h_mutex: Arc>>, pub hc_client: Arc, @@ -419,6 +424,7 @@ impl BucketTargetSys { arn_remotes_map: Arc::new(RwLock::new(HashMap::new())), ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())), targets_map: Arc::new(RwLock::new(HashMap::new())), + unreadable_targets: Arc::new(RwLock::new(HashSet::new())), h_mutex: Arc::new(RwLock::new(HashMap::new())), target_h_mutex: Arc::new(RwLock::new(HashMap::new())), hc_client: Arc::new(build_health_check_client()), @@ -628,30 +634,40 @@ impl BucketTargetSys { health_map.clone() } - pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Vec { + /// Targets of one bucket, or of every bucket when `bucket` is empty. + /// + /// A bucket that simply has no targets yields an empty list; a bucket + /// whose persisted configuration cannot be decoded is an error, so an + /// admin listing reports the fault instead of an empty list that reads as + /// "replication is not configured" (rustfs/backlog#2282). + pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Result, BucketTargetError> { let health_stats = self.target_health_stats().await; let mut targets = Vec::new(); if !bucket.is_empty() { - if let Ok(bucket_targets) = self.list_bucket_targets(bucket).await { - for mut target in bucket_targets.targets { - if arn_type.is_empty() || target.target_type.to_string() == arn_type { - if let Some(health) = health_stats.get(&target.arn) { - target.total_downtime = health.offline_duration; - target.online = health.online; - target.last_online = health.last_online; - target.latency = target::LatencyStat { - curr: health.latency.curr, - avg: health.latency.avg, - max: health.latency.peak, - }; - target.offline_count = health.offline_count; + match self.list_bucket_targets(bucket).await { + Ok(bucket_targets) => { + for mut target in bucket_targets.targets { + if arn_type.is_empty() || target.target_type.to_string() == arn_type { + if let Some(health) = health_stats.get(&target.arn) { + target.total_downtime = health.offline_duration; + target.online = health.online; + target.last_online = health.last_online; + target.latency = target::LatencyStat { + curr: health.latency.curr, + avg: health.latency.avg, + max: health.latency.peak, + }; + target.offline_count = health.offline_count; + } + targets.push(target); } - targets.push(target); } } + Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => {} + Err(err) => return Err(err), } - return targets; + return Ok(targets); } let targets_map = self.targets_map.read().await; @@ -674,10 +690,16 @@ impl BucketTargetSys { } } - targets + Ok(targets) } pub async fn list_bucket_targets(&self, bucket: &str) -> Result { + if self.unreadable_targets.read().await.contains(bucket) { + return Err(BucketTargetError::BucketRemoteTargetsUnreadable { + bucket: bucket.to_string(), + }); + } + let targets_map = self.targets_map.read().await; if let Some(targets) = targets_map.get(bucket) { Ok(BucketTargets { @@ -690,13 +712,30 @@ impl BucketTargetSys { } } + /// Record that this bucket's persisted targets configuration exists but + /// cannot be decoded (rustfs/backlog#2282). + /// + /// Any snapshot published from an earlier readable load is deliberately + /// left in place: withdrawing it would produce exactly the silent "no + /// targets configured" state this marker exists to prevent. The marker is + /// cleared by the next successful publish, which is what makes a repaired + /// configuration take effect without a restart. + pub async fn mark_targets_unreadable(&self, bucket: &str) { + let update_mutex = self.target_update_mutex(bucket).await; + let _update_guard = update_mutex.lock().await; + + self.unreadable_targets.write().await.insert(bucket.to_string()); + } + pub async fn delete(&self, bucket: &str) { let update_mutex = self.target_update_mutex(bucket).await; let _update_guard = update_mutex.lock().await; - // Lock order: targets_map, then arn_remotes_map, then target_h_mutex, - // then ssec_passthrough_map (always last; also taken standalone by the - // capability accessors). + // Lock order: unreadable_targets, then targets_map, then + // arn_remotes_map, then target_h_mutex, then ssec_passthrough_map + // (always last; also taken standalone by the capability accessors). + self.unreadable_targets.write().await.remove(bucket); + let mut targets_map = self.targets_map.write().await; let mut arn_remotes_map = self.arn_remotes_map.write().await; let mut health_map = self.target_h_mutex.write().await; @@ -1093,6 +1132,11 @@ impl BucketTargetSys { /// Keeping persisted-config reads under the same mutex prevents a stale /// reload from overwriting a concurrent credential rotation. async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) { + // Reaching here means the persisted configuration decoded, so the + // unreadable marker (if any) is stale. Cleared before the maps below + // so `unreadable_targets` stays the outermost of this module's locks. + self.unreadable_targets.write().await.remove(bucket); + let mut clients = Vec::new(); if let Some(new_targets) = targets { for target in &new_targets.targets { @@ -1100,9 +1144,9 @@ impl BucketTargetSys { } } - // Lock order: targets_map, then arn_remotes_map, then target_h_mutex, - // then ssec_passthrough_map (always last; also taken standalone by the - // capability accessors). + // Lock order: unreadable_targets (above), then targets_map, then + // arn_remotes_map, then target_h_mutex, then ssec_passthrough_map + // (always last; also taken standalone by the capability accessors). let mut targets_map = self.targets_map.write().await; let mut arn_remotes_map = self.arn_remotes_map.write().await; let mut health_map = self.target_h_mutex.write().await; @@ -1161,6 +1205,11 @@ impl BucketTargetSys { } pub async fn set(&self, bucket: &str, meta: &BucketMetadata) { + if meta.bucket_targets_unreadable() { + self.mark_targets_unreadable(bucket).await; + return; + } + let Some(config) = &meta.bucket_target_config else { return; }; @@ -2276,6 +2325,13 @@ pub enum BucketTargetError { BucketRemoteTargetNotFound { bucket: String, }, + /// The bucket's persisted targets configuration exists but cannot be + /// decoded. Distinct from `BucketRemoteTargetNotFound`, which means the + /// bucket genuinely has no targets: callers must not degrade this one to + /// an empty target set (rustfs/backlog#2282). + BucketRemoteTargetsUnreadable { + bucket: String, + }, BucketRemoteArnTypeInvalid { bucket: String, }, @@ -2309,6 +2365,9 @@ impl fmt::Display for BucketTargetError { BucketTargetError::BucketRemoteTargetNotFound { bucket } => { write!(f, "Remote target not found for bucket: {bucket}") } + BucketTargetError::BucketRemoteTargetsUnreadable { bucket } => { + write!(f, "Persisted replication target configuration is unreadable for bucket: {bucket}") + } BucketTargetError::BucketRemoteArnTypeInvalid { bucket } => { write!(f, "Invalid ARN type for bucket: {bucket}") } @@ -3256,7 +3315,7 @@ mod tests { }], ); - let targets = sys.list_targets("", "").await; + let targets = sys.list_targets("", "").await.expect("listing every bucket's targets"); assert_eq!(targets.len(), 1); assert!(!targets[0].online); diff --git a/crates/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs index bd10f0f63..41b2afdc5 100644 --- a/crates/ecstore/src/bucket/metadata.rs +++ b/crates/ecstore/src/bucket/metadata.rs @@ -477,6 +477,18 @@ impl BucketMetadata { !self.table_bucket_config_json.is_empty() } + /// `bucket-targets.json` is stored for this bucket but this build cannot + /// decode it. + /// + /// Keeps "no replication targets configured" and "the target + /// configuration cannot be read" apart, the same distinction the + /// `fabricated` marker draws for the bucket metadata as a whole. Only + /// meaningful after [`Self::parse_all_configs`] has run; readers must fail + /// closed on `true` instead of serving an empty target set. + pub fn bucket_targets_unreadable(&self) -> bool { + !self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none() + } + /// Parsed per-bucket durability override, if a valid one is stored. /// /// Absent/empty/unparsable payloads all mean "no override" (the bucket @@ -964,7 +976,32 @@ impl BucketMetadata { Ok(()) } - fn parse_all_configs(&mut self) -> Result<()> { + /// Decode every stored sub-configuration into its typed field. + /// + /// A decode failure never fails the whole load: this runs on every bucket + /// metadata read, including startup and peer reload, so one bucket's + /// corrupt sub-configuration must not make the bucket — or the node — + /// unloadable. Instead the failure is *retained*: the raw bytes stay + /// untouched and the typed field stays `None`, so `!raw.is_empty() && + /// typed.is_none()` is the durable "exists but cannot be read" signal that + /// each accessor keys off. Which accessors must fail closed on it: + /// + /// | Config | Verdict | + /// |---|---| + /// | policy | Fails closed: `get_bucket_policy` re-parses the raw JSON and propagates the error; `get_bucket_policy_raw` returns the stored bytes. | + /// | object lock | Fails closed in `object_lock_config_state_from_authoritative_metadata`; a retention decision may never be taken on a guess. | + /// | versioning | Fails closed in `get_versioning_config`; guessing Unversioned would make delete markers and version ids diverge from what is on disk. | + /// | replication | Fails closed in `get_replication_config`. | + /// | bucket targets | Fails closed in `get_bucket_targets_config`, and `sync_bucket_target_sys` marks the bucket unreadable in `BucketTargetSys` instead of publishing an empty target set (rustfs/backlog#2282). | + /// | encryption | Fails closed in `get_sse_config`: degrading to "no default encryption" stores plaintext objects the operator required to be encrypted. | + /// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. | + /// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. | + /// | lifecycle | Safe to degrade: no rules means no expiration and no transition, so nothing is deleted or moved on the strength of an unreadable rule set. The bucket keeps serving reads and writes. | + /// | notification | Safe to degrade: events are an outbound side channel; no consumer draws a durability or authorization conclusion from their absence. | + /// | tagging | Safe to degrade: bucket tags are cost-allocation labels here; object-level tag conditions come from object metadata, not this blob. | + /// | CORS | Safe to degrade: an absent CORS configuration rejects cross-origin browser requests, which is already the restrictive direction. | + /// | logging, website, accelerate, request payment, bucket ACL | Safe to degrade: each only shapes an optional response or an optional side channel, and none of them authorizes an action or decides whether data is retained. | + pub(super) fn parse_all_configs(&mut self) -> Result<()> { if let Err(e) = self.parse_policy_config() { tracing::warn!( event = "bucket_metadata_parse_failed", @@ -1088,20 +1125,26 @@ impl BucketMetadata { "Failed to parse bucket metadata config" ); } + // A stored targets blob that cannot be decoded must not collapse into + // the empty target set: that is indistinguishable from "no replication + // configured", so replication stops and no caller ever sees an error + // (rustfs/backlog#2282). Leaving the typed field `None` while the raw + // bytes stay non-empty is the retained parse failure every targets + // reader keys off; the bytes are preserved so the configuration is + // still recoverable. + self.bucket_target_config = None; if !self.bucket_targets_config_json.is_empty() { - if let Err(e) = serde_json::from_slice::(&self.bucket_targets_config_json) - .map(|t| self.bucket_target_config = Some(t)) - { - tracing::warn!( + match serde_json::from_slice::(&self.bucket_targets_config_json) { + Ok(targets) => self.bucket_target_config = Some(targets), + Err(e) => tracing::error!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "bucket_targets", error = %e, - "Failed to parse bucket metadata config" - ); - self.bucket_target_config = Some(BucketTargets::default()); + "Bucket replication targets are unreadable; replication for this bucket fails closed" + ), } } else { self.bucket_target_config = Some(BucketTargets::default()); @@ -1535,6 +1578,117 @@ mod test { assert_eq!(bucket_targets.targets[0].target_bucket, "target-bucket"); } + /// rustfs/backlog#2282: a stored targets blob this build cannot decode + /// must not become the empty target set, and must stay distinguishable + /// from a bucket that never configured a target. + #[test] + fn unreadable_bucket_targets_never_degrade_to_an_empty_target_set() { + let truncated = br#"{"targets":[{"endpoint":"s3.example.com","#.to_vec(); + let mut corrupt = BucketMetadata::new("corrupt-targets"); + corrupt.bucket_targets_config_json = truncated.clone(); + + corrupt + .parse_all_configs() + .expect("one unreadable sub-config must not fail the whole metadata load"); + + assert!( + corrupt.bucket_target_config.is_none(), + "an undecodable targets blob must not produce a target set at all" + ); + assert!(corrupt.bucket_targets_unreadable()); + assert_eq!( + corrupt.bucket_targets_config_json, truncated, + "the raw bytes must survive so the configuration stays recoverable" + ); + + // The genuinely-absent case is unchanged, and the two now diverge. + let mut absent = BucketMetadata::new("no-targets"); + absent.parse_all_configs().expect("absent targets parse"); + assert!( + absent.bucket_target_config.as_ref().is_some_and(BucketTargets::is_empty), + "a bucket that configured no target still reads as an empty target set" + ); + assert!(!absent.bucket_targets_unreadable()); + } + + /// `Credentials` carries no struct-level `serde(default)`, so one target + /// missing `secretKey` is a hard parse error for the whole document. That + /// must surface as "unreadable", never as "no targets configured". + #[test] + fn bucket_targets_missing_secret_key_are_unreadable_not_empty() { + let mut bm = BucketMetadata::new("missing-secret-key"); + bm.bucket_targets_config_json = br#"{"targets":[{"endpoint":"s3.example.com","targetbucket":"remote","arn":"arn:rustfs:replication:us-east-1:src:1","credentials":{"accessKey":"AKIAEXAMPLE"}}]}"#.to_vec(); + + bm.parse_all_configs() + .expect("a rejected targets document must not fail the whole metadata load"); + + assert!( + bm.bucket_targets_unreadable(), + "a targets document rejected for a missing secretKey is unreadable, not empty" + ); + assert!(bm.bucket_target_config.is_none()); + } + + /// The invariant every branch of `parse_all_configs` shares: a stored but + /// undecodable payload keeps its raw bytes and leaves the typed field + /// `None`, so no branch fabricates a value. What a reader may then do with + /// that state is decided per config; see the table on `parse_all_configs`. + #[test] + fn every_config_branch_retains_its_parse_failure_instead_of_defaulting() { + let malformed_xml = b">) { } async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) { + if bm.bucket_targets_unreadable() { + // "The configuration cannot be read" is not "no targets configured". + // Publishing an empty snapshot here is what silently stopped + // replication (rustfs/backlog#2282): mark the bucket instead, so every + // targets reader gets a typed error, and leave any snapshot from an + // earlier readable load in place rather than withdrawing it. + BucketTargetSys::get().mark_targets_unreadable(bucket).await; + return; + } + BucketTargetSys::get() .update_all_targets(bucket, bm.bucket_target_config.as_ref()) .await; @@ -2118,7 +2128,9 @@ impl BucketMetadataSys { pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> { let (bm, _) = self.get_config(bucket).await?; - if let Some(config) = &bm.public_access_block_config { + if !bm.public_access_block_config_xml.is_empty() && bm.public_access_block_config.is_none() { + Err(Error::other("persisted bucket public access block configuration is invalid")) + } else if let Some(config) = &bm.public_access_block_config { Ok((config.clone(), bm.public_access_block_config_updated_at)) } else { Err(Error::ConfigNotFound) @@ -2429,7 +2441,9 @@ impl BucketMetadataSys { pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> { let (bm, _) = self.get_config(bucket).await?; - if let Some(config) = &bm.sse_config { + if !bm.encryption_config_xml.is_empty() && bm.sse_config.is_none() { + Err(Error::other("persisted bucket encryption configuration is invalid")) + } else if let Some(config) = &bm.sse_config { Ok((config.clone(), bm.encryption_config_updated_at)) } else { Err(Error::ConfigNotFound) @@ -2500,7 +2514,9 @@ impl BucketMetadataSys { pub async fn get_quota_config(&self, bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> { let (bm, _) = self.get_config(bucket).await?; - if let Some(config) = &bm.quota_config { + if !bm.quota_config_json.is_empty() && bm.quota_config.is_none() { + Err(Error::other("persisted bucket quota configuration is invalid")) + } else if let Some(config) = &bm.quota_config { Ok((config.clone(), bm.quota_config_updated_at)) } else { Err(Error::ConfigNotFound) @@ -2522,7 +2538,9 @@ impl BucketMetadataSys { pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result { let (bm, _) = self.get_config(bucket).await?; - if let Some(config) = &bm.bucket_target_config { + if bm.bucket_targets_unreadable() { + Err(Error::other("persisted bucket replication target configuration is invalid")) + } else if let Some(config) = &bm.bucket_target_config { Ok(config.clone()) } else { Err(Error::ConfigNotFound) @@ -2593,6 +2611,7 @@ pub(crate) mod test_support { mod tests { use super::test_support::isolated_store_over_temp_disks; use super::*; + use crate::bucket::bucket_target_sys::BucketTargetError; use crate::bucket::metadata::{ BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, @@ -2788,6 +2807,36 @@ mod tests { ); } + /// The `parse_all_configs` audit (rustfs/backlog#2282): every accessor + /// whose configuration grants something — plaintext storage, anonymous + /// access, capacity, replication targets — reports a corrupt payload as + /// invalid rather than as absent, because "absent" is what grants it. + #[tokio::test] + async fn malformed_permissive_configs_are_not_reported_as_absent() { + let (_dirs, ecstore) = isolated_store_over_temp_disks().await; + let sys = BucketMetadataSys::new(ecstore); + let bucket = "malformed-permissive-config"; + let mut metadata = BucketMetadata::new(bucket); + metadata.encryption_config_xml = b" Some(targets), + // A bucket whose persisted target configuration cannot be decoded has + // an unknown target set, not an empty one: scheduling against `None` + // here would drop every heal for it without a trace + // (rustfs/backlog#2282). Report it missed so the object is retried + // once the configuration is readable again. + Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) => { + warn!( + event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION, + bucket, + reason = "target_config_unreadable", + "Bucket replication targets are unreadable; replication heal queue fails closed" + ); + + return ReplicationQueueAdmission::Missed; + } Err(err) => { debug!( event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index a5cbf6b34..f7531319e 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -15,7 +15,8 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys}; +pub(crate) use crate::bucket::bucket_target_sys::BucketTargetError; +use crate::bucket::bucket_target_sys::BucketTargetSys; use aws_sdk_s3::operation::head_object::HeadObjectOutput; use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode}; use http::HeaderMap; diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index b2b2b80dc..b7bf5ef78 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -121,6 +121,11 @@ fn map_bucket_target_error(err: BucketTargetError) -> S3Error { | BucketTargetError::BucketRemoteRemoveDisallowed { .. } => { S3Error::with_message(S3ErrorCode::InvalidRequest, err.to_string()) } + // A stored target configuration this node cannot decode is a + // server-side data fault, not a bad request (rustfs/backlog#2282). + BucketTargetError::BucketRemoteTargetsUnreadable { .. } => { + S3Error::with_message(S3ErrorCode::InternalError, err.to_string()) + } BucketTargetError::Io(io_err) => S3Error::with_message(S3ErrorCode::InternalError, io_err.to_string()), } } @@ -753,7 +758,12 @@ impl Operation for ListRemoteTargetHandler { .map_err(ApiError::from)?; let sys = BucketTargetSys::get(); - let targets = sys.list_targets(bucket, "").await; + // An unreadable targets configuration must not be reported as an + // empty target list (rustfs/backlog#2282). + let targets = sys.list_targets(bucket, "").await.map_err(|e| { + error!("list remote targets failed: {}", e); + map_bucket_target_error(e) + })?; let targets: Vec<_> = targets .iter() From 4dbc58887afe4c0e38dc11704f403479a9f7c79e Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 5 Sep 2026 14:00:14 +0800 Subject: [PATCH 02/40] fix(tier): probe legacy transition version state (#7138) --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 782 +++++++++++++++++- crates/ecstore/src/services/tier/test_util.rs | 5 +- crates/ecstore/src/services/tier/tier.rs | 13 + .../ecstore/src/services/tier/warm_backend.rs | 53 +- .../src/services/tier/warm_backend_s3.rs | 43 +- crates/ecstore/src/store/init.rs | 160 +++- crates/filemeta/src/filemeta/version.rs | 125 ++- 7 files changed, 1133 insertions(+), 48 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 7649e3ac9..dbb5dd154 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -584,33 +584,173 @@ impl ExpiryOp for FreeVersionTask { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TransitionDeleteVersionPlan { + Direct { version_id_exact: bool }, + ProbeLegacyUnknown, +} + +fn legacy_transition_version_state_missing(oi: &ObjectInfo) -> Result { + use rustfs_utils::http::metadata_compat::{ + SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_str, get_consistent_str, + }; + + if !contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE) { + let version_key_present = contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID); + if version_key_present { + if oi.transitioned_object.version_id.is_empty() { + let has_non_empty_version = oi.user_defined.iter().any(|(key, value)| { + rustfs_utils::http::metadata_compat::strip_internal_prefix_preserving_case(key) + .is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID)) + && !value.is_empty() + }); + if !has_non_empty_version { + // MinIO writes the transitioned-versionID key with an empty value + // for unversioned tier objects. The backend probe remains the proof. + return Ok(true); + } + } else if get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID) + == Some(oi.transitioned_object.version_id.as_str()) + { + return Ok(true); + } + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "legacy remote tier version metadata is conflicting or malformed", + )); + } + if !oi.transitioned_object.version_id.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "legacy remote tier version metadata is missing or inconsistent", + )); + } + return Ok(true); + } + let persisted = get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote tier object has conflicting transition version state metadata", + ) + })?; + if persisted != oi.transition_version_state.as_str() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote tier object transition version state metadata changed during decoding", + )); + } + Ok(false) +} + +fn transition_remote_version_delete_plan(oi: &ObjectInfo) -> Result { + match oi.transition_version_state { + rustfs_filemeta::TransitionVersionState::Unknown => { + if legacy_transition_version_state_missing(oi)? { + Ok(TransitionDeleteVersionPlan::ProbeLegacyUnknown) + } else { + validate_transition_remote_version(oi) + .map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact }) + } + } + _ => validate_transition_remote_version(oi) + .map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact }), + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ResolvedTransitionDeleteVersion { + version_id_exact: bool, + remote_already_missing: bool, +} + async fn acquire_free_version_tier_lease( oi: &ObjectInfo, tier_config_mgr: &Arc>, -) -> Result<(TierOperationLease, bool), std::io::Error> { - let version_id_exact = validate_transition_remote_version(oi)?; +) -> Result<(TierOperationLease, TransitionDeleteVersionPlan), std::io::Error> { + let delete_plan = transition_remote_version_delete_plan(oi)?; let identity = tier_destination_id_from_metadata(&oi.user_defined)? .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?; let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity) .await .map_err(std::io::Error::other)?; - Ok((lease, version_id_exact)) + Ok((lease, delete_plan)) +} + +async fn resolve_transition_delete_version_plan( + oi: &ObjectInfo, + lease: &TierOperationLease, + delete_plan: TransitionDeleteVersionPlan, +) -> Result { + match delete_plan { + TransitionDeleteVersionPlan::Direct { version_id_exact } => Ok(ResolvedTransitionDeleteVersion { + version_id_exact, + remote_already_missing: false, + }), + TransitionDeleteVersionPlan::ProbeLegacyUnknown => { + let expected_version = oi.transitioned_object.version_id.as_str(); + if expected_version.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "remote tier cannot safely delete a legacy object without an exact version ID", + )); + } + let probe = lease + .probe_transition_version(&oi.transitioned_object.name, expected_version) + .await?; + match (expected_version, probe) { + (expected, crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(actual)) + if expected == actual => + { + lease.validate_remote_version_id(expected)?; + Ok(ResolvedTransitionDeleteVersion { + version_id_exact: true, + remote_already_missing: false, + }) + } + (_, crate::services::tier::warm_backend::TransitionCandidateProbe::Missing) => { + Ok(ResolvedTransitionDeleteVersion { + version_id_exact: false, + remote_already_missing: true, + }) + } + (_, crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported) => Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "remote tier cannot prove legacy transition delete state", + )), + _ => Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "remote tier object version state is unknown", + )), + } + } + } +} + +async fn execute_resolved_transition_delete( + oi: &ObjectInfo, + lease: &TierOperationLease, + resolved: ResolvedTransitionDeleteVersion, +) -> Result<(), std::io::Error> { + if !resolved.remote_already_missing { + delete_object_from_remote_tier_with_lease_idempotent( + &oi.transitioned_object.name, + &oi.transitioned_object.version_id, + lease, + resolved.version_id_exact, + ) + .await?; + } + Ok(()) } async fn delete_free_version_remote_object_with_lease( oi: &ObjectInfo, lease: &TierOperationLease, - version_id_exact: bool, + delete_plan: TransitionDeleteVersionPlan, ) -> Result<(), std::io::Error> { - delete_object_from_remote_tier_with_lease_idempotent( - &oi.transitioned_object.name, - &oi.transitioned_object.version_id, - lease, - version_id_exact, - ) - .await?; - Ok(()) + let resolved = resolve_transition_delete_version_plan(oi, lease, delete_plan).await?; + execute_resolved_transition_delete(oi, lease, resolved).await } fn free_version_physical_topology_generation(api: &ECStore) -> String { @@ -641,6 +781,16 @@ fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectIn if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown || expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown { + let candidate_legacy_missing = legacy_transition_version_state_missing(candidate)?; + let expected_legacy_missing = legacy_transition_version_state_missing(expected)?; + if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown + && expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown + && candidate_legacy_missing + && expected_legacy_missing + && candidate.transitioned_object.version_id == expected.transitioned_object.version_id + { + return Ok(true); + } return Err(std::io::Error::new( std::io::ErrorKind::WouldBlock, "tier free-version remote version state is unknown", @@ -716,7 +866,7 @@ async fn cleanup_free_version_exact(api: Arc, oi: &ObjectInfo, cancel: .acquire_bucket_lifecycle_read_lock(&oi.bucket) .await .map_err(std::io::Error::other)?; - let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?; + let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?; let local_object = encode_dir_object(&oi.name); let object_guards = api .acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object) @@ -734,16 +884,30 @@ async fn cleanup_free_version_exact(api: Arc, oi: &ObjectInfo, cancel: "tier free-version cleanup fence is invalid before remote delete", )); } + let resolved = tokio::select! { + _ = cancel.cancelled() => { + return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled")); + } + result = tokio::time::timeout_at(deadline, resolve_transition_delete_version_plan(oi, &lease, delete_plan)) => { + result.map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote probe timed out") + })?? + } + }; + if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier free-version cleanup fence changed after remote probe", + )); + } tokio::select! { _ = cancel.cancelled() => { return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled")); } - result = tokio::time::timeout_at( - deadline, - delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact), - ) => { - result - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??; + result = tokio::time::timeout_at(deadline, execute_resolved_transition_delete(oi, &lease, resolved)) => { + result.map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out") + })??; } } if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) { @@ -791,8 +955,8 @@ async fn delete_free_version_remote_object( oi: &ObjectInfo, tier_config_mgr: &Arc>, ) -> Result<(), std::io::Error> { - let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; - delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await + let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; + delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await } #[allow( @@ -808,8 +972,8 @@ where F: FnOnce() -> Fut, Fut: std::future::Future, { - let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; - delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?; + let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; + delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await?; let result = delete_local().await; drop(lease); Ok(result) @@ -4688,6 +4852,39 @@ fn validate_transition_remote_version(oi: &ObjectInfo) -> Result Result { + let version = oi.transitioned_object.version_id.as_str(); + match oi.transition_version_state { + rustfs_filemeta::TransitionVersionState::Unknown => { + if !legacy_transition_version_state_missing(oi)? { + return validate_transition_remote_version(oi).map(|_| TransitionReadVersionPlan::Direct); + } + if version.is_empty() { + Ok(TransitionReadVersionPlan::ProbeLegacyUnversioned) + } else { + Ok(TransitionReadVersionPlan::Direct) + } + } + rustfs_filemeta::TransitionVersionState::KnownDisabled if version.is_empty() => Ok(TransitionReadVersionPlan::Direct), + rustfs_filemeta::TransitionVersionState::SuspendedNull if version == "null" => Ok(TransitionReadVersionPlan::Direct), + rustfs_filemeta::TransitionVersionState::Exact if !version.is_empty() && version != "null" => { + Ok(TransitionReadVersionPlan::Direct) + } + _ => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote tier object version state conflicts with its version ID", + )), + } +} + // The resolver joins the tier manager as the second injected port this read // needs; grouping the request half into a struct would churn every call site of // a bug fix. @@ -4702,7 +4899,12 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager( tier_config_mgr: &Arc>, resolver: Option<&dyn ObjectEncryptionResolver>, ) -> Result { - validate_transition_remote_version(oi)?; + let read_plan = transition_remote_version_read_plan(oi)?; + // Reject invalid ranges and encryption requests before a compatibility + // probe can amplify them into remote listing work. + let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver) + .await + .map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?; let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?; let lease = match expected_identity { Some(identity) => { @@ -4716,7 +4918,36 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager( Err(err) => return Err(std::io::Error::other(err)), }; - tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?; + match read_plan { + TransitionReadVersionPlan::Direct => { + tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?; + } + TransitionReadVersionPlan::ProbeLegacyUnversioned => { + // RUSTFS_COMPAT_TODO(backlog#2203): remove operation-time probing + // after an admin reconcile can persist every proven legacy state. + let probe = tokio::time::timeout( + LEGACY_TRANSITION_READ_PROBE_TIMEOUT, + tgt_client.probe_transition_candidate(&oi.transitioned_object.name), + ) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "legacy remote tier version probe timed out"))??; + match probe { + crate::services::tier::warm_backend::TransitionCandidateProbe::UnversionedPresent => {} + crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported => { + return Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "remote tier cannot prove legacy unversioned transition state", + )); + } + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote tier object version state is unknown", + )); + } + } + } + } // The same read plan the local path uses, so the tier fetch is positioned in // the object's *stored* coordinate system and the stream is handed the same @@ -4724,9 +4955,6 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager( // through a plaintext-coordinate range and skipping the transform is how a // transitioned SSE object used to come back as silently corrupt bytes of the // right length (rustfs/rustfs#6025). - let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver) - .await - .map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?; let (off, length) = (plan.storage_offset() as i64, plan.storage_length()); let mut gopts = WarmBackendGetOpts::default(); @@ -5599,11 +5827,13 @@ mod tests { use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader}; #[cfg(feature = "test-util")] + use crate::services::tier::test_util::MockWarmOp; + #[cfg(feature = "test-util")] use crate::services::tier::test_util::register_mock_tier; #[cfg(feature = "test-util")] use crate::services::tier::tier::TierConfigMgr; #[cfg(feature = "test-util")] - use crate::services::tier::warm_backend::WarmBackend as _; + use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _}; use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause}; use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY}; use crate::storage_api_contracts::namespace::NamespaceLocking as _; @@ -6299,7 +6529,75 @@ mod tests { #[cfg(feature = "test-util")] #[tokio::test] - async fn transitioned_get_rejects_unknown_version_state_before_backend_io() { + async fn transitioned_get_allows_legacy_unknown_exact_version_for_non_destructive_read() { + let manager = TierConfigMgr::new(); + let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&manager, &tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + let body = Bytes::from_static(b"legacy transitioned object body"); + let remote_version = backend + .put( + &remote_object, + ReaderImpl::Body(body.clone()), + i64::try_from(body.len()).expect("body length should fit"), + ) + .await + .expect("mock remote object should be stored"); + let mut user_defined = HashMap::new(); + insert_legacy_transition_version_id(&mut user_defined, &remote_version); + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: i64::try_from(body.len()).expect("body length should fit"), + transitioned_object: TransitionedObject { + name: remote_object, + version_id: remote_version, + status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(), + tier: tier.clone(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + let range = Some(crate::storage_api_contracts::range::HTTPRangeSpec { + is_suffix_length: false, + start: 7, + end: 18, + }); + let mut reader = get_transitioned_object_reader_with_tier_manager( + &object_info.bucket, + &object_info.name, + &range, + &HeaderMap::new(), + &object_info, + &ObjectOptions::default(), + &manager, + None, + ) + .await + .expect("legacy unknown state should still allow a non-destructive read"); + let mut got = Vec::new(); + reader + .stream + .read_to_end(&mut got) + .await + .expect("transitioned reader should drain"); + + assert_eq!(got, &body.as_ref()[7..=18]); + assert_eq!(backend.get_count().await, 1); + assert_eq!(backend.remove_count().await, 0); + assert_eq!( + TierConfigMgr::active_operation_lease_count(&manager, &tier).await, + 0, + "tier generation lease should release after EOF" + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn transitioned_get_rejects_explicit_unknown_version_state_before_backend_io() { let manager = TierConfigMgr::new(); let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); let backend = register_mock_tier(&manager, &tier).await; @@ -6315,6 +6613,181 @@ mod tests { ..Default::default() }, transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined_with_transition_version_state(rustfs_filemeta::TransitionVersionState::Unknown).into(), + ..Default::default() + }; + + let err = match get_transitioned_object_reader_with_tier_manager( + &object_info.bucket, + &object_info.name, + &None, + &HeaderMap::new(), + &object_info, + &ObjectOptions::default(), + &manager, + None, + ) + .await + { + Ok(_) => panic!("explicit unknown remote version state must fail before backend IO"), + Err(err) => err, + }; + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(backend.op_log().await, Vec::::new()); + assert_eq!(backend.get_count().await, 0); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn transitioned_get_rejects_present_but_invalid_legacy_version_metadata() { + let manager = TierConfigMgr::new(); + let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&manager, &tier).await; + + for persisted_version in [ + Uuid::nil().to_string(), + "\u{fffd}".to_string(), + "bad\u{0001}version".to_string(), + ] { + let mut user_defined = HashMap::new(); + insert_legacy_transition_version_id(&mut user_defined, &persisted_version); + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: 1, + transitioned_object: TransitionedObject { + name: "remote/object".to_string(), + version_id: String::new(), + status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(), + tier: tier.clone(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + let err = match get_transitioned_object_reader_with_tier_manager( + &object_info.bucket, + &object_info.name, + &None, + &HeaderMap::new(), + &object_info, + &ObjectOptions::default(), + &manager, + None, + ) + .await + { + Ok(_) => panic!("present but invalid legacy version metadata must fail before backend IO"), + Err(err) => err, + }; + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + } + + assert_eq!(backend.op_log().await, Vec::::new()); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn transitioned_get_probes_legacy_empty_unknown_state_before_unversioned_read() { + let manager = TierConfigMgr::new(); + let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&manager, &tier).await; + backend.set_put_remote_version(Some(String::new())).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + let body = Bytes::from_static(b"legacy unversioned transitioned object body"); + let remote_version = backend + .put( + &remote_object, + ReaderImpl::Body(body.clone()), + i64::try_from(body.len()).expect("body length should fit"), + ) + .await + .expect("mock remote object should be stored"); + assert!(remote_version.is_empty()); + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: i64::try_from(body.len()).expect("body length should fit"), + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: String::new(), + status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(), + tier: tier.clone(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: HashMap::from([("x-minio-internal-transitioned-versionID".to_string(), String::new())]).into(), + ..Default::default() + }; + + let mut reader = get_transitioned_object_reader_with_tier_manager( + &object_info.bucket, + &object_info.name, + &None, + &HeaderMap::new(), + &object_info, + &ObjectOptions::default(), + &manager, + None, + ) + .await + .expect("probe-proven legacy unversioned state should allow a non-destructive read"); + let mut got = Vec::new(); + reader + .stream + .read_to_end(&mut got) + .await + .expect("transitioned reader should drain"); + + assert_eq!(got, body.as_ref()); + assert_eq!(backend.remove_count().await, 0); + assert_eq!( + backend.op_log().await, + vec![ + MockWarmOp::Put { + object: remote_object.clone() + }, + MockWarmOp::Probe { + object: remote_object.clone() + }, + MockWarmOp::Get { object: remote_object }, + ] + ); + assert_eq!( + TierConfigMgr::active_operation_lease_count(&manager, &tier).await, + 0, + "tier generation lease should release after EOF" + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn transitioned_get_rejects_ambiguous_empty_unknown_state_without_backend_get() { + let manager = TierConfigMgr::new(); + let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&manager, &tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + backend + .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent( + "versioned-candidate".to_string(), + ))) + .await; + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: 1, + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: String::new(), + status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(), + tier, + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, ..Default::default() }; @@ -6330,19 +6803,28 @@ mod tests { ) .await { - Ok(_) => panic!("unknown remote version state must fail before backend IO"), + Ok(_) => panic!("versioned legacy unknown state without stored version must fail before backend GET"), Err(err) => err, }; assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(backend.op_log().await, vec![MockWarmOp::Probe { object: remote_object }]); assert_eq!(backend.get_count().await, 0); + assert_eq!(backend.remove_count().await, 0); } #[cfg(feature = "test-util")] #[tokio::test] - async fn free_version_delete_rejects_unknown_version_state_before_backend_io() { + async fn free_version_delete_rejects_explicit_unknown_before_backend_io() { let manager = TierConfigMgr::new(); let backend = register_mock_tier(&manager, "WARM").await; + let identity = test_tier_destination_identity(&manager, "WARM").await; + let mut user_defined = user_defined_with_tier_destination_identity(identity); + rustfs_utils::http::metadata_compat::insert_str( + &mut user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(), + ); let object_info = ObjectInfo { transitioned_object: TransitionedObject { name: "remote/object".to_string(), @@ -6351,17 +6833,251 @@ mod tests { ..Default::default() }, transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), ..Default::default() }; let err = super::delete_free_version_remote_object(&object_info, &manager) .await - .expect_err("unknown remote version state must fail before backend IO"); + .expect_err("explicit unknown cleanup must fail before backend IO"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("version state is unknown")); + assert_eq!(backend.op_log().await, Vec::::new()); assert_eq!(backend.remove_count().await, 0); } + #[cfg(feature = "test-util")] + async fn test_tier_destination_identity( + manager: &Arc>, + tier: &str, + ) -> crate::services::tier::tier::TierDestinationId { + TierConfigMgr::acquire_operation_lease(manager, tier) + .await + .expect("test tier lease should be available") + .backend_identity() + } + + #[cfg(feature = "test-util")] + fn user_defined_with_tier_destination_identity( + identity: crate::services::tier::tier::TierDestinationId, + ) -> HashMap { + let mut user_defined = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(identity), + ); + user_defined + } + + #[cfg(feature = "test-util")] + fn user_defined_with_transition_version_state(state: rustfs_filemeta::TransitionVersionState) -> HashMap { + let mut user_defined = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + state.as_str().to_string(), + ); + user_defined + } + + #[cfg(feature = "test-util")] + fn insert_legacy_transition_version_id(user_defined: &mut HashMap, version_id: &str) { + rustfs_utils::http::metadata_compat::insert_str( + user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID, + version_id.to_string(), + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn free_version_tuple_rejects_mixed_legacy_missing_and_explicit_unknown() { + let manager = TierConfigMgr::new(); + register_mock_tier(&manager, "WARM").await; + let identity = test_tier_destination_identity(&manager, "WARM").await; + let mut legacy_metadata = user_defined_with_tier_destination_identity(identity); + insert_legacy_transition_version_id(&mut legacy_metadata, "legacy-version"); + let mut explicit_metadata = legacy_metadata.clone(); + rustfs_utils::http::metadata_compat::insert_str( + &mut explicit_metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(), + ); + let make_info = |user_defined: HashMap| ObjectInfo { + transitioned_object: TransitionedObject { + name: "remote/object".to_string(), + version_id: "legacy-version".to_string(), + tier: "WARM".to_string(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + let err = super::free_version_remote_tuple_matches(&make_info(legacy_metadata), &make_info(explicit_metadata)) + .expect_err("mixed legacy-missing and explicit unknown provenance must fail closed"); + + assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn free_version_delete_probes_exact_version_hidden_by_current_delete_marker() { + let manager = TierConfigMgr::new(); + let tier = "WARM"; + let backend = register_mock_tier(&manager, tier).await; + let identity = test_tier_destination_identity(&manager, tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + let body = Bytes::from_static(b"legacy exact cleanup body"); + let remote_version = backend + .put( + &remote_object, + ReaderImpl::Body(body), + i64::try_from(b"legacy exact cleanup body".len()).expect("body length should fit"), + ) + .await + .expect("mock remote object should be stored"); + let mut user_defined = user_defined_with_tier_destination_identity(identity); + insert_legacy_transition_version_id(&mut user_defined, &remote_version); + backend + .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Missing)) + .await; + assert_eq!( + backend + .probe_transition_candidate_state(&remote_object) + .await + .expect("current remote view should be readable"), + TransitionCandidateProbe::Missing, + "a current delete marker must hide the historical data version from an unversioned probe" + ); + backend.clear_op_log().await; + let object_info = ObjectInfo { + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: remote_version, + tier: tier.to_string(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + super::delete_free_version_remote_object(&object_info, &manager) + .await + .expect("probe-proven legacy exact cleanup should delete the remote version"); + super::delete_free_version_remote_object(&object_info, &manager) + .await + .expect("a retry after the exact remote version is already missing should be idempotent"); + + assert_eq!( + backend.op_log().await, + vec![ + MockWarmOp::Get { + object: remote_object.clone() + }, + MockWarmOp::Remove { + object: remote_object.clone() + }, + MockWarmOp::Get { + object: remote_object.clone() + }, + ] + ); + assert_eq!( + backend.remove_versions().await, + vec![(remote_object, object_info.transitioned_object.version_id)] + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn free_version_delete_retains_legacy_unknown_unversioned_object() { + let manager = TierConfigMgr::new(); + let tier = "WARM"; + let backend = register_mock_tier(&manager, tier).await; + backend.set_put_remote_version(Some(String::new())).await; + let identity = test_tier_destination_identity(&manager, tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + let body = Bytes::from_static(b"legacy unversioned cleanup body"); + let remote_version = backend + .put( + &remote_object, + ReaderImpl::Body(body), + i64::try_from(b"legacy unversioned cleanup body".len()).expect("body length should fit"), + ) + .await + .expect("mock remote object should be stored"); + assert!(remote_version.is_empty()); + backend.clear_op_log().await; + let mut user_defined = user_defined_with_tier_destination_identity(identity); + user_defined.insert("x-minio-internal-transitioned-versionID".to_string(), String::new()); + let object_info = ObjectInfo { + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: String::new(), + tier: tier.to_string(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + let err = super::delete_free_version_remote_object(&object_info, &manager) + .await + .expect_err("legacy unversioned cleanup cannot exclude a versioning-state race"); + + assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock); + assert!(backend.op_log().await.is_empty()); + assert_eq!(backend.remove_count().await, 0); + assert!(backend.remove_versions().await.is_empty()); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn free_version_delete_does_not_remove_a_different_remote_version() { + let manager = TierConfigMgr::new(); + let tier = "WARM"; + let backend = register_mock_tier(&manager, tier).await; + let identity = test_tier_destination_identity(&manager, tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + backend.set_put_remote_version(Some("different-version".to_string())).await; + backend + .put( + &remote_object, + ReaderImpl::Body(Bytes::from_static(b"different remote version")), + i64::try_from(b"different remote version".len()).expect("body length should fit"), + ) + .await + .expect("different remote version should be stored"); + backend.clear_op_log().await; + let mut user_defined = user_defined_with_tier_destination_identity(identity); + insert_legacy_transition_version_id(&mut user_defined, "legacy-version"); + let object_info = ObjectInfo { + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: "legacy-version".to_string(), + tier: tier.to_string(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + super::delete_free_version_remote_object(&object_info, &manager) + .await + .expect("a missing exact legacy version should be an idempotent cleanup success"); + + assert_eq!(backend.op_log().await, vec![MockWarmOp::Get { object: remote_object }]); + assert_eq!(backend.remove_count().await, 0); + assert!(backend.remove_versions().await.is_empty()); + } + #[cfg(feature = "test-util")] #[tokio::test] async fn free_version_remote_delete_requires_persisted_destination_identity() { diff --git a/crates/ecstore/src/services/tier/test_util.rs b/crates/ecstore/src/services/tier/test_util.rs index f2c4ddeb6..59b4d6f4e 100644 --- a/crates/ecstore/src/services/tier/test_util.rs +++ b/crates/ecstore/src/services/tier/test_util.rs @@ -701,7 +701,7 @@ impl WarmBackend for MockWarmBackend { Ok(version) } - async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result { + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { self.precondition().await?; let barrier = self.inner.get_barrier.lock().await.take(); if let Some(barrier) = barrier { @@ -719,6 +719,9 @@ impl WarmBackend for MockWarmBackend { let Some(stored) = objects.get(object) else { return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found")); }; + if !rv.is_empty() && stored.remote_version_id != rv { + return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion")); + } let bytes = &stored.bytes; let start = opts.start_offset.max(0) as usize; diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index 887015a1e..af24423fa 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -2346,6 +2346,10 @@ impl WarmBackend for SharedWarmBackendProxy { self.0.probe_transition_candidate(object).await } + async fn probe_transition_version(&self, object: &str, remote_version_id: &str) -> io::Result { + self.0.probe_transition_version(object, remote_version_id).await + } + async fn in_use(&self) -> io::Result { self.0.in_use().await } @@ -2458,6 +2462,15 @@ impl TierOperationLease { Ok(()) } + pub(crate) async fn probe_transition_version( + &self, + object: &str, + remote_version_id: &str, + ) -> io::Result { + self.validate_remote_version_id(remote_version_id)?; + self.inner.driver.probe_transition_version(object, remote_version_id).await + } + pub(crate) fn is_current_generation(&self) -> bool { lock_unpoisoned(&self.runtime) .generations diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index ee48c116c..b5cf4ab38 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -40,6 +40,7 @@ use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value}; use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore}; use rustfs_s3_client::{ admin_handler_utils::AdminError, + api_error_response::to_error_response, api_put_object::{AdvancedPutOptions, PutObjectOptions}, transition_api::{ReadCloser, ReaderImpl}, }; @@ -48,11 +49,14 @@ use rustfs_utils::egress::validate_outbound_url; use rustfs_utils::http::headers::{ CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _, }; -use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus}; use s3s::header::{ X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS, X_AMZ_STORAGE_CLASS, }; +use s3s::{ + S3ErrorCode, + dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus}, +}; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -141,6 +145,42 @@ pub trait WarmBackend { async fn probe_transition_candidate(&self, _object: &str) -> Result { Ok(TransitionCandidateProbe::Unsupported) } + async fn probe_transition_version( + &self, + object: &str, + remote_version_id: &str, + ) -> Result { + if remote_version_id.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "an exact tier probe requires a remote version ID", + )); + } + self.validate_remote_version_id(remote_version_id)?; + match self + .get( + object, + remote_version_id, + WarmBackendGetOpts { + start_offset: 0, + length: 1, + }, + ) + .await + { + Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())), + Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => { + Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())) + } + Err(err) + if err.kind() == std::io::ErrorKind::NotFound + || matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) => + { + Ok(TransitionCandidateProbe::Missing) + } + Err(err) => Err(err), + } + } async fn in_use(&self) -> Result; } @@ -437,6 +477,17 @@ impl WarmBackend for MeteredWarmBackend { Self::record(TierRequestOperation::Probe, result) } + async fn probe_transition_version( + &self, + object: &str, + remote_version_id: &str, + ) -> Result { + Self::record( + TierRequestOperation::Probe, + self.inner.probe_transition_version(object, remote_version_id).await, + ) + } + async fn in_use(&self) -> Result { Self::record(TierRequestOperation::InUse, self.inner.in_use().await) } diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index 5462fc52c..b830ea7f2 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -529,6 +529,10 @@ mod tests { "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\nNoSuchObjectmissing", "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\nAccessDenieddenied", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", + "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\nInvalidRangeempty version", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\nNoSuchVersionmissing", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", ]; let mut requests = Vec::new(); for response in responses { @@ -622,15 +626,52 @@ mod tests { .await .expect_err("an authorization failure must not be mistaken for a missing key"); assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied); + assert_eq!( + backend + .probe_transition_candidate("delete-marker-hidden") + .await + .expect("a current delete marker should hide the data version"), + TransitionCandidateProbe::Missing + ); + assert_eq!( + backend + .probe_transition_version("delete-marker-hidden", "historical-version") + .await + .expect("the stored historical version should be probed exactly"), + TransitionCandidateProbe::VersionedPresent("historical-version".to_string()) + ); + assert_eq!( + backend + .probe_transition_version("delete-marker-hidden", "missing-version") + .await + .expect("a missing exact version should be classified"), + TransitionCandidateProbe::Missing + ); + assert_eq!( + backend + .probe_transition_version("missing-object", "historical-version") + .await + .expect("a missing key for an exact version probe should be classified"), + TransitionCandidateProbe::Missing + ); let requests = fixture.await.expect("candidate fixture should join"); - for request in requests { + for request in &requests[..6] { let request = request.to_ascii_lowercase(); assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET"); assert!(request.contains("\r\nrange: bytes=0-0\r\n")); assert!(!request.contains("?versioning")); assert!(!request.contains("?versions")); } + for request in &requests[6..] { + let request = request.to_ascii_lowercase(); + assert!(request.starts_with("get /bucket/"), "exact discovery must use object GET"); + assert!(request.contains("\r\nrange: bytes=0-0\r\n")); + } + assert!(!requests[5].to_ascii_lowercase().contains("versionid=")); + assert!(requests[6].to_ascii_lowercase().contains("?versionid=historical-version")); + assert!(requests[7].to_ascii_lowercase().contains("?versionid=missing-version")); + assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version")); } fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 22d7e37c3..463f50a82 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -11575,6 +11575,7 @@ mod tests { pool_index: usize, bucket: &str, object: &str, + minio_unversioned: bool, ) { for disk_index in 0..4 { let metadata_path = @@ -11608,6 +11609,11 @@ mod tests { ] { rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix); } + if minio_unversioned { + object_meta + .meta_sys + .insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new()); + } *shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version) .expect("legacy transitioned version should re-encode"); } @@ -11618,6 +11624,152 @@ mod tests { } } + #[cfg(feature = "test-util")] + async fn read_store_body( + store: &Arc, + bucket: &str, + object: &str, + range: Option, + opts: &ObjectOptions, + ) -> Vec { + let mut reader = store + .get_object_reader(bucket, object, range, HeaderMap::new(), opts) + .await + .expect("object reader should open"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("object body should drain"); + body + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() { + let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + backend.set_put_remote_version(Some(String::new())).await; + let bucket = "legacy-unknown-unversioned-read-bucket"; + let object = "object.bin"; + let payload = b"legacy unversioned remote tier object remains readable".repeat(1024); + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("legacy source bucket should be created"); + let mut reader = PutObjReader::from_vec(payload.clone()); + let source = store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("legacy source should be written"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("legacy source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("legacy source should transition"); + rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await; + backend.clear_op_log().await; + + let opts = ObjectOptions { + metadata_cache_safe: false, + ..Default::default() + }; + let head = store + .get_object_info(bucket, object, &opts) + .await + .expect("legacy transitioned HEAD should use local metadata"); + assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown); + assert!(head.transitioned_object.version_id.is_empty()); + assert_eq!( + head.user_defined + .get("x-minio-internal-transitioned-versionID") + .map(String::as_str), + Some(""), + "the MinIO empty version-key provenance must survive xl.meta decoding" + ); + assert!( + !rustfs_utils::http::metadata_compat::contains_key_str( + &head.user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + ), + "the compatibility read must not synthesize version-state metadata" + ); + + let full_body = read_store_body(&store, bucket, object, None, &opts).await; + assert_eq!(full_body, payload); + + let range = HTTPRangeSpec { + is_suffix_length: false, + start: 7, + end: 38, + }; + let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await; + assert_eq!(ranged_body, &payload[7..=38]); + + let after_read = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("legacy metadata should remain readable after GET") + .expect("legacy object metadata should remain on disk") + .versions + .into_iter() + .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE) + .expect("legacy transitioned source should remain visible after GET"); + assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown); + assert!(after_read.transition_version.is_none()); + assert!(after_read.transition_version_id.is_none()); + assert_eq!( + after_read + .metadata + .get("x-minio-internal-transitioned-versionID") + .map(String::as_str), + Some(""), + "the MinIO empty version-key provenance must remain after GET and Range GET" + ); + assert!( + !rustfs_utils::http::metadata_compat::contains_key_str( + &after_read.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + ), + "the compatibility read must remain side-effect free" + ); + + assert_eq!( + backend.op_log().await, + vec![ + MockWarmOp::Probe { + object: after_read.transitioned_objname.clone(), + }, + MockWarmOp::Get { + object: after_read.transitioned_objname.clone(), + }, + MockWarmOp::Probe { + object: after_read.transitioned_objname.clone(), + }, + MockWarmOp::Get { + object: after_read.transitioned_objname, + }, + ], + "legacy reads should probe before each unversioned GET and never mutate local metadata" + ); + assert_eq!(backend.remove_count().await, 0); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] @@ -11658,7 +11810,7 @@ mod tests { ) .await .expect("legacy source should transition"); - rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object).await; + rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await; let legacy = store.pools[0] .get_disks_by_key(object) .load_file_info_versions_exact(bucket, object) @@ -12799,7 +12951,7 @@ mod tests { .expect("merge-loser source should transition"); copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await; } - rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin").await; + rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await; backend.set_remove_failure(true); store.pools[1] .delete_object(bucket, "hidden/item.bin", ObjectOptions::default()) @@ -16866,6 +17018,10 @@ mod tests { .find(|version| version.version_id == history.version_id) .expect("transitioned history should exist"); transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown; + rustfs_utils::http::metadata_compat::remove_str( + &mut transitioned.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + ); metadata .add_version(transitioned) .expect("unknown state should replace the transitioned version"); diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index f0969c3cf..42aa5a74d 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -297,6 +297,20 @@ fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersio } } +fn transition_version_metadata_value(raw: &[u8], decoded: Option<&str>) -> String { + decoded.map(str::to_owned).unwrap_or_else(|| { + if raw.is_empty() { + String::new() + } else { + String::from_utf8_lossy(raw).into_owned() + } + }) +} + +fn is_transition_version_metadata_key(key: &str) -> bool { + strip_internal_prefix_preserving_case(key).is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID)) +} + fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> { let valid = match state { TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(), @@ -366,14 +380,26 @@ impl<'a> DerivedInternalMetadata<'a> { } *slot = Some(value.as_slice()); } + fn merge_consistent<'a>(canonical: Option<&'a [u8]>, legacy: Option<&'a [u8]>) -> Result> { + if let (Some(canonical), Some(legacy)) = (canonical, legacy) + && canonical != legacy + { + return Err(Error::FileCorrupt); + } + Ok(canonical.or(legacy)) + } + Ok(Self { checksum: canonical.checksum.or(legacy.checksum), part_checksums: canonical.part_checksums.or(legacy.part_checksums), - transition_status: canonical.transition_status.or(legacy.transition_status), - transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object), - transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version), - transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state), - transition_tier: canonical.transition_tier.or(legacy.transition_tier), + transition_status: merge_consistent(canonical.transition_status, legacy.transition_status)?, + transitioned_object: merge_consistent(canonical.transitioned_object, legacy.transitioned_object)?, + transitioned_version: merge_consistent(canonical.transitioned_version, legacy.transitioned_version)?, + transitioned_version_state: merge_consistent( + canonical.transitioned_version_state, + legacy.transitioned_version_state, + )?, + transition_tier: merge_consistent(canonical.transition_tier, legacy.transition_tier)?, }) } } @@ -438,8 +464,14 @@ impl FileInfo { } } -fn set_transition_version_state(meta_sys: &mut HashMap>, state: TransitionVersionState) { - if state == TransitionVersionState::Unknown { +fn set_transition_version_state( + meta_sys: &mut HashMap>, + state: TransitionVersionState, + source_metadata: &HashMap, +) { + if state == TransitionVersionState::Unknown + && !rustfs_utils::http::metadata_compat::contains_key_str(source_metadata, SUFFIX_TRANSITIONED_VERSION_STATE) + { remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE); } else { insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec()); @@ -2643,6 +2675,11 @@ impl MetaObject { if derived_metadata.transitioned_version_state.is_some() { validate_transition_version_state(transition_version_state, transition_version.as_deref())?; } + for (key, value) in &self.meta_sys { + if is_transition_version_metadata_key(key) { + metadata.insert(key.to_owned(), transition_version_metadata_value(value, transition_version.as_deref())); + } + } let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok()); let transition_tier = derived_metadata .transition_tier @@ -2689,7 +2726,7 @@ impl MetaObject { } else { remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID); } - set_transition_version_state(&mut self.meta_sys, fi.transition_version_state); + set_transition_version_state(&mut self.meta_sys, fi.transition_version_state, &fi.metadata); insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec()); if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) { insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes()); @@ -2830,7 +2867,7 @@ impl From for MetaObject { insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version); } if !value.transition_status.is_empty() { - set_transition_version_state(&mut meta_sys, value.transition_version_state); + set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata); } if !value.transition_tier.is_empty() { @@ -2985,6 +3022,12 @@ impl MetaDeleteMarker { fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?; fi.transition_version = transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state); + for (key, value) in &self.meta_sys { + if is_transition_version_metadata_key(key) { + fi.metadata + .insert(key.to_owned(), transition_version_metadata_value(value, fi.transition_version.as_deref())); + } + } fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok()); if derived_metadata.transitioned_version_state.is_some() { validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?; @@ -3152,7 +3195,7 @@ impl From for MetaDeleteMarker { insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version); } if !value.transition_status.is_empty() || value.tier_free_version() { - set_transition_version_state(&mut meta_sys, value.transition_version_state); + set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata); } if !value.transition_tier.is_empty() { insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec()); @@ -4574,6 +4617,7 @@ mod tests { .into_fileinfo("b", "k", false) .expect("into_fileinfo"); assert_eq!(fi.transition_version_id, None); + assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(String::new())); } #[test] @@ -4585,6 +4629,10 @@ mod tests { .into_fileinfo("b", "k", false) .expect("into_fileinfo"); assert_eq!(fi.transition_version_id, None); + assert!( + get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()), + "nil UUID bytes must remain distinguishable from an empty MinIO version" + ); } #[test] @@ -4598,6 +4646,7 @@ mod tests { assert_eq!(fi.transition_version_id, Some(id)); assert_eq!(fi.transition_version, Some(id.to_string())); assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown); + assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string())); } #[test] @@ -4637,6 +4686,36 @@ mod tests { assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown); } + #[test] + fn meta_object_transition_version_state_explicit_unknown_is_not_legacy_missing() { + let mut metadata = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut metadata, + SUFFIX_TRANSITIONED_VERSION_STATE, + TransitionVersionState::Unknown.as_str().to_string(), + ); + let fi = FileInfo { + transition_status: "complete".to_string(), + transition_version_state: TransitionVersionState::Unknown, + metadata, + ..Default::default() + }; + + let object = MetaObject::from(fi); + assert_eq!( + get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE), + Some(b"unknown".as_slice()) + ); + let decoded = object + .into_fileinfo("b", "k", false) + .expect("explicit unknown state should decode"); + assert_eq!(decoded.transition_version_state, TransitionVersionState::Unknown); + assert_eq!( + rustfs_utils::http::metadata_compat::get_consistent_str(&decoded.metadata, SUFFIX_TRANSITIONED_VERSION_STATE,), + Some("unknown") + ); + } + #[test] fn meta_object_transition_version_state_exact_round_trips_dual_keys() { let id = sample_version_id(); @@ -4753,6 +4832,10 @@ mod tests { .expect("invalid transition version bytes must not fail the object read"); assert_eq!(fi.transition_version_id, None); assert_eq!(fi.transition_version, None); + assert!( + get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()), + "invalid raw bytes must remain distinguishable from an empty MinIO version" + ); } #[test] @@ -4795,6 +4878,10 @@ mod tests { .into_fileinfo("b", "k", false) .expect("nil tier version should remain an absent remote version"); assert_eq!(fi.transition_version_id, None); + assert!( + get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()), + "nil UUID bytes must remain distinguishable from an empty MinIO version" + ); } #[test] @@ -4812,6 +4899,7 @@ mod tests { .expect("legacy binary UUID tier version should decode"); assert_eq!(fi.transition_version_id, Some(id)); assert_eq!(fi.transition_version, Some(id.to_string())); + assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string())); } #[test] @@ -4910,6 +4998,23 @@ mod tests { assert_eq!(err, Error::FileCorrupt); } + #[test] + fn meta_object_transition_version_state_mixed_case_alias_conflict_fails_closed() { + let sys = HashMap::from([ + ( + format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"), + b"unknown".to_vec(), + ), + ("X-Minio-Internal-transitioned-version-state".to_string(), b"exact".to_vec()), + ]); + + let err = make_meta_object_with_sys(sys) + .into_fileinfo("b", "k", false) + .expect_err("mixed-case transition state aliases must agree"); + + assert_eq!(err, Error::FileCorrupt); + } + #[test] fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() { let object = FileMetaVersionHeader { From a3b8183be9cc75e9ea3775e8e8f4709a1de90182 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 5 Sep 2026 14:12:45 +0800 Subject: [PATCH 03/40] test(ecstore): narrow barrier re-export cfgs (#7152) Co-authored-by: Zhengchao An --- crates/ecstore/src/set_disk/mod.rs | 2 +- crates/ecstore/src/store/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 1bc87619d..0690d70cb 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -876,7 +876,7 @@ pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; pub(crate) use ops::object::DeleteObjectCommitBarrier; #[cfg(any(test, feature = "test-util"))] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier; pub(crate) use ops::object::body_cache_plaintext_len; #[cfg(all(test, feature = "test-util"))] diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 77aaf98c8..8a4579e1b 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -425,7 +425,7 @@ pub(crate) mod init_format; pub(crate) mod list_objects; mod multipart; mod object; -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] pub use object::DeleteAfterObjectLockSnapshotBarrier; pub(crate) use object::{ DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence, From d8c3b1bb26250a6ec9008b7100df436ac7720643 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 14:13:59 +0800 Subject: [PATCH 04/40] fix(app): fail closed on an unreadable bucket encryption config (#7183) The object write path read the bucket default encryption configuration with `.ok()`, which made "this bucket has no default encryption" and "the encryption configuration cannot be read" the same value. A bucket whose encryption blob is damaged therefore stored plaintext objects the operator had mandated be encrypted, with nothing returned to the client and nothing in the object to tell those writes apart afterwards. PUT, COPY and the snowball extract path now share one resolver: an absent configuration still writes plaintext exactly as before, and every other outcome refuses the write, carrying the accessor's typed error so a damaged blob surfaces as a deterministic InternalError while a transient metadata read failure surfaces as the retryable ServiceUnavailable. A missing bucket and a cold metadata cache both still resolve to "no configuration", so neither becomes a refusal. This matches `prepare_sse_configuration` in `storage::sse`, the resolver the multipart writer has always used, which fails closed on this lookup. --- rustfs/src/app/object/copy.rs | 46 +++++++++- rustfs/src/app/object/extract.rs | 2 +- rustfs/src/app/object/put.rs | 120 ++++++++++++++++++++++++- rustfs/src/app/object/shared.rs | 123 ++++++++++++++++++++++++++ rustfs/src/app/object/test_support.rs | 33 +++++++ 5 files changed, 321 insertions(+), 3 deletions(-) diff --git a/rustfs/src/app/object/copy.rs b/rustfs/src/app/object/copy.rs index fb9496c13..13393f797 100644 --- a/rustfs/src/app/object/copy.rs +++ b/rustfs/src/app/object/copy.rs @@ -394,7 +394,7 @@ impl DefaultObjectUsecase { // Bucket metadata uses the bucket name as its namespace-lock key. Load // every copy-time bucket snapshot before a same-object key can collide // with that key (for example, copying `bucket/bucket` onto itself). - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let bucket_sse_config = load_bucket_default_sse_config(&bucket).await?; let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?; if cp_src_dst_same && key == bucket { dst_opts.object_lock_config_snapshot = @@ -1388,4 +1388,48 @@ mod tests { .unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); } + + #[tokio::test] + #[serial_test::serial] + async fn execute_copy_object_refuses_a_bucket_whose_encryption_config_is_unreadable() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_store_test_context().await; + let bucket = format!("copy-sse-unreadable-{}", Uuid::new_v4()); + let source = "source.bin"; + let destination = "destination.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("unreadable-encryption copy bucket must be created"); + let mut reader = PutObjReader::from_vec(b"copied while the bucket still had a readable configuration".to_vec()); + store + .put_object(&bucket, source, &mut reader, &ObjectOptions::default()) + .await + .expect("copy source object must be written"); + install_unreadable_bucket_sse_config(&bucket).await; + + let input = CopyObjectInput::builder() + .copy_source(CopySource::Bucket { + bucket: bucket.clone().into(), + key: source.into(), + version_id: None, + }) + .bucket(bucket.clone()) + .key(destination.to_string()) + .build() + .expect("copy input must build"); + let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + + let err = Box::pin(usecase.execute_copy_object(build_request(input, Method::PUT))) + .await + .expect_err("an unreadable bucket encryption configuration must refuse the copy"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + let lookup_err = store + .get_object_info(&bucket, destination, &ObjectOptions::default()) + .await + .expect_err("a refused copy must not leave a destination object behind"); + assert!(is_err_object_not_found(&lookup_err), "{lookup_err}"); + } } diff --git a/rustfs/src/app/object/extract.rs b/rustfs/src/app/object/extract.rs index e5b7fb4da..2db042fa3 100644 --- a/rustfs/src/app/object/extract.rs +++ b/rustfs/src/app/object/extract.rs @@ -2037,7 +2037,7 @@ impl DefaultObjectUsecase { let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5); let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let bucket_sse_config = load_bucket_default_sse_config(&bucket).await?; let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse( bucket_sse_config.as_ref().map(|(config, _timestamp)| config), original_sse, diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index d108b1885..86d6e255a 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -1485,8 +1485,9 @@ impl DefaultObjectUsecase { }; let sse_config_stage_start = put_stage_metrics_enabled.then(Instant::now); - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let bucket_sse_config = load_bucket_default_sse_config(&bucket).await; rustfs_io_metrics::record_put_object_stage_duration_from("app_sse_config_lookup", sse_config_stage_start); + let bucket_sse_config = bucket_sse_config?; debug!( target: "rustfs::app::object_usecase", component = "app", @@ -3912,4 +3913,121 @@ mod tests { .expect_err("writes after the zero-byte quota update must be denied"); assert!(matches!(err, StorageError::QuotaExceeded { current: 4096, limit: 0 })); } + + #[tokio::test] + #[serial_test::serial] + async fn execute_put_object_refuses_a_bucket_whose_encryption_config_is_unreadable() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_store_test_context().await; + let bucket = format!("put-sse-unreadable-{}", Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("unreadable-encryption PUT bucket must be created"); + install_unreadable_bucket_sse_config(&bucket).await; + + let payload = Bytes::from_static(b"an operator mandated encryption for this bucket"); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .body(Some(StreamingBlob::from(s3s::Body::from(payload.clone())))) + .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64"))) + .build() + .expect("PUT input must build"); + let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + + let err = Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT))) + .await + .expect_err("an unreadable bucket encryption configuration must refuse the write"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + let lookup_err = store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect_err("a refused PUT must not leave an object behind"); + assert!(is_err_object_not_found(&lookup_err), "{lookup_err}"); + } + + #[tokio::test] + #[serial_test::serial] + async fn execute_put_object_still_writes_plaintext_without_bucket_encryption() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_store_test_context().await; + let bucket = format!("put-sse-absent-{}", Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("plaintext PUT bucket must be created"); + + let payload = Bytes::from_static(b"no default encryption is configured for this bucket"); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .body(Some(StreamingBlob::from(s3s::Body::from(payload.clone())))) + .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64"))) + .build() + .expect("PUT input must build"); + let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + + Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT))) + .await + .expect("a bucket without default encryption must still accept a plaintext write"); + + let stored = store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("the plaintext object must be readable"); + assert_eq!(stored.size, i64::try_from(payload.len()).expect("test payload length must fit i64")); + assert!( + !stored + .user_defined + .keys() + .any(|key| key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION) + || key.starts_with("x-rustfs-encryption-") + || key.starts_with("x-minio-encryption-")), + "the object must carry no encryption metadata: {:?}", + stored.user_defined + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn execute_put_object_extract_refuses_a_bucket_whose_encryption_config_is_unreadable() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_store_test_context().await; + let bucket = format!("extract-sse-unreadable-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("unreadable-encryption extract bucket must be created"); + install_unreadable_bucket_sse_config(&bucket).await; + + let payload = Bytes::from_static(b"archive bytes that must never be unpacked in plaintext"); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key("archive.tar".to_string()) + .body(Some(StreamingBlob::from(s3s::Body::from(payload.clone())))) + .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64"))) + .build() + .expect("extract PUT input must build"); + let mut req = build_request(input, Method::PUT); + req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + + let err = Box::pin(usecase.execute_put_object(&FS::new(), req)) + .await + .expect_err("an unreadable bucket encryption configuration must refuse the extract upload"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + let lookup_err = store + .get_object_info(&bucket, "archive.tar", &ObjectOptions::default()) + .await + .expect_err("a refused extract upload must not leave an object behind"); + assert!(is_err_object_not_found(&lookup_err), "{lookup_err}"); + } } diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs index bb382ec11..c35619edb 100644 --- a/rustfs/src/app/object/shared.rs +++ b/rustfs/src/app/object/shared.rs @@ -269,6 +269,129 @@ pub(super) fn resolve_bucket_default_sse( (effective_sse, effective_kms_key_id) } +/// The bucket's default encryption configuration for a write path. +/// +/// `Ok(None)` carries one meaning only — this bucket has no default encryption +/// — and the write proceeds in plaintext exactly as before. Every other +/// outcome refuses the write rather than collapsing onto that same value: an +/// encryption blob that exists but cannot be read fails closed in +/// `get_sse_config` since rustfs/rustfs#7172, and swallowing that error here +/// stores plaintext into a bucket whose operator mandated encryption, with +/// nothing returned to the client and nothing in the object to tell it apart +/// afterwards (rustfs/backlog#2287). +/// +/// The states the lookup can report, and what each one does: +/// +/// * configured and readable — apply the bucket default; +/// * no encryption blob at all, including a bucket that does not exist and a +/// bucket whose metadata document is absent — `ConfigNotFound`, so a cold +/// cache and a missing bucket are never turned into a refusal, and the write +/// still fails later with its own `NoSuchBucket`; +/// * blob present but unparseable — deterministic, so retrying cannot help; +/// surfaces as `InternalError` until an operator repairs or removes it; +/// * the metadata read itself failed (namespace lock, quorum, disk, an +/// uninitialized metadata system) — transient, and the typed error maps to +/// the retryable `ServiceUnavailable`. +/// +/// The last two are distinguished by the typed error the accessor returns, not +/// re-derived here: [`ApiError`] already separates them. This mirrors +/// `prepare_sse_configuration` in `storage::sse`, the resolver the multipart +/// writer uses, which has always failed closed on the same lookup. +pub(super) async fn load_bucket_default_sse_config( + bucket: &str, +) -> S3Result> { + classify_bucket_default_sse_lookup(bucket, metadata_sys::get_sse_config(bucket).await) +} + +fn classify_bucket_default_sse_lookup( + bucket: &str, + lookup: Result<(ServerSideEncryptionConfiguration, OffsetDateTime), StorageError>, +) -> S3Result> { + match lookup { + Ok(config) => Ok(Some(config)), + Err(err) if err == StorageError::ConfigNotFound => Ok(None), + Err(err) => { + let api_error = ApiError::from(err); + error!( + event = "bucket_sse_config_lookup_failed", + component = LOG_COMPONENT_APP, + subsystem = LOG_SUBSYSTEM_OBJECT, + result = "write_refused", + bucket = %bucket, + code = %api_error.code.as_str(), + error = %api_error, + "Bucket default encryption is unreadable; refusing the write instead of storing plaintext" + ); + Err(api_error.into()) + } + } +} + +#[cfg(test)] +mod bucket_default_sse_lookup_tests { + use super::*; + use s3s::dto::{ServerSideEncryptionByDefault, ServerSideEncryptionRule}; + use time::OffsetDateTime; + + fn sse_config() -> ServerSideEncryptionConfiguration { + ServerSideEncryptionConfiguration { + rules: vec![ServerSideEncryptionRule { + apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault { + sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AES256), + kms_master_key_id: None, + }), + blocked_encryption_types: None, + bucket_key_enabled: None, + }], + } + } + + #[test] + fn an_absent_configuration_still_writes_plaintext() { + let resolved = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ConfigNotFound)) + .expect("a bucket without default encryption must keep writing plaintext"); + + assert!(resolved.is_none()); + assert_eq!(resolve_bucket_default_sse(None, None, None, false), (None, None)); + } + + #[test] + fn a_readable_configuration_is_returned() { + let resolved = classify_bucket_default_sse_lookup("bucket", Ok((sse_config(), OffsetDateTime::UNIX_EPOCH))) + .expect("a readable configuration must not refuse the write") + .expect("a readable configuration must be applied"); + + assert_eq!(resolved.0.rules.len(), 1); + } + + #[test] + fn an_unreadable_configuration_refuses_the_write() { + let err = classify_bucket_default_sse_lookup( + "bucket", + Err(StorageError::other("persisted bucket encryption configuration is invalid")), + ) + .expect_err("a corrupt encryption blob must never degrade to plaintext"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + } + + #[test] + fn an_unavailable_metadata_read_refuses_the_write_as_retryable() { + let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ErasureReadQuorum)) + .expect_err("an unreadable metadata subsystem must never degrade to plaintext"); + + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); + } + + #[test] + fn a_missing_bucket_keeps_its_own_error() { + let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::BucketNotFound("bucket".to_string()))) + .expect_err("a bucket-not-found lookup must not be reported as an encryption failure"); + + assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket); + } +} + #[cfg(test)] mod deadlock_request_guard_tests { use super::DeadlockRequestGuard; diff --git a/rustfs/src/app/object/test_support.rs b/rustfs/src/app/object/test_support.rs index 61b368cf6..55ebb28b2 100644 --- a/rustfs/src/app/object/test_support.rs +++ b/rustfs/src/app/object/test_support.rs @@ -96,3 +96,36 @@ pub(super) fn real_cold_fill_plan( }; plan } + +/// A store with an ambient `AppContext`, for tests that drive a handler end to +/// end without the object-data-cache overrides of +/// [`real_cold_fill_test_context`]. +pub(super) async fn real_store_test_context() -> (Arc, Arc) { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + if current_app_context().is_none() { + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + } + let ambient = current_app_context().expect("real-store tests require an ambient AppContext"); + let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms())); + (store, context) +} + +/// Leave the bucket in the state a damaged encryption blob produces: the raw +/// document is retained and the typed configuration stays `None`, which is the +/// durable "exists but cannot be read" signal `get_sse_config` fails closed on +/// (rustfs/rustfs#7172). +pub(super) async fn install_unreadable_bucket_sse_config(bucket: &str) { + use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata}; + + let sys = get_global_bucket_metadata_sys().expect("bucket metadata system must be initialized"); + let metadata = { + let sys = sys.read().await; + sys.get(bucket).await.expect("bucket metadata must be cached") + }; + let mut metadata = (*metadata).clone(); + metadata.encryption_config_xml = b"truncated".to_vec(); + metadata.sse_config = None; + set_bucket_metadata(bucket.to_string(), metadata) + .await + .expect("unreadable bucket encryption configuration must be installed"); +} From 2477e31059c3ddb496b50f4bf2eef934e3c20422 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 14:22:48 +0800 Subject: [PATCH 05/40] test(ecstore): require core regressions in the existing CI lane (#7162) * test(ecstore): require core invariant tests in existing CI lane * test(ci): require a fresh core JUnit report * test(ecstore): match sealed context fixture map type --- .config/ecstore-required-tests.json | 72 ++++++++++++++++++++++++++ .github/workflows/ci.yml | 7 +++ docs/testing/ci-gates.md | 10 ++++ scripts/check_test_wiring.py | 79 ++++++++++++++++++++++++++++- 4 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 .config/ecstore-required-tests.json diff --git a/.config/ecstore-required-tests.json b/.config/ecstore-required-tests.json new file mode 100644 index 000000000..6cadc7815 --- /dev/null +++ b/.config/ecstore-required-tests.json @@ -0,0 +1,72 @@ +{ + "lane": "ci/test-and-lint", + "tests": [ + { + "invariant": "write-quorum", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one" + }, + { + "invariant": "metadata-rollback", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails" + }, + { + "invariant": "stale-writer", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss" + }, + { + "invariant": "range-body", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice" + }, + { + "invariant": "multipart-cancellation", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup" + }, + { + "invariant": "list-uncommitted-version", + "suite": "rustfs-filemeta", + "name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge" + }, + { + "invariant": "minio-object-fixture", + "suite": "rustfs-filemeta", + "name": "filemeta::test::parses_real_minio_object_xlmeta" + }, + { + "invariant": "corrupt-part-arrays", + "suite": "rustfs-filemeta", + "name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics" + } + ], + "fixtures": [ + { + "path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex", + "sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex", + "sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex", + "sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex", + "sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex", + "sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dc00bf5f..98fc6cc30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -269,6 +269,7 @@ jobs: CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }} run: | mkdir -p artifacts/test-and-lint + rm -f target/nextest/ci/junit.xml ./scripts/ci/resource_sampler.sh start nextest trap './scripts/ci/resource_sampler.sh stop' EXIT set +e @@ -277,6 +278,12 @@ jobs: --status-level all --final-status-level all \ 2>&1 | tee artifacts/test-and-lint/nextest.log status=${PIPESTATUS[0]} + if [[ "${status}" -eq 0 ]]; then + cargo nextest list --profile ci --all --exclude e2e_test --message-format json \ + > artifacts/test-and-lint/core-test-listing.json \ + && python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \ + && test -s target/nextest/ci/junit.xml || status=$? + fi { echo "command=cargo nextest run --profile ci --all --exclude e2e_test" echo "exit_status=${status}" diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 256b41bd4..93770b74d 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -109,3 +109,13 @@ Use an exact preview tag for an end-to-end release rehearsal. Manual dispatches ## Change checklist Update this file in the same PR when a job or check name changes, a workflow gains or loses a `pull_request` or `schedule` trigger, required contexts or strict/merge-queue policy change, report-only vs gating semantics change, or `.github/scheduled-validations.json` membership changes. Do not copy timeouts, crons, or test counts here. + +## ECStore invariant selection + +The existing `ci.yml` test-and-lint job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing test-and-lint artifact. + +The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, and corrupt part arrays. Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered. + +The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability. + +Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review. diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 1df46c234..8b1cf1246 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -764,8 +764,53 @@ def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]: return [] +def core_requirements(root: Path) -> dict: + data = json.loads((root / ".config/ecstore-required-tests.json").read_text()) + if not data["tests"] or not data["fixtures"]: + raise ValueError("core test and fixture requirements must not be empty") + identities = [(test["suite"], test["name"]) for test in data["tests"]] + if len(set(identities)) != len(identities): + raise ValueError("duplicate core test requirement") + return data + + +def check_core_fixtures(root: Path) -> list[str]: + try: + fixtures = core_requirements(root)["fixtures"] + errors = [] + for fixture in fixtures: + path = (root / fixture["path"]).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError("core fixture path escapes repository") + if not path.is_file(): + errors.append(f"{fixture['path']}: required core fixture missing") + elif hashlib.sha256(path.read_bytes()).hexdigest() != fixture["sha256"]: + errors.append(f"{fixture['path']}: core fixture sha256 mismatch") + return errors + except (OSError, KeyError, TypeError, ValueError) as error: + return [f"cannot validate core fixtures: {error}"] + + +def check_core_listing(root: Path, listing: Path) -> list[str]: + """Check the existing CI run's selection, not a second filtered test run.""" + try: + required = core_requirements(root)["tests"] + suites = json.loads(listing.read_text())["rust-suites"] + if not isinstance(suites, dict): + raise ValueError("rust-suites must be an object") + errors = check_core_fixtures(root) + for test in required: + testcase = suites.get(test["suite"], {}).get("testcases", {}).get(test["name"], {}) + if testcase.get("ignored") is not False or testcase.get("filter-match", {}).get("status") != "matches": + errors.append(f"{test['invariant']}: required test not selected: {test['suite']}::{test['name']}") + return errors + except (OSError, KeyError, TypeError, ValueError) as error: + return [f"cannot read core nextest listing: {error}"] + + def validate(root: Path) -> list[str]: errors: list[str] = [] + errors.extend(check_core_fixtures(root)) errors.extend(check_e2e_modules(root)) errors.extend(check_vault_test_groups(root)) errors.extend(check_ilm_build_budget(root)) @@ -779,6 +824,32 @@ def validate(root: Path) -> list[str]: class SelfTests(unittest.TestCase): + def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / ".config").mkdir() + fixture = root / "fixture.hex" + fixture.write_text("4142") + requirements = { + "tests": [{"invariant": "commit", "suite": "store", "name": "commit_test"}], + "fixtures": [{"path": "fixture.hex", "sha256": hashlib.sha256(fixture.read_bytes()).hexdigest()}], + } + (root / ".config/ecstore-required-tests.json").write_text(json.dumps(requirements)) + listing = root / "listing.json" + good = {"ignored": False, "filter-match": {"status": "matches"}} + for case, testcase in (("selected", good), ("missing", {}), ("ignored", dict(good, ignored=True)), + ("filtered", dict(good, **{"filter-match": {"status": "mismatch"}}))): + with self.subTest(case=case): + listing.write_text(json.dumps({"rust-suites": {"store": {"testcases": {"commit_test": testcase}}}})) + self.assertEqual(bool(check_core_listing(root, listing)), case != "selected") + listing.write_text(json.dumps({"rust-suites": {"store": {"testcases": {"commit_test": good}}}})) + fixture.write_text("4143") + self.assertIn("sha256 mismatch", check_core_listing(root, listing)[0]) + fixture.unlink() + self.assertIn("fixture missing", check_core_listing(root, listing)[0]) + listing.write_text("not json") + self.assertIn("cannot read", check_core_listing(root, listing)[0]) + def test_ilm_lane_keeps_the_measured_cargo_build_budget(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -981,6 +1052,7 @@ class SelfTests(unittest.TestCase): mock.patch(__name__ + ".check_e2e_modules", return_value=[]), mock.patch(__name__ + ".check_vault_test_groups", return_value=[]), mock.patch(__name__ + ".check_fuzz_targets", return_value=[]), + mock.patch(__name__ + ".check_core_fixtures", return_value=[]), mock.patch(__name__ + ".check_runner_selection", return_value=[]), mock.patch(__name__ + ".check_workflow_readiness", return_value=[]), mock.patch(__name__ + ".check_profile_definitions", return_value=[]), @@ -1393,6 +1465,11 @@ def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 + if len(sys.argv) == 3 and sys.argv[1] == "--check-core": + errors = check_core_listing(ROOT, Path(sys.argv[2])) + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors else 0 if len(sys.argv) == 4 and sys.argv[1] == "--check-profile": errors = check_profile_listing(ROOT, sys.argv[2], Path(sys.argv[3])) if errors: @@ -1410,7 +1487,7 @@ def main() -> int: return 0 if sys.argv[1:]: print( - "usage: check_test_wiring.py [--self-test | --check-profile PROFILE LISTING | " + "usage: check_test_wiring.py [--self-test | --check-core LISTING | --check-profile PROFILE LISTING | " "--update-profile PROFILE LISTING PLATFORM]", file=sys.stderr, ) From eaf5159d0fd24da5e7af0fc82793f1a1ac44f024 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 14:27:59 +0800 Subject: [PATCH 06/40] ci: refresh Linux full E2E membership after test additions (#7156) Co-authored-by: cxymds --- .config/e2e-full-selection.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index 8facca0e2..2cde94551 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34 -sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535 +sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b From 0885c721feadfcad3b6eb75710c0ace0454a9d0e Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:39:47 +0800 Subject: [PATCH 07/40] ci(upgrade): support manual runs between any two release versions (#7145) The workflow_dispatch inputs already accept arbitrary release tags, but the run failed late and unclearly when a tag had no .deb asset, and the from_version default pointed at 1.0.0-rc.4-preview.1, whose release ships no .deb at all - so scheduled runs died on a 404 while installing the old package. - Add a fail-fast preflight that resolves each requested tag via the GitHub release API and verifies the rustfs__amd64.deb asset exists before the suite starts, with an actionable error message otherwise (e.g. 1.0.0-rc.4 ships only zip/sbom assets). - Change the from_version default to 1.0.0-rc.3, the newest release that actually ships a .deb asset. - Reword the from_version/to_version descriptions so manual triggers state the .deb-asset requirement and the nightly fallback. - Pass PF_TESTING_GH_TOKEN as GH_TOKEN to the suite step for the gh api release lookups, matching the other functional workflows. Co-authored-by: Zhengchao An --- .github/workflows/rustfs-upgrade-test.yml | 28 +++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml index d822b4990..0c8c72cd0 100644 --- a/.github/workflows/rustfs-upgrade-test.yml +++ b/.github/workflows/rustfs-upgrade-test.yml @@ -18,7 +18,7 @@ on: workflow_dispatch: inputs: from_version: - description: 'OLD RustFS release tag (must ship a .deb asset, e.g. 1.0.0-rc.3)' + description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.' required: false default: '1.0.0-rc.3' from_url: @@ -26,7 +26,7 @@ on: required: false type: string to_version: - description: 'NEW RustFS release tag (leave empty for latest nightly)' + description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.' required: false to_url: description: 'NEW .deb URL. Overrides to_version / nightly default.' @@ -145,6 +145,7 @@ jobs: continue-on-error: true env: LOG_FILE: /tmp/rustfs-upgrade.log + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | set -euo pipefail chmod +x auto-testing/rustfs-upgrade-test.sh @@ -175,6 +176,29 @@ jobs: else ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") fi + # Fail fast with a clear message when a requested release tag has + # no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of + # letting the suite die mid-run on a 404. + check_release_asset() { + local version="$1" tag asset url + [ -n "${version}" ] && [ "${version}" != "null" ] || return 0 + tag="${version#v}" + asset="rustfs_${tag//-/.}_amd64.deb" + url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}" + if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then + echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2 + echo " ${url}" >&2 + echo "Pick a tag whose release ships a .deb (check its release assets)." >&2 + exit 1 + fi + echo "resolved ${tag} -> ${url}" + } + if [ -z "${FROM_URL}" ]; then + check_release_asset "${FROM_VERSION}" + fi + if [ -z "${TO_URL}" ]; then + check_release_asset "${TO_VERSION}" + fi ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}" - name: Generate report From a6589c19e3efbfbd386fed2db0aa78e9f21a2f44 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 5 Sep 2026 14:43:05 +0800 Subject: [PATCH 08/40] chore(tier): remove stage-a blanket lint allowances (#7153) Co-authored-by: Zhengchao An --- .../src/bucket/lifecycle/tier_sweeper.rs | 30 +++++++++---------- .../ecstore/src/services/tier/tier_admin.rs | 6 ++-- .../src/services/tier/warm_backend_aliyun.rs | 2 -- .../src/services/tier/warm_backend_azure.rs | 2 -- .../src/services/tier/warm_backend_gcs.rs | 18 +++++------ .../services/tier/warm_backend_huaweicloud.rs | 2 -- .../src/services/tier/warm_backend_minio.rs | 2 -- .../src/services/tier/warm_backend_r2.rs | 2 -- .../src/services/tier/warm_backend_rustfs.rs | 2 -- .../src/services/tier/warm_backend_tencent.rs | 2 -- scripts/ecstore-module-lint-register.txt | 20 ------------- 11 files changed, 25 insertions(+), 63 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 6518451ff..ad2241629 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use super::runtime_boundary as runtime_sources; use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp; @@ -72,9 +70,11 @@ static REMOTE_DELETE_BREAKER: LazyLock> = LazyLock::n }); #[cfg(test)] -static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock< - std::sync::Mutex std::io::Result<()> + Send + Sync>>>, -> = std::sync::LazyLock::new(|| std::sync::Mutex::new(None)); +type RemoteTierDeleteTestHook = Box std::io::Result<()> + Send + Sync>; + +#[cfg(test)] +static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| std::sync::Mutex::new(None)); #[derive(Debug)] struct RemoteDeleteBreaker { @@ -107,7 +107,7 @@ impl RemoteDeleteBreaker { fn prune(&mut self, now: Instant) { while let Some(ts) = self.failures.front().copied() { if now.duration_since(ts) > self.window { - self.failures.pop_front(); + let _ = self.failures.pop_front(); } else { break; } @@ -137,10 +137,10 @@ fn is_signer_header_error(err: &std::io::Error) -> bool { return false; } - if let Some(source) = err.get_ref() { - if error_chain_contains_signer_header_marker(source) { - return true; - } + if let Some(source) = err.get_ref() + && error_chain_contains_signer_header_marker(source) + { + return true; } let message = err.to_string().to_ascii_lowercase(); @@ -205,7 +205,7 @@ impl ObjSweeper { #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn with_version(&mut self, vid: Option) -> &Self { - self.version_id = vid.clone(); + self.version_id = vid; self } @@ -219,7 +219,7 @@ impl ObjSweeper { #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn get_opts(&self) -> lifecycle::ObjectOpts { let mut opts = ObjectOpts { - version_id: self.version_id.clone(), + version_id: self.version_id, versioned: self.versioned, version_suspended: self.suspended, ..Default::default() @@ -388,8 +388,8 @@ impl Jentry { impl ExpiryOp for Jentry { fn op_hash(&self) -> u64 { let mut hasher = Sha256::new(); - hasher.update(format!("{}", self.tier_name).as_bytes()); - hasher.update(format!("{}", self.obj_name).as_bytes()); + hasher.update(self.tier_name.as_bytes()); + hasher.update(self.obj_name.as_bytes()); xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED) } @@ -436,7 +436,7 @@ async fn delete_object_from_remote_tier_raw_with_manager( tier_name: &str, tier_config_mgr: &Arc>, ) -> Result<(), std::io::Error> { - let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name) + let lease = TierConfigMgr::acquire_operation_lease(tier_config_mgr, tier_name) .await .map_err(std::io::Error::other)?; delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await diff --git a/crates/ecstore/src/services/tier/tier_admin.rs b/crates/ecstore/src/services/tier/tier_admin.rs index 32cd844a0..6c3bc1c4a 100644 --- a/crates/ecstore/src/services/tier/tier_admin.rs +++ b/crates/ecstore/src/services/tier/tier_admin.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; @@ -145,7 +143,7 @@ mod tests { assert_eq!(creds.access_key, "access"); assert_eq!(creds.secret_key, "secret"); - assert_eq!(creds.creds_json.as_slice(), &service_account[..]); + assert_eq!(creds.creds_json.as_slice(), service_account); let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode"); assert_eq!(wire["access"], "access"); @@ -162,7 +160,7 @@ mod tests { .expect("the former RustFS field names and byte-array encoding should remain readable"); assert_eq!(legacy.access_key, "legacy-access"); assert_eq!(legacy.secret_key, "legacy-secret"); - assert_eq!(legacy.creds_json.as_slice(), &service_account[..]); + assert_eq!(legacy.creds_json.as_slice(), service_account); } #[test] diff --git a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs index 27fb7decd..f08c8fe2e 100644 --- a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs +++ b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_azure.rs b/crates/ecstore/src/services/tier/warm_backend_azure.rs index ed9ef0cf8..073501e65 100644 --- a/crates/ecstore/src/services/tier/warm_backend_azure.rs +++ b/crates/ecstore/src/services/tier/warm_backend_azure.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_gcs.rs b/crates/ecstore/src/services/tier/warm_backend_gcs.rs index 03c2df988..1aa6b538c 100644 --- a/crates/ecstore/src/services/tier/warm_backend_gcs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_gcs.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::{HashMap, HashSet}; use std::future::Future; @@ -146,11 +144,11 @@ pub struct WarmBackendGCS { impl WarmBackendGCS { pub async fn new(conf: &TierGCS, tier: &str) -> Result { - if conf.creds == "" { + if conf.creds.is_empty() { return Err(std::io::Error::other("both access and secret keys are required")); } - if conf.bucket == "" { + if conf.bucket.is_empty() { return Err(std::io::Error::other("no bucket name was provided")); } @@ -195,11 +193,11 @@ impl WarmBackendGCS { } pub fn get_dest(&self, object: &str) -> String { - let mut dest_obj = object.to_string(); - if self.prefix != "" { - dest_obj = format!("{}/{}", &self.prefix, object); + if self.prefix.is_empty() { + object.to_string() + } else { + format!("{}/{}", self.prefix, object) } - return dest_obj; } } @@ -223,7 +221,7 @@ impl WarmBackend for WarmBackendGCS { let bucket = gcs_bucket_resource_name(&self.bucket); let Ok(res) = Box::pin( self.client - .write_object(&bucket, &self.get_dest(object), Bytes::from(d)) + .write_object(&bucket, self.get_dest(object), Bytes::from(d)) .send_buffered(), ) .await @@ -240,7 +238,7 @@ impl WarmBackend for WarmBackendGCS { async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { let bucket = gcs_bucket_resource_name(&self.bucket); - let mut req = self.client.read_object(&bucket, &self.get_dest(object)); + let mut req = self.client.read_object(&bucket, self.get_dest(object)); let mut max_response_bytes = None; if let Some(generation) = parse_generation(rv)? { req = req.set_generation(generation); diff --git a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs index 73ced2cef..e626f6c9f 100644 --- a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs +++ b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_minio.rs b/crates/ecstore/src/services/tier/warm_backend_minio.rs index 020c2e319..6baef149b 100644 --- a/crates/ecstore/src/services/tier/warm_backend_minio.rs +++ b/crates/ecstore/src/services/tier/warm_backend_minio.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_r2.rs b/crates/ecstore/src/services/tier/warm_backend_r2.rs index 071ccefad..79a47d600 100644 --- a/crates/ecstore/src/services/tier/warm_backend_r2.rs +++ b/crates/ecstore/src/services/tier/warm_backend_r2.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs index 32821c427..0bb19bcdf 100644 --- a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_tencent.rs b/crates/ecstore/src/services/tier/warm_backend_tencent.rs index 20eb7ee81..8045b3332 100644 --- a/crates/ecstore/src/services/tier/warm_backend_tencent.rs +++ b/crates/ecstore/src/services/tier/warm_backend_tencent.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/scripts/ecstore-module-lint-register.txt b/scripts/ecstore-module-lint-register.txt index bf777b0f7..3bf9bd823 100644 --- a/scripts/ecstore-module-lint-register.txt +++ b/scripts/ecstore-module-lint-register.txt @@ -20,8 +20,6 @@ crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|clippy::all crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_must_use crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_variables -crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|clippy::all -crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_must_use crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_variables crates/s3-client/src/api_error_response.rs|clippy::all crates/s3-client/src/api_error_response.rs|unused_must_use @@ -74,36 +72,18 @@ crates/ecstore/src/services/event_notification.rs|unused_variables crates/ecstore/src/services/tier/tier.rs|clippy::all crates/ecstore/src/services/tier/tier.rs|unused_must_use crates/ecstore/src/services/tier/tier.rs|unused_variables -crates/ecstore/src/services/tier/tier_admin.rs|clippy::all -crates/ecstore/src/services/tier/tier_admin.rs|unused_must_use crates/ecstore/src/services/tier/tier_admin.rs|unused_variables crates/ecstore/src/services/tier/warm_backend.rs|clippy::all crates/ecstore/src/services/tier/warm_backend.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_aliyun.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_azure.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_gcs.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_minio.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_r2.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_rustfs.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_variables crates/ecstore/src/services/tier/warm_backend_s3.rs|clippy::all crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_tencent.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_variables From 971f9acdf49627f196872cda61d5772a878c71c3 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 15:17:25 +0800 Subject: [PATCH 09/40] fix(ecstore): reject stalled ODM pagination before merging pages (#7164) * fix(odm): reject non-progressing listing cursors * docs(odm): clarify folded source probe pagination * test(odm): match SDK bucket-root listing requests * test(ecstore): match sealed context fixture map type * test(odm): use app facade for listing wire types * fix(odm): resolve pagination Clippy failures --- .../src/bucket/on_demand_migration/breaker.rs | 7 +- .../on_demand_migration/list_through.rs | 311 ++++++++++++++- .../on_demand_migration/source_client.rs | 113 +++++- .../src/bucket/on_demand_migration/stats.rs | 2 +- rustfs/src/app/bucket_list_through.rs | 359 +++++++++++++++++- rustfs/src/app/storage_api.rs | 8 +- 6 files changed, 763 insertions(+), 37 deletions(-) diff --git a/crates/ecstore/src/bucket/on_demand_migration/breaker.rs b/crates/ecstore/src/bucket/on_demand_migration/breaker.rs index c41305f1d..46d25fca2 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/breaker.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/breaker.rs @@ -86,7 +86,12 @@ impl BreakerVerdict { Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => { BreakerVerdict::Failure } - Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral, + Some( + SourceError::AccessDenied + | SourceError::Unsupported(_) + | SourceError::InvalidPagination(_) + | SourceError::Other(_), + ) => BreakerVerdict::Neutral, } } } diff --git a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs index 8ff71196a..2720c7718 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs @@ -189,8 +189,8 @@ pub enum SourceListPlan { /// delimiter — the source's own roll-up boundary matches the request's. Page { prefix: String }, /// `filter.prefix` reaches past a delimiter, so every key the source could - /// contribute rolls into this one common prefix. One bounded probe listing - /// decides whether it exists; there is nothing to paginate. + /// contribute rolls into this one common prefix. Bounded probes follow + /// empty progressing pages until a key proves existence or the source ends. Folded { probe_prefix: String, common_prefix: String }, } @@ -279,6 +279,29 @@ pub struct FetchRequest { pub token: Option, } +/// Invalid pagination metadata. Opaque cursor values are never included in errors. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ListPageError { + #[error("truncated listing has no continuation token")] + Missing, + #[error("truncated listing has an empty continuation token")] + Empty, + #[error("truncated listing repeats a continuation token")] + Repeated, +} + +pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> { + if is_truncated { + match next_token { + None => return Err(ListPageError::Missing), + Some("") => return Err(ListPageError::Empty), + Some(next) if Some(next) == token => return Err(ListPageError::Repeated), + Some(_) => {} + } + } + Ok(()) +} + #[derive(Debug, Default)] struct SideState { start: SideCursor, @@ -364,6 +387,11 @@ impl ListThroughMerger { /// or `filter.prefix` excludes it. pub fn disable_source(&mut self) { self.source.disabled = true; + // A refill can fail after a valid first page. A local-only response + // must discard both that source payload and its ordering horizon. + self.source.entries.clear(); + self.source.pages.clear(); + self.source.more = false; } pub fn next_fetch(&self) -> Option { @@ -378,7 +406,13 @@ impl ListThroughMerger { /// Records one fetched page. `entries` must be sorted by `name` and already /// filtered with [`Self::accepts`]; the caller keeps the matching payloads /// in the same order. - pub fn push_page(&mut self, side: MergeSide, entries: Vec, is_truncated: bool, next_token: Option) { + pub fn push_page( + &mut self, + side: MergeSide, + entries: Vec, + is_truncated: bool, + next_token: Option, + ) -> Result<(), ListPageError> { let state = match side { MergeSide::Local => &mut self.local, MergeSide::Source => &mut self.source, @@ -387,15 +421,19 @@ impl ListThroughMerger { Some(last) => last.next_token.clone(), None => state.start.token.clone(), }; - // A truncated page without a cursor cannot be continued; treating the - // side as finished is the only alternative to looping on it forever. - state.more = is_truncated && next_token.is_some(); + validate_list_page(is_truncated, token.as_deref(), next_token.as_deref())?; + // Also reject a cycle through an earlier page in this bounded fetch. + if is_truncated && state.pages.iter().any(|page| page.token == next_token) { + return Err(ListPageError::Repeated); + } + state.more = is_truncated; state.pages.push(FetchedPage { token, count: entries.len(), next_token: is_truncated.then_some(next_token).flatten(), }); state.entries.extend(entries); + Ok(()) } pub fn finish(self) -> MergeOutcome { @@ -599,9 +637,15 @@ mod tests { let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys); let kept: Vec = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect(); buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned()); - merger.push_page(fetch.side, kept, truncated, next); + merger + .push_page(fetch.side, kept, truncated, next) + .expect("reference provider pages must advance"); } let outcome = merger.finish(); + assert_eq!(outcome.is_truncated, outcome.next_token.is_some()); + if outcome.is_truncated { + assert_ne!(outcome.next_token, token, "every truncated merged page must make progress"); + } page_sizes.push(outcome.picks.len()); for pick in &outcome.picks { let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone(); @@ -616,11 +660,25 @@ mod tests { } fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec { - let mut all: Vec = local.iter().chain(source.iter()).cloned().collect(); - all.sort(); - all.dedup(); - let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX); - entries + // This oracle builds the complete namespace independently of the + // provider's page/marker helper and the production merger. + let mut namespace = std::collections::BTreeMap::new(); + for key in local.iter().chain(source) { + let Some(suffix) = key.strip_prefix(prefix) else { + continue; + }; + if let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty()) + && let Some((directory, _)) = suffix.split_once(delimiter) + { + namespace.insert(format!("{prefix}{directory}{delimiter}"), true); + continue; + } + namespace.insert(key.clone(), false); + } + namespace + .into_iter() + .map(|(name, is_prefix)| ListEntryKey { name, is_prefix }) + .collect() } #[test] @@ -662,7 +720,9 @@ mod tests { token: None }) ); - merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None); + merger + .push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None) + .expect("local EOF is valid"); assert_eq!(merger.next_fetch(), None); let outcome = merger.finish(); assert_eq!(outcome.picks.len(), 1); @@ -683,12 +743,14 @@ mod tests { }; let mut merger = ListThroughMerger::new(1, Some(&resume)); merger.disable_source(); - merger.push_page( - MergeSide::Local, - vec![ListEntryKey::object("b"), ListEntryKey::object("c")], - true, - Some("local-2".to_string()), - ); + merger + .push_page( + MergeSide::Local, + vec![ListEntryKey::object("b"), ListEntryKey::object("c")], + true, + Some("local-2".to_string()), + ) + .expect("local cursor advances"); let outcome = merger.finish(); assert!(outcome.is_truncated); let token = outcome.next_token.expect("truncated page carries a token"); @@ -698,6 +760,212 @@ mod tests { assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed"); } + #[test] + fn truncated_pages_require_a_nonempty_advancing_cursor() { + for side in [MergeSide::Local, MergeSide::Source] { + for entries in [vec![], vec![ListEntryKey::object("a")]] { + for (next, expected) in [ + (None, Err(ListPageError::Missing)), + (Some(""), Err(ListPageError::Empty)), + (Some("stuck"), Err(ListPageError::Repeated)), + (Some("advances"), Ok(())), + ] { + let resume = ListThroughToken::new( + SideCursor { + token: Some("stuck".into()), + done: false, + }, + SideCursor { + token: Some("stuck".into()), + done: false, + }, + None, + ); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + let result = merger.push_page(side, entries.clone(), true, next.map(str::to_string)); + assert_eq!(result, expected, "{side:?}, {entries:?}, {next:?}"); + let state = if side == MergeSide::Local { + &merger.local + } else { + &merger.source + }; + assert_eq!(state.pages.len(), usize::from(result.is_ok()), "invalid page must not be accepted"); + } + } + } + } + + #[test] + fn repeated_empty_cursor_is_rejected_before_an_identical_page_can_escape() { + let resume = ListThroughToken::new( + SideCursor { token: None, done: true }, + SideCursor { + token: Some("stuck".into()), + done: false, + }, + None, + ); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + assert_eq!( + merger.next_fetch(), + Some(FetchRequest { + side: MergeSide::Source, + token: Some("stuck".into()) + }) + ); + assert_eq!( + merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())), + Err(ListPageError::Repeated) + ); + } + + #[test] + fn empty_pages_may_advance_within_the_fetch_budget_until_eof() { + let mut merger = ListThroughMerger::new(2, None); + merger.push_page(MergeSide::Local, vec![], false, None).expect("local EOF"); + for next in ["opaque-z", "opaque-a"] { + assert_eq!(merger.next_fetch().expect("bounded source fetch").side, MergeSide::Source); + merger + .push_page(MergeSide::Source, vec![], true, Some(next.into())) + .expect("opaque cursor advances regardless of sort order"); + } + assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget"); + let outcome = merger.finish(); + assert!(outcome.picks.is_empty()); + assert!(outcome.is_truncated); + let token = outcome.next_token.expect("empty progressing page has a cursor"); + assert_eq!(token.source.as_deref(), Some("opaque-a")); + let mut merger = ListThroughMerger::new(2, Some(&token)); + assert_eq!(merger.next_fetch().expect("source resumes").token.as_deref(), Some("opaque-a")); + merger + .push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None) + .expect("source EOF"); + let outcome = merger.finish(); + assert_eq!( + outcome.picks, + vec![MergePick { + side: MergeSide::Source, + index: 0 + }] + ); + assert!(!outcome.is_truncated); + assert!(outcome.next_token.is_none()); + } + + #[test] + fn a_cursor_cycle_inside_the_fetch_budget_is_rejected() { + let resume = ListThroughToken::new( + SideCursor { token: None, done: true }, + SideCursor { + token: Some("first".into()), + done: false, + }, + None, + ); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + merger + .push_page(MergeSide::Source, vec![], true, Some("second".into())) + .expect("first page advances"); + assert_eq!( + merger.push_page(MergeSide::Source, vec![], true, Some("first".into())), + Err(ListPageError::Repeated) + ); + } + + #[test] + fn source_refill_failure_discards_buffered_source_entries_and_horizon() { + let mut merger = ListThroughMerger::new(2, None); + merger + .push_page(MergeSide::Local, vec![ListEntryKey::object("z")], false, None) + .expect("local EOF"); + merger + .push_page(MergeSide::Source, vec![ListEntryKey::object("a")], true, Some("stuck".into())) + .expect("first source page advances"); + assert_eq!(merger.next_fetch().expect("source refill is required").token.as_deref(), Some("stuck")); + assert_eq!( + merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())), + Err(ListPageError::Repeated) + ); + merger.disable_source(); + let outcome = merger.finish(); + assert_eq!( + outcome.picks, + vec![MergePick { + side: MergeSide::Local, + index: 0 + }] + ); + assert!(!outcome.is_truncated); + assert!(outcome.next_token.is_none()); + } + + #[test] + fn list_through_static_namespace_boundary_matrix() { + let corpus = [ + "a", + "a/", + "a/b", + "a/b/child", + "a0", + "b", + "b/leaf", + "quote\"&<", + "space key", + "z", + "é", + "中/文", + ]; + for count in [0, 1, 3, 4, corpus.len()] { + let keys: Vec = corpus[..count].iter().map(|key| (*key).to_string()).collect(); + for placement in 0..3 { + let (local, source): (Vec<_>, Vec<_>) = + keys.iter() + .enumerate() + .fold((vec![], vec![]), |(mut local, mut source), (index, key)| { + if placement != 1 || index % 2 == 0 { + local.push(key.clone()); + } + if placement != 0 || index % 2 == 0 { + source.push(key.clone()); + } + (local, source) + }); + for prefix in ["", "a", "a/", "中/"] { + for delimiter in [None, Some("/")] { + for max_keys in [1, 3, 4] { + let oracle = expected(&local, &source, prefix, delimiter); + let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys); + assert_eq!( + emitted.iter().map(|(entry, _)| entry.clone()).collect::>(), + oracle, + "count={count}, placement={placement}, prefix={prefix}, delimiter={delimiter:?}, max={max_keys}" + ); + let expected_sizes: Vec<_> = if oracle.is_empty() { + vec![0] + } else { + oracle.chunks(max_keys).map(<[ListEntryKey]>::len).collect() + }; + assert_eq!(sizes, expected_sizes, "exact max and max+1 boundaries must agree"); + } + } + } + } + } + } + + #[test] + fn list_through_large_overlap_walk_keeps_all_5300_keys() { + let source: Vec<_> = (0..5000).map(|index| format!("k{index:05}")).collect(); + let local: Vec<_> = (4800..5300).map(|index| format!("k{index:05}")).collect(); + let (emitted, sizes) = walk(&local, &source, "", None, 333); + assert_eq!(emitted.len(), 5300); + for (index, (entry, side)) in emitted.iter().enumerate() { + assert_eq!(entry.name, format!("k{index:05}")); + assert_eq!(*side, if index >= 4800 { MergeSide::Local } else { MergeSide::Source }); + } + assert_eq!(sizes, [vec![333; 15], vec![305]].concat()); + } + #[test] fn token_round_trips_and_rejects_tampering() { let token = ListThroughToken::new( @@ -796,7 +1064,10 @@ mod tests { } proptest! { - #![proptest_config(ProptestConfig::with_cases(256))] + #![proptest_config(ProptestConfig { + rng_seed: proptest::test_runner::RngSeed::Fixed(0xec5706), + ..ProptestConfig::with_cases(256) + })] /// Full pagination of a merged listing equals the sorted, deduplicated /// union of both sides, with every shared key served by local, and no diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs index 4782e05f0..f06301036 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs @@ -25,6 +25,7 @@ //! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never //! forwarded: v1 rejects SSE-C source objects outright. +use super::list_through::{ListPageError, validate_list_page}; use crate::bucket::remote_s3_client::{ PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config, }; @@ -223,6 +224,8 @@ pub enum SourceError { ServerError(u16), #[error("unsupported source object: {0}")] Unsupported(String), + #[error("invalid source listing: {0}")] + InvalidPagination(#[from] ListPageError), #[error("source request failed: {0}")] Other(String), } @@ -245,6 +248,7 @@ impl SourceError { SourceError::Connect(_) => "connect", SourceError::ServerError(_) => "server_error", SourceError::Unsupported(_) => "unsupported", + SourceError::InvalidPagination(_) => "invalid_pagination", SourceError::Other(_) => "other", } } @@ -714,6 +718,7 @@ impl SourceClient { ..*request }) .await?; + validate_list_page(page.is_truncated, request.continuation_token, page.next_continuation_token.as_deref())?; page.objects = page .objects .into_iter() @@ -800,11 +805,6 @@ impl SourceBackend for S3SourceBackend { let is_truncated = output.is_truncated.unwrap_or(false); let next_continuation_token = output.next_continuation_token; - if is_truncated && next_continuation_token.is_none() { - return Err(SourceError::Other( - "source reported a truncated listing without a continuation token".to_string(), - )); - } let objects = output .contents .unwrap_or_default() @@ -1274,7 +1274,9 @@ mod tests { data/photos/ outside/ "#; - let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await; + let next_body = body.replace("data/opaque", "data/next"); + let (client, requests) = + scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), &next_body)]).await; let first = client .list_page(&SourceListRequest { prefix: Some("photos/"), @@ -1336,7 +1338,104 @@ mod tests { .list_objects_v2(None, None, 10) .await .expect_err("truncated page without token is corrupt"); - assert!(matches!(err, SourceError::Other(_)), "{err:?}"); + assert!(matches!(err, SourceError::InvalidPagination(ListPageError::Missing)), "{err:?}"); + } + + #[tokio::test] + async fn list_page_validates_s3_cursor_progress_before_mapping_entries() { + for contents in ["", "data/a1"] { + for (truncated, next, expected) in [ + (true, None, Some(ListPageError::Missing)), + (true, Some(""), Some(ListPageError::Empty)), + (true, Some("stuck"), Some(ListPageError::Repeated)), + (true, Some("opaque-next"), None), + (false, None, None), + (false, Some("stuck"), None), + ] { + let next_xml = next + .map(|next| format!("{next}")) + .unwrap_or_default(); + let body = format!( + "{truncated}{next_xml}{contents}" + ); + let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), &body)]).await; + let result = client + .list_page(&SourceListRequest { + continuation_token: Some("stuck"), + max_keys: 2, + ..Default::default() + }) + .await; + match expected { + Some(expected) => { + let error = result.expect_err("malformed pagination must fail at the provider boundary"); + assert!( + matches!(&error, SourceError::InvalidPagination(actual) if *actual == expected), + "{error:?}" + ); + assert_eq!(error.class_label(), "invalid_pagination"); + assert!(!error.is_retryable()); + assert!(!error.to_string().contains("stuck"), "errors must not echo opaque tokens"); + } + None => { + let page = result.expect("progressing empty/nonempty pages and EOF are valid"); + assert_eq!(page.is_truncated, truncated); + assert_eq!(page.next_continuation_token.as_deref(), next); + assert_eq!(page.objects.len(), usize::from(!contents.is_empty())); + if let Some(object) = page.objects.first() { + assert_eq!(object.key, "a"); + } + } + } + let requests = recorded(&requests); + assert_eq!(requests.len(), 1, "invalid pagination must not be retried"); + assert!(requests[0].uri.contains("continuation-token=stuck")); + } + } + } + + struct ListOnlyBackend(SourcePage); + + #[async_trait::async_trait] + impl SourceBackend for ListOnlyBackend { + async fn list(&self, request: &SourceListRequest<'_>) -> Result { + assert_eq!(request.continuation_token, Some("stuck"), "opaque cursors reach every provider unchanged"); + Ok(self.0.clone()) + } + + async fn head(&self, _key: &str) -> Result { + panic!("unexpected HEAD in list test") + } + async fn get(&self, _key: &str, _range: Option<&HTTPRangeSpec>) -> Result { + panic!("unexpected GET in list test") + } + async fn tagging(&self, _key: &str) -> Result, SourceError> { + panic!("unexpected tagging in list test") + } + async fn probe(&self) -> Result<(), SourceError> { + panic!("unexpected probe in list test") + } + } + + #[tokio::test] + async fn list_page_validates_non_s3_provider_cursors_at_the_common_boundary() { + for (next, expected) in [ + (None, ListPageError::Missing), + (Some(""), ListPageError::Empty), + (Some("stuck"), ListPageError::Repeated), + ] { + let mut client = prefix_client(Some("data/".into())); + client.backend = Box::new(ListOnlyBackend(SourcePage { + is_truncated: true, + next_continuation_token: next.map(str::to_string), + ..Default::default() + })); + let error = client + .list_objects_v2(None, Some("stuck"), 2) + .await + .expect_err("all providers must advance pagination"); + assert!(matches!(error, SourceError::InvalidPagination(actual) if actual == expected)); + } } const TAGGING_BODY: &str = r#" diff --git a/crates/ecstore/src/bucket/on_demand_migration/stats.rs b/crates/ecstore/src/bucket/on_demand_migration/stats.rs index ed3a7de2c..a483abde0 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/stats.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/stats.rs @@ -177,7 +177,7 @@ impl From<&SourceError> for PullFailureReason { SourceError::Connect(_) => PullFailureReason::SourceConnect, SourceError::ServerError(_) => PullFailureReason::SourceServerError, SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported, - SourceError::Other(_) => PullFailureReason::SourceOther, + SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther, } } } diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs index 02c4c7420..a97470f9a 100644 --- a/rustfs/src/app/bucket_list_through.rs +++ b/rustfs/src/app/bucket_list_through.rs @@ -252,8 +252,16 @@ pub(crate) async fn merged_list_objects_v2( .filter(|entry| merger.accepts(&entry.key().name)) .collect(); let keys: Vec = kept.iter().map(SideEntry::key).collect(); + if let Err(error) = merger.push_page(fetch.side, keys, is_truncated, next_token) { + match fetch.side { + MergeSide::Source => { + degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?; + continue; + } + MergeSide::Local => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())), + } + } buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some)); - merger.push_page(fetch.side, keys, is_truncated, next_token); } let outcome = merger.finish(); @@ -340,10 +348,12 @@ async fn fetch_source_page( continuation_token: token, max_keys: params.max_keys, }, - // Everything under `filter.prefix` rolls into one common prefix, so a - // single bounded listing settles whether it exists. + // Everything under `filter.prefix` rolls into one common prefix. An + // empty truncated probe must still follow its cursor before declaring + // that prefix absent. SourceListPlan::Folded { probe_prefix, .. } => SourceListRequest { prefix: Some(probe_prefix.as_str()), + continuation_token: token, max_keys: 1, ..Default::default() }, @@ -368,8 +378,8 @@ async fn fetch_source_page( } else { Vec::new() }, - false, - None, + !exists && page.is_truncated, + if exists { None } else { page.next_continuation_token }, )) } _ => { @@ -417,6 +427,17 @@ async fn local_delete_markers(store: &Arc, bucket: &str, keys: &[String #[cfg(test)] mod tests { use super::*; + use crate::app::bucket_usecase::DefaultBucketUsecase; + use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore}; + use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{ + FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig, + }; + use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response}; + use crate::app::storage_api::test::StoragePutObjReader; + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use crate::app::storage_api::test::contract::object::ObjectIO as _; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; fn token(local: Option<&str>, local_done: bool) -> ListThroughToken { ListThroughToken { @@ -526,4 +547,332 @@ mod tests { assert!(degraded); assert_eq!(merger.next_fetch().map(|fetch| fetch.side), Some(MergeSide::Local)); } + + /// Serves exactly the scripted S3 pages and joins every connection before + /// returning. A source retry or unexpected operation fails the test. + async fn scripted_list_source(pages: Vec) -> (String, tokio_util::task::AbortOnDropHandle>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind listing source"); + let address = listener.local_addr().expect("listing source address"); + let server = tokio::spawn(async move { + let mut requests = Vec::new(); + for body in pages { + let (mut stream, _) = listener.accept().await.expect("accept source listing"); + let mut request = Vec::new(); + let mut chunk = [0; 4096]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = stream.read(&mut chunk).await.expect("read signed listing request"); + assert!(count > 0, "source request must include complete headers"); + request.extend_from_slice(&chunk[..count]); + assert!(request.len() <= 32 * 1024, "listing request headers must be bounded"); + } + let first_line = String::from_utf8_lossy(&request) + .lines() + .next() + .expect("request line") + .to_string(); + // The SDK joins the bucket endpoint with the LIST operation's `/` path. + assert!( + first_line.starts_with("GET /source-bucket/?"), + "expected a path-style bucket-root LIST request, got {first_line:?}" + ); + assert!(first_line.contains("list-type=2"), "expected a ListObjectsV2 query, got {first_line:?}"); + requests.push(first_line); + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.expect("write source page"); + stream.shutdown().await.expect("finish source response"); + } + requests + }); + (format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server)) + } + + fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String { + let next = next + .map(|token| format!("{token}")) + .unwrap_or_default(); + let contents = key + .map(|key| format!("{key}1")) + .unwrap_or_default(); + format!( + "{truncated}{next}{contents}" + ) + } + + struct ListThroughTestState { + bucket: String, + module_enabled: bool, + } + + impl Drop for ListThroughTestState { + fn drop(&mut self) { + let sys = OnDemandMigrationSys::get(); + sys.remove(&self.bucket); + sys.set_module_enabled(self.module_enabled); + } + } + + async fn source_policy_request( + pages: Vec, + policy: SourceErrorPolicy, + resume_source: Option<&str>, + filter_prefix: Option<&str>, + ) -> (S3Result>, Vec) { + let store = shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create list-through bucket"); + store + .put_object( + &bucket, + "z-local", + &mut StoragePutObjReader::from_vec(vec![1]), + &StorageObjectOptions::default(), + ) + .await + .expect("seed real local listing"); + let (endpoint, server) = scripted_list_source(pages).await; + let sys = OnDemandMigrationSys::get(); + let _state_guard = ListThroughTestState { + bucket: bucket.clone(), + module_enabled: sys.is_module_enabled(), + }; + sys.set_module_enabled(true); + let config = OnDemandMigrationConfig { + version: 1, + enabled: true, + source: SourceConfig { + provider: Provider::Minio, + endpoint: Some(endpoint), + region: "us-east-1".into(), + bucket: "source-bucket".into(), + path_style: PathStyle::Path, + credentials: Some(SourceCredentials { + access_key: "test-access".into(), + secret_key: "test-secret".into(), + session_token: None, + }), + tls: TlsConfig::default(), + }, + filter: FilterConfig { + prefix: filter_prefix.map(str::to_string), + ..Default::default() + }, + policy: PolicyConfig { + list_through: true, + source_error: policy, + ..Default::default() + }, + }; + sys.apply(&bucket, Some(&config)).await; + assert!( + sys.state(&bucket).expect("ODM state installed").client().is_ok(), + "fake source client must build" + ); + let continuation_token = resume_source.map(|source| { + let token = ListThroughToken { + t: "odm-list".into(), + v: 1, + local: None, + local_done: false, + source: Some(source.into()), + source_done: false, + last_key: None, + }; + base64_simd::STANDARD.encode_to_string(token.encode().as_bytes()) + }); + let input = ListObjectsV2Input { + bucket, + max_keys: Some(2), + continuation_token, + delimiter: filter_prefix.map(|_| "/".to_string()), + encoding_type: None, + expected_bucket_owner: None, + fetch_owner: None, + optional_object_attributes: None, + prefix: None, + request_payer: None, + start_after: None, + }; + let request = S3Request { + input, + method: http::Method::GET, + uri: http::Uri::from_static("/?list-type=2"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + let result = tokio::time::timeout( + Duration::from_secs(10), + DefaultBucketUsecase::from_global().execute_list_objects_v2(request), + ) + .await + .expect("listing must complete within its bounded source budget"); + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("source connections must finish") + .expect("source server must not panic"); + (result, requests) + } + + #[test] + #[serial_test::serial] + fn list_through_invalid_source_pagination_obeys_policy_on_the_handler_path() { + run_large_stack_test("list-through-source-policy", || async { + temp_env::async_with_vars( + [ + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] { + for next in [None, Some(""), Some("stuck")] { + for key in [None, Some("a-source")] { + let (result, requests) = + source_policy_request(vec![source_xml(next, true, key)], policy, Some("stuck"), None).await; + assert_eq!(requests.len(), 1, "a malformed source page must not be retried"); + assert!(requests[0].contains("continuation-token=stuck")); + assert_source_policy_result(result, policy); + } + } + let (result, requests) = source_policy_request( + vec![ + source_xml(Some("stuck"), true, Some("a-source")), + source_xml(Some("stuck"), true, None), + ], + policy, + None, + None, + ) + .await; + assert_eq!(requests.len(), 2, "the failure must occur during a real refill"); + assert!(!requests[0].contains("continuation-token=")); + assert!(requests[1].contains("continuation-token=stuck")); + assert_source_policy_result(result, policy); + } + }, + ) + .await; + }); + } + + #[test] + #[serial_test::serial] + fn list_through_empty_advancing_source_pages_reach_eof_on_the_handler_path() { + run_large_stack_test("list-through-empty-source-pages", || async { + temp_env::async_with_vars( + [ + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + for filter_prefix in [None, Some("photos/2024/")] { + let source_key = if filter_prefix.is_some() { + "photos/2024/a-source" + } else { + "a-source" + }; + let (result, requests) = source_policy_request( + vec![ + source_xml(Some("opaque-next"), true, None), + source_xml(None, false, Some(source_key)), + ], + SourceErrorPolicy::Propagate, + None, + filter_prefix, + ) + .await; + assert_eq!(requests.len(), 2, "an empty truncated source page must reach its successor"); + assert!(requests[1].contains("continuation-token=opaque-next")); + let response = result.expect("empty progressing source page is valid"); + assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list")); + let output = response.output; + let objects: Vec<_> = output + .contents + .unwrap_or_default() + .into_iter() + .map(|object| object.key.expect("listed object key")) + .collect(); + if filter_prefix.is_some() { + assert_eq!(objects, vec!["z-local"]); + assert_eq!( + output + .common_prefixes + .unwrap_or_default() + .into_iter() + .map(|prefix| prefix.prefix.expect("rolled-up prefix")) + .collect::>(), + vec!["photos/"] + ); + } else { + assert_eq!(objects, vec!["a-source", "z-local"]); + assert!(output.common_prefixes.unwrap_or_default().is_empty()); + } + assert_eq!(output.key_count, Some(2)); + assert_eq!(output.is_truncated, Some(false)); + assert!(output.next_continuation_token.is_none()); + } + }, + ) + .await; + }); + } + + fn assert_source_policy_result(result: S3Result>, policy: SourceErrorPolicy) { + match policy { + SourceErrorPolicy::Propagate => { + let error = result.expect_err("propagate must expose malformed pagination"); + assert_eq!(error.status_code(), Some(http::StatusCode::FAILED_DEPENDENCY)); + assert_eq!(error.code(), &S3ErrorCode::Custom("SourceUnavailable".into())); + assert_eq!(error.message(), Some("invalid_pagination")); + } + SourceErrorPolicy::NotFound => { + let response = result.expect("not_found must preserve the local listing"); + assert_eq!( + response + .headers + .get("x-rustfs-on-demand-migration-list") + .expect("local_only header"), + "local_only" + ); + let output = response.output; + assert_eq!( + output + .contents + .unwrap_or_default() + .into_iter() + .map(|object| object.key.expect("local key")) + .collect::>(), + vec!["z-local"] + ); + assert_eq!(output.is_truncated, Some(false)); + assert_eq!(output.key_count, Some(1)); + assert!(output.next_continuation_token.is_none()); + } + } + } } diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 19548aa66..9f543ac95 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -29,11 +29,13 @@ pub(crate) fn EndpointServerPools( pub(crate) mod s3 { #[cfg(test)] pub(crate) use s3s::dto::{ - BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration, - ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault, - ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration, + BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ListObjectsV2Input, + ListObjectsV2Output, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, + ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration, }; pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result}; + #[cfg(test)] + pub(crate) use s3s::{S3Request, S3Response}; } pub(crate) mod admin { From cbfd5b92f4603b464c9ce2a16e3ac2293d398193 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 15:19:29 +0800 Subject: [PATCH 10/40] refactor(ecstore): isolate local object rename commit (#7166) * fix(ecstore): drain durable control-plane write tails * fix(ecstore): retain PUT staging after incomplete rollback * fix(ecstore): drain backfill checkpoint before confirmation * refactor(ecstore): isolate local object rename commit * refactor(ecstore): remove moved quota fence import * fix(ecstore): retain per-disk rename rollback outcomes * fix(ecstore): retain indeterminate rename recovery evidence * test(ecstore): mark rollback fixtures as inline data * test(ecstore): match sealed context fixture map type * test(ecstore): match sealed context fixture map type * fix(ecstore): preserve known preflight rename rejections * test(ecstore): cover observed rename outer failures * test(ecstore): count decommission faults across retry restarts --- .../bucket/lifecycle/manual_transition_job.rs | 5 + .../bucket/lifecycle/tier_delete_journal.rs | 6 + .../lifecycle/transition_transaction.rs | 2 + .../bucket/on_demand_migration/backfill.rs | 1 + crates/ecstore/src/core/pools.rs | 4 + crates/ecstore/src/disk/disk_store.rs | 85 +- crates/ecstore/src/disk/local.rs | 1345 +++-------------- crates/ecstore/src/disk/local/commit.rs | 1233 +++++++++++++++ crates/ecstore/src/disk/mod.rs | 49 + crates/ecstore/src/object_api/types.rs | 16 + .../src/services/tier/tier_mutation_intent.rs | 2 + .../src/services/tier/tier_probe_intent.rs | 2 + .../src/set_disk/core/io_primitives.rs | 1131 ++++++++++++-- crates/ecstore/src/set_disk/ops/object.rs | 475 +++++- .../src/set_disk/transition_matrix_tests.rs | 5 +- crates/ecstore/src/store/init.rs | 90 +- .../ecstore-validation-suite-design.md | 16 + 17 files changed, 3098 insertions(+), 1369 deletions(-) create mode 100644 crates/ecstore/src/disk/local/commit.rs diff --git a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs index b480ba468..050228b25 100644 --- a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs +++ b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs @@ -1170,6 +1170,7 @@ pub async fn save_manual_transition_job_record_if_current( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current_etag.to_string()), ..Default::default() @@ -1242,6 +1243,7 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent( data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1270,6 +1272,7 @@ pub(crate) async fn save_manual_transition_task_if_absent( data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1621,6 +1624,7 @@ pub async fn save_manual_transition_scope_admission_if_absent( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1672,6 +1676,7 @@ pub async fn save_manual_transition_scope_admission_if_current( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current_etag.to_string()), ..Default::default() diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 3a6a7e451..2270bce3b 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -1733,6 +1733,7 @@ async fn save_config_if_none_fenced( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1832,6 +1833,7 @@ async fn save_decommission_manifest_checkpoint_if_match( let mut opts = ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, no_lock: true, http_preconditions: Some(HTTPPreconditions { if_match: Some(observed_etag), @@ -1960,6 +1962,7 @@ async fn save_config_if_match_fenced( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag.to_string()), ..Default::default() @@ -3780,6 +3783,7 @@ where data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -3869,6 +3873,7 @@ where data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag), ..Default::default() @@ -3893,6 +3898,7 @@ where data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index 87df9fe0d..82e32f598 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -612,6 +612,7 @@ pub(crate) async fn save_transition_transaction_record( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -658,6 +659,7 @@ pub(crate) async fn save_transition_transaction_record_if_current( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag), ..Default::default() diff --git a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs index 6768c9717..ddcbce8da 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs @@ -684,6 +684,7 @@ async fn write_checkpoint( }; let opts = ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(preconditions), ..Default::default() }; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 4973f6831..94f3a6344 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -5493,6 +5493,7 @@ where fence.ensure_held()?; let mut opts = ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, no_lock: true, http_preconditions: Some(pool_meta_cas_preconditions(token, object)?), ..Default::default() @@ -14412,6 +14413,7 @@ impl ECStore { encoded.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -14566,6 +14568,7 @@ impl ECStore { encoded, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(http_preconditions), ..Default::default() }, @@ -14957,6 +14960,7 @@ impl ECStore { encoded, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag), ..Default::default() diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index e5eccba32..b98b294ca 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -317,6 +317,22 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper { dst_path: &str, external_guard: Option>, ) -> Result { + self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard) + .await + .result + } +} + +impl LocalDiskWrapper { + pub(in crate::disk) async fn rename_data_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + external_guard: Option>, + ) -> super::RenameDataObservation { let operation = self.clone(); let src_volume = src_volume.to_owned(); let src_path = src_path.to_owned(); @@ -333,22 +349,35 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper { } else { get_max_timeout_duration() }; - run_owned_mutation(external_guard, move || async move { - operation + let observed = run_owned_mutation(external_guard, move || async move { + let mut preflight_rejection = None; + let result = operation .track_disk_health_mutation( "rename_data", DiskMetricMutation::Write, || async { - operation - .disk - .rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path) - .await + // Preserve the former DiskAPI future's single boxing boundary. + let observed = + Box::pin( + operation + .disk + .rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path), + ) + .await; + preflight_rejection = observed.preflight_rejection; + observed.result }, timeout_duration, ) - .await + .await; + // Health tracking must observe the real disk error, not an Ok tuple. + Ok(super::RenameDataObservation { + result, + preflight_rejection, + }) }) - .await + .await; + observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error))) } } @@ -2588,6 +2617,46 @@ mod tests { assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1)); } + #[tokio::test] + async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() { + for source_exists in [false, true] { + for guarded in [false, true] { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")) + .expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + if source_exists { + disk.make_volume("source").await.expect("source volume should exist"); + } + let wrapper = LocalDiskWrapper::new(disk, false); + let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc); + let mut file_info = FileInfo::new("object", 1, 0); + file_info.mod_time = Some(::time::OffsetDateTime::now_utc()); + file_info.erasure.index = 1; + let observed = wrapper + .rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard) + .await; + assert!(observed.rejected_before_publication(), "normal access rejection must carry proof"); + assert!(matches!(observed.result, Err(DiskError::VolumeNotFound))); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1)); + assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok"); + assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded)); + + wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + let observed = wrapper + .rename_data_observed("source", "object", &file_info, "missing-destination", "object", None) + .await; + assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof"); + assert!(matches!(observed.result, Err(DiskError::FaultyDisk))); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.total_errors_availability, 1); + assert_eq!(snapshot.total_writes, 0); + } + } + } + #[tokio::test] async fn local_disk_health_wrapper_counts_returned_availability_errors() { let dir = tempfile::tempdir().expect("temp dir should be created"); diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 6baa92a3e..9cd683578 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -12,6 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub(in crate::disk) use self::commit::LocalRenamePreflightRejection; +#[cfg(test)] +use self::commit::lock_rename_commit_directories; + +mod commit; + use crate::crash_inject::{self, CrashPoint}; use crate::data_usage::local_snapshot::ensure_data_usage_layout; use crate::diagnostics::get::{ @@ -26,10 +32,10 @@ use crate::disk::{ BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, - PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, - QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, - ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, - SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, + PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, + RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, + STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + conv_part_err_to_int, endpoint::Endpoint, error::{DiskError, Error, FileAccessDeniedWithContext, Result}, error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}, @@ -184,65 +190,6 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r } } -fn rollback_committed_rename_std( - dst_file_path: &Path, - new_data_path: Option<&Path>, - rollback_data_dir: Option, -) -> std::io::Result<()> { - if let Some(old_data_dir) = rollback_data_dir { - let Some(dst_parent) = dst_file_path.parent() else { - return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent")); - }; - let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); - std::fs::rename(backup_path, dst_file_path)?; - } else { - remove_file_if_exists(dst_file_path)?; - } - - if let Some(new_data_path) = new_data_path { - remove_dir_all_if_exists(new_data_path)?; - } - - Ok(()) -} - -fn rollback_inline_metadata_commit_std( - dst_file_path: &Path, - rollback_data_dir: Option, - local_rollback_path: Option<&Path>, -) -> std::io::Result<()> { - if let Some(backup_path) = local_rollback_path { - // The commit immediately before this rollback renamed the staged - // xl.meta from the same directory as `backup_path` onto - // `dst_file_path`, proving both paths are on the same filesystem. - // Unix rename atomically replaces the committed destination; never - // unlink it first or an interrupted rollback could lose xl.meta. - std::fs::rename(backup_path, dst_file_path)?; - } else { - rollback_committed_rename_std(dst_file_path, None, rollback_data_dir)?; - } - Ok(()) -} - -fn create_local_inline_rollback_backup( - dst_file_path: &Path, - staging_file_path: &Path, - old_metadata: &[u8], -) -> std::io::Result { - let Some(staging_parent) = staging_file_path.parent() else { - return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent")); - }; - let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP); - remove_file_if_exists(&backup_path)?; - if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err()) - && let Err(err) = std::fs::write(&backup_path, old_metadata) - { - let _ = remove_file_if_exists(&backup_path); - return Err(err); - } - Ok(backup_path) -} - async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> { let backup_dir = object_dir.join(rollback_dir.to_string()); fs::create_dir_all(&backup_dir).await.map_err(to_file_error)?; @@ -269,126 +216,6 @@ async fn restore_metadata_backup( Ok(()) } -async fn lock_rename_commit_directories( - source_parent: &Path, - destination_parent: &Path, - base_dir: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result { - #[cfg(windows)] - let result = { - let source_parent = source_parent.to_path_buf(); - let destination_parent = destination_parent.to_path_buf(); - let base_dir = base_dir.to_path_buf(); - let publication_root = publication_root.clone(); - os::run_blocking_namespace_operation(mutation_lease, move || { - let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root); - #[cfg(test)] - if result.is_ok() { - run_destination_commit_directory_preparation(&destination_parent); - } - result - }) - .await - }; - #[cfg(not(windows))] - let result = { - let _ = mutation_lease; - os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) - }; - - let result = result.map_err(|err| match std::fs::symlink_metadata(base_dir) { - Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err, - _ => err, - }); - - result.map_err(to_file_error).map_err(DiskError::from) -} - -async fn read_rename_destination_metadata( - file_path: &Path, - rename_commit_guard: &os::RenameCommitGuard, - mutation_lease: Arc, -) -> Result> { - #[cfg(windows)] - let result = { - let file_path = file_path.to_path_buf(); - let rename_commit_guard = rename_commit_guard.clone(); - os::run_blocking_namespace_operation(mutation_lease, move || { - os::read_destination_file_with_commit_guard(&file_path, &rename_commit_guard) - }) - .await - }; - #[cfg(not(windows))] - let _ = (rename_commit_guard, mutation_lease); - #[cfg(not(windows))] - let result = match super::fs::read_file(file_path).await { - Ok(data) => Ok(Some(data)), - Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - }; - - result - .map(|data| data.map(Bytes::from)) - .map_err(to_file_error) - .map_err(DiskError::from) -} - -async fn restore_renamed_data_source( - src_volume_dir: &Path, - src_data_path: &Path, - dst_data_path: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result<()> { - if fs::symlink_metadata(src_data_path).await.is_ok() { - return Ok(()); - } - let result = - match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await { - Ok(()) => Ok(()), - Err(DiskError::FileNotFound) => { - let source_exists = fs::symlink_metadata(src_data_path).await.is_ok(); - let destination_missing = matches!( - fs::symlink_metadata(dst_data_path).await, - Err(err) if err.kind() == ErrorKind::NotFound - ); - if source_exists && destination_missing { - Ok(()) - } else { - Err(DiskError::FileNotFound) - } - } - Err(err) => Err(err), - }; - if let Err(err) = &result { - warn!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "restore_staged_data_source_failed", - src_path = ?src_data_path, - dst_path = ?dst_data_path, - error = ?err, - "Failed to restore staged data after a metadata commit was rejected" - ); - } - result -} - -async fn restore_published_data_source( - data_paths: Option<&(PathBuf, PathBuf)>, - src_volume_dir: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result<()> { - let Some((src_data_path, dst_data_path)) = data_paths else { - return Ok(()); - }; - restore_renamed_data_source(src_volume_dir, src_data_path, dst_data_path, publication_root, mutation_lease).await -} - async fn restore_delete_rollback( object_dir: &Path, xl_path: &Path, @@ -9040,7 +8867,6 @@ impl DiskAPI for LocalDisk { Ok(()) } - #[tracing::instrument(level = "trace", skip_all)] async fn rename_data( &self, src_volume: &str, @@ -9049,966 +8875,8 @@ impl DiskAPI for LocalDisk { dst_volume: &str, dst_path: &str, ) -> Result { - crate::hp_guard!("LocalDisk::rename_data"); - let mut fi = fi; - // A non-force DeleteBucket must not remove a directory while a local - // object commit is publishing into it. The peer's empty scan remains - // optimistic; this lease establishes the local commit/delete order and - // remains owned by any blocking syscall that outlives async cancellation. - let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; - let quota_fence_token = - match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { - Some(value) => { - let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; - Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) - } - None if rustfs_utils::http::metadata_compat::contains_key_str( - &fi.metadata, - QUOTA_MUTATION_FENCE_METADATA_SUFFIX, - ) => - { - return Err(DiskError::FileCorrupt); - } - None => None, - }; - rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); - let quota_fence_claim = match quota_fence_token { - Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), - None => None, - }; - let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; - if let Some(claim) = quota_fence_claim { - mutation_lease.attach_external_guard(claim); - } - if fi.is_legacy_indexed_delete_marker() { - fi.erasure.index = 0; - } - fi.validate_for_metadata_read()?; - // Snapshot the destination part paths before `fi` is consumed below. These - // are the descriptors a reader may hold for the version this call is about - // to replace (backlog#1145); readers build the identical string in - // `io_primitives`. An inline-data version has no parts and yields none. - let invalidate_part_paths: Vec = { - let data_dir = fi.data_dir.unwrap_or_default(); - fi.parts - .iter() - .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) - .collect() - }; - let src_volume_dir = self.io_get_bucket_path(src_volume)?; - if !skip_access_checks(src_volume) - && let Err(e) = super::fs::access_std(&src_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?src_volume_dir, - operation = "rename_data_src_access", - error = %e, - "Disk local access check failed" - ); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; - if !skip_access_checks(dst_volume) - && let Err(e) = super::fs::access_std(&dst_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?dst_volume_dir, - operation = "rename_data_dst_access", - error = %e, - "Disk local access check failed" - ); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - // xl.meta path - let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; - let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; - - // data_dir path - let has_data_dir_path = { - let has_data_dir = { - if !fi.is_remote() { - fi.data_dir - .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) - } else { - None - } - }; - - if let Some(data_dir) = has_data_dir { - let src_data_path = self.io_get_object_path( - src_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), - )?; - let dst_data_path = self.io_get_object_path( - dst_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), - )?; - - Some((src_data_path, dst_data_path)) - } else { - None - } - }; - - check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; - check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - - let no_inline = fi.data.is_none() && fi.size > 0; - // Captured before `fi` is consumed by add_version; gates the stale - // destination purge below. - let fi_healing = fi.is_healing(); - - // Resolved once for the whole commit so a concurrent configuration - // change can never leave a single rename_data half-synced. The tier is - // keyed on the destination volume: user data staged in scratch - // namespaces follows the configured tier, while commits into - // system-critical namespaces (IAM, config, bucket metadata) stay - // pinned to strict. - let durability = effective_durability(dst_volume); - - let src_file_parent = src_file_path - .parent() - .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; - let dst_file_parent = dst_file_path - .parent() - .ok_or_else(|| DiskError::other("missing object metadata parent"))?; - if !no_inline { - fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; - } - // Acquire the common trees before reading destination metadata. On - // Windows this pins the object directory identity across metadata - // preparation, data publication, rollback backup, and final commit. - let rename_commit_guard = lock_rename_commit_directories( - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; - - if no_inline { - // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta - let mut xlmeta = FileMeta::new(); - // An existing dst xl.meta that fails to parse leaves `xlmeta` empty - // and gets overwritten by the commit below (pre-existing behavior); - // track that so the old-size observation reports unknown instead of - // a false `Absent` (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(dst_buf) = has_dst_buf.as_ref() { - if FileMeta::is_xl2_v1_format(dst_buf) - && let Ok(nmeta) = FileMeta::load(dst_buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let mut skip_parent = dst_volume_dir.clone(); - if has_dst_buf.as_ref().is_some() - && let Some(parent) = dst_file_path.parent() - { - skip_parent = parent.to_path_buf(); - } - - let version_id = fi.version_id.unwrap_or_default(); - let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = has_old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - if let Some(old_data_dir) = has_old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *old_data_dir); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_dst_buf = xlmeta.marshal_msg()?; - - // This tmp xl.meta is renamed onto dst_file_path at the commit - // point below, so only its contents must be durable before the - // rename (SyncMode::FileOnly); the dst parent directory is fsynced - // after the commit rename, and a crash before the rename means the - // PUT was never acknowledged. A metadata commit: relaxed tiers - // leave it to the page cache. - let tmp_meta_sync = if durability.syncs_commit_metadata() { - SyncMode::FileOnly - } else { - SyncMode::None - }; - // The tmp xl.meta write and the shard-file fdatasync are independent - // (disjoint paths) and both only need to be durable before the commit - // renames below, so run them concurrently to drop a blocking - // round-trip from the PUT commit critical path (rustfs/backlog#922 - // step 2). The "contents durable -> rename -> dst dir fsync" ordering - // is unchanged — both futures complete before any rename — which the - // rename_data crash-consistency harness (backlog#935) exercises. - // - // Shard durability: once rename_data succeeds the write is - // acknowledged, so data must not live only in the page cache. - // Multipart parts were already synced during rename_part, so their - // fdatasync here is a cheap no-op. A missing source dir is left for the - // rename below to report through the existing rollback path. Payload - // durability is kept by both strict and relaxed. - let tmp_meta_write = { - let src_file_path = src_file_path.clone(); - let dst_file_path = dst_file_path.clone(); - let rename_commit_guard = rename_commit_guard.clone(); - let mutation_lease = mutation_lease.clone(); - async move { - os::run_blocking_namespace_operation(mutation_lease, move || { - #[cfg(test)] - run_owned_file_write_before_open(&src_file_path); - let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( - &src_file_path, - &dst_file_path, - &rename_commit_guard, - )?; - prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; - Ok(prepared_metadata_source) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from) - } - }; - let shard_sync = async { - if durability.syncs_data_shards() - && let Some((src_data_path, _)) = has_data_dir_path.as_ref() - && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await - && err.kind() != ErrorKind::NotFound - { - return Err::<(), DiskError>(to_file_error(err).into()); - } - Ok(()) - }; - let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); - // Surface a tmp-meta failure first (its prior serial position), then a - // shard-sync failure; either aborts before any rename, exactly as the - // sequential version did. - let prepared_metadata_source = tmp_meta_res?; - shard_sync_res?; - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - std::fs::remove_file(&src_file_path).map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - // Heal reuses the version's data_dir, so for in-place corruption - // the destination dir still exists — and rename(2) cannot replace - // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge - // it first, healing commits only; fresh PUTs mint a new data_dir - // and never collide. Best effort: a real failure surfaces in the - // rename below. - if fi_healing - && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = self.move_to_trash(dst_data_path, true, false).await - { - warn!( - event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - dst_path = ?dst_data_path, - error = ?err, - "Healing commit could not purge the stale destination data dir" - ); - } - if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = os::rename_all_with_commit_guard( - src_data_path, - dst_data_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_data_path_failed", - src_path = ?src_data_path, - dst_path = ?dst_data_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - #[cfg(test)] - if has_data_dir_path.is_some() { - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - } - - // Crash-consistency injection: hard power loss after the data dir - // is in place but before xl.meta commits. No cleanup — the harness - // reopens the disk and asserts the object still reads as the old - // version (the staged data dir is a harmless orphan for GC). - if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { - return Err(DiskError::Unexpected); - } - - if should_fail_before_old_metadata_backup(dst_path) { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "test_fail_before_old_metadata_backup", - "Disk local rename flow failed before metadata commit" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::Unexpected); - } - - // The rollback backup stays where it is written (no rename) and is - // the sole restore source for a later undo_write, so under strict - // it keeps SyncMode::FileAndDir: contents and directory entry both - // durable. It is part of the metadata commit machinery, so relaxed - // tiers leave it to the page cache like the xl.meta it mirrors. - let backup_sync = if durability.syncs_commit_metadata() { - SyncMode::FileAndDir - } else { - SyncMode::None - }; - if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { - let backup_parent = dst_file_parent.join(old_data_dir.to_string()); - #[cfg(not(windows))] - if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { - Ok(guard) => guard, - Err(err) => { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::from(to_file_error(err))); - } - }; - let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); - if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { - #[cfg(windows)] - drop(backup_path_guard); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_bytes = dst_buf.clone(); - // Keep the volume, commit-tree, and exact destination-path - // guards in this task until the backup write and durability - // sync finish. A detached spawn_blocking writer could survive - // cancellation and later truncate a newer transaction's - // deterministic rollback backup. - let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { - #[cfg(test)] - run_owned_file_write_before_open(&backup_path); - backup_path_guard.write_file_for_path_access( - &backup_path, - backup_bytes.as_ref(), - backup_sync != SyncMode::None, - backup_sync == SyncMode::FileAndDir, - ) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from); - if let Err(err) = write_result { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "write_old_metadata_backup_failed", - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - } - - // Crash-consistency injection: hard power loss after the rollback - // backup is durable but before the xl.meta commit rename. No - // cleanup — the harness asserts the object still reads as the old - // version, since the destination xl.meta is untouched here. - if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - if let Err(err) = os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) + self.rename_data_inner(src_volume, src_path, fi, dst_volume, dst_path, &mut None) .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_metadata_failed", - src_path = ?src_file_path, - dst_path = ?dst_file_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - - let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); - if should_fail_after_metadata_commit(dst_path) { - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - return Err(DiskError::Unexpected); - } - - // Crash-consistency injection: hard power loss immediately after the - // xl.meta commit rename but before the durability fsync. Unlike the - // graceful failpoint above, no rollback runs — the commit rename is - // already on disk, so the harness asserts the object reads back as - // the new version. - if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - // Persist the directory entries for both the data dir and xl.meta renames; - // without this the commit itself can vanish on power loss. Relaxed tiers - // accept that window (documented in docs/operations/durability-modes.md). - if durability.syncs_commit_metadata() - && let Some(parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // The commit rename changed the dst part inodes before this fsync - // failed and rolled them back; drop any fd cached during that - // window so readers re-open the restored inode (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // First PUT of an object creates its directory (and any missing prefix - // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The - // commit fsync above persists the object dir's *contents*, not its own - // entry in the bucket/prefix dir, so on power loss after ack the whole - // object dir could vanish (rustfs/backlog#922 step 4). For a new object - // (no prior xl.meta) fsync the ancestor chain from the object dir's - // parent up to and including the bucket so those new directory entries - // are durable. Overwrites already have a durable object dir. The - // starts_with guard bounds the walk to the bucket subtree. Relaxed/none - // accept the wider window, like the commit fsync above. - if has_dst_buf.is_none() && durability.syncs_commit_metadata() { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(dir) = ancestor { - if !dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dir(dir).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // Same post-commit rollback window as above — drop cached - // dst part fds so readers re-open the restored inode - // (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if dir == dst_volume_dir.as_path() { - break; - } - ancestor = dir.parent(); - } - } - - // Publication and every rollback-capable durability step are now - // complete. Do not retain the Windows object identity guard while - // cleaning staging paths or invalidating cached descriptors. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(src_file_path_parent) = src_file_path.parent() { - if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { - let _ = std::fs::remove_dir(src_file_path_parent); - } else { - let _ = self - .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) - .await; - } - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: has_old_data_dir, - rollback_data_dir, - cleanup_data_dir: has_old_data_dir, - sign: version_signature, - old_current_size, - }) - } else { - // Inline metadata preparation is blocking. The transaction lease is - // moved into that work so a timeout can release the async waiter without - // allowing a retry to reuse the deterministic staging path too early. - let src = src_file_path.clone(); - let dst = dst_file_path.clone(); - let cleanup_path = if src_volume == super::RUSTFS_META_MULTIPART_BUCKET { - src_file_path.parent().map(|p| p.to_path_buf()) - } else { - None - }; - let dst_path_for_failpoint = dst_path.to_string(); - #[cfg(windows)] - let source_parent = src_file_parent.to_path_buf(); - let rename_commit_guard_for_preparation = rename_commit_guard.clone(); - let sync = durability.syncs_commit_metadata(); - #[cfg(test)] - run_inline_before_file_sync_admission(dst_path); - let mut file_sync_admission = if sync { - Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ) - } else { - None - }; - let prepare_inline_metadata = move || { - let mut prepared_metadata_source = - os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; - #[cfg(windows)] - let source_metadata_guard = - rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; - let mut xlmeta = FileMeta::new(); - // Same as the non-inline branch: an unparsable existing dst - // xl.meta must surface as unknown, not `Absent` - // (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(ref buf) = has_dst_buf { - if FileMeta::is_xl2_v1_format(buf) - && let Ok(nmeta) = FileMeta::load(buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let version_id = fi.version_id.unwrap_or_default(); - let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - let mut staged_rollback_path = None; - if let Some(d) = old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *d); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_buf = xlmeta.marshal_msg()?; - // Write the staged xl.meta. Inline objects carry their data inside - // xl.meta, so this is the durable preparation for the metadata commit: - // relaxed tiers do no per-object fsync here at all (aligned - // with MinIO's default), trading a documented power-loss - // window for latency. - prepared_metadata_source.write_all(&new_buf, sync)?; - run_inline_preparation_before_backup(&dst_path_for_failpoint); - if let Some(ref old_metadata) = has_dst_buf - && (rollback_data_dir.is_some() || sync || cfg!(test)) - { - #[cfg(windows)] - let backup_path = { - let backup_path = src - .parent() - .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? - .join(STORAGE_FORMAT_FILE_BACKUP); - source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; - backup_path - }; - #[cfg(not(windows))] - let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; - #[cfg(not(windows))] - if sync { - std::fs::File::open(&backup_path)?.sync_data()?; - } - staged_rollback_path = Some(backup_path); - } - - Ok::<_, std::io::Error>(( - rollback_data_dir, - old_data_dir, - version_signature, - old_current_size, - staged_rollback_path, - has_dst_buf.is_none(), - prepared_metadata_source, - )) - }; - let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { - os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await - } else { - os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await - } - .map_err(to_file_error) - .map_err(DiskError::from); - - let ( - rollback_data_dir, - cleanup_data_dir, - version_signature, - old_current_size, - mut local_rollback_path, - destination_was_absent, - prepared_metadata_source, - ) = match inline_preparation { - Ok(prepared) => prepared, - Err(err) => { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - }; - - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - let remove_result = std::fs::remove_file(&src_file_path); - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - remove_result.map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { - let Some(dst_parent) = dst_file_path.parent() else { - return Err(DiskError::other("missing object metadata parent")); - }; - let backup_path = dst_parent - .join(rollback_data_dir.to_string()) - .join(STORAGE_FORMAT_FILE_BACKUP); - // rename_all acquires the backup path's namespace lease. Do not - // hold a disk admission while acquiring another namespace lock. - drop(file_sync_admission.take()); - if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { - let _ = remove_file_if_exists(staged_backup); - return Err(err); - } - #[cfg(test)] - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - if sync { - file_sync_admission = Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ); - } - if let Some(admission) = file_sync_admission.as_ref() - && let Some(backup_parent) = backup_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - return Err(DiskError::from(to_file_error(err))); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - } - local_rollback_path = None; - } - - let commit_result = if should_fail_commit_rename(dst_path) { - Err(DiskError::other("test fail during metadata commit rename")) - } else { - os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &dst_volume_dir, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - }; - if let Err(err) = commit_result { - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - - let post_commit = async { - if should_fail_after_metadata_commit(dst_path) { - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(std::io::Error::other("test fail after metadata commit")); - } - - // Persist the commit rename's directory entry across power loss. - if let Some(admission) = file_sync_admission.as_ref() - && let Some(dst_parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) - .await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // Same power-loss gap as the non-inline path (rustfs/backlog#922 - // step 4): a first PUT creates the object dir (and any missing - // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all - // never fsynced. The fsync above persists the object dir's contents, - // not its own entry, so for a new inline object fsync the ancestor - // chain up to and including the bucket. Overwrites already have a - // durable object dir; the starts_with guard bounds the walk. - if let Some(admission) = file_sync_admission.as_ref() - && destination_was_absent - { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(ancestor_dir) = ancestor { - if !ancestor_dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std( - &dst_file_path, - rollback_data_dir, - local_rollback_path.as_deref(), - )?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if ancestor_dir == dst_volume_dir.as_path() { - break; - } - ancestor = ancestor_dir.parent(); - } - } - - Ok::<(), std::io::Error>(()) - } - .await; - - // The disk admission protects the durability chain, not staging - // cleanup or cache invalidation after that chain has completed. - drop(file_sync_admission.take()); - - // A post-commit rollback (for example, a commit-metadata fsync - // failure under strict durability) restores the old metadata; drop any - // descriptors cached during the committed window before propagating the - // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so - // this is mostly defensive and keeps both commit branches consistent. - if let Err(err) = post_commit { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(DiskError::from(err)); - } - - // The commit no longer has a rollback path. Release the Windows - // object identity guard before best-effort staging cleanup. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - - // Cleanup - if let Some(ref cleanup) = cleanup_path { - let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; - } else if let Some(parent) = src_file_path.parent() { - let _ = std::fs::remove_dir(parent); - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: cleanup_data_dir, - rollback_data_dir, - cleanup_data_dir, - sign: version_signature, - old_current_size, - }) - } } #[tracing::instrument(level = "trace", skip_all)] @@ -11055,6 +9923,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf #[cfg(test)] mod test { + use super::commit::create_local_inline_rollback_backup; use super::*; use rustfs_filemeta::ErasureInfo; use std::io::{self, Write}; @@ -14102,6 +12971,196 @@ mod test { ); } + #[tokio::test] + async fn observed_rename_timeout_has_no_preflight_proof_and_retains_namespace_lease() { + use crate::disk::disk_store::LocalDiskWrapper; + use futures::FutureExt; + use std::sync::mpsc; + + temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let bucket = "observed-timeout-bucket"; + let object = "prefix/object"; + let tmp_object = "observed-timeout-stage"; + let data_dir = Uuid::new_v4(); + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + let staged_part = disk + .get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}/part.1")) + .expect("staged part path should resolve"); + fs::create_dir_all(staged_part.parent().expect("staged part should have a parent")) + .await + .expect("staged data directory should be created"); + fs::write(&staged_part, b"new-payload") + .await + .expect("staged part should be written"); + let staged_metadata = disk + .get_object_path_for_io(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{STORAGE_FORMAT_FILE}")) + .expect("staged metadata path should resolve"); + let destination = disk.io_get_object_path(bucket, object).expect("destination should resolve"); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + set_owned_file_write_before_open(&staged_metadata, move || { + entered_tx.send(()).expect("signal staged writer entry"); + // Dropping the sender also unblocks the syscall if the test fails. + let _ = release_rx.recv(); + }); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let operation = wrapper.clone(); + let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None); + let rename = tokio::spawn(async move { + operation + .rename_data_observed(RUSTFS_META_TMP_BUCKET, tmp_object, &fi, bucket, object, None) + .await + }); + tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10))) + .await + .expect("staged writer waiter should run") + .expect("rename must enter the real staged metadata write"); + + // Advance only after the blocking syscall owns its lease and the wrapper's timer exists. + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + let observed = tokio::time::timeout(Duration::from_secs(5), rename) + .await + .expect("wrapper timeout must not wait for the blocked syscall") + .expect("the wrapper waiter must not panic"); + assert!(!observed.rejected_before_publication(), "a timeout must carry no local preflight proof"); + assert!(matches!(observed.result, Err(DiskError::Timeout))); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1)); + assert_eq!(snapshot.total_errors_timeout, 1); + assert_eq!(snapshot.total_writes, 0); + assert_eq!(snapshot.total_waiting, 0); + let volume_lock = os::disk_volume_mutation_lock(&disk.root, bucket); + assert!( + Arc::clone(&volume_lock).try_write_owned().is_err(), + "the blocked syscall must retain its volume guard" + ); + assert!( + os::acquire_rename_data_mutation_lease(&disk.root, bucket, &destination) + .now_or_never() + .is_none(), + "a same-object mutation must still wait for the blocked syscall" + ); + assert_eq!(fs::read(&staged_part).await.expect("staged data must remain"), b"new-payload"); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists()); + + release_tx.send(()).expect("release timed-out staged writer"); + let lease = tokio::time::timeout( + Duration::from_secs(5), + os::acquire_rename_data_mutation_lease(&disk.root, bucket, &destination), + ) + .await + .expect("the namespace lease must be released when the syscall drains"); + drop(lease); + let _exclusive = tokio::time::timeout(Duration::from_secs(5), volume_lock.write_owned()) + .await + .expect("the volume guard must be released when the syscall drains"); + assert!( + !destination.join(STORAGE_FORMAT_FILE).exists(), + "timed-out waiter must not publish metadata later" + ); + }) + .await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn observed_rename_owned_task_panic_has_no_preflight_proof_and_releases_guard() { + use crate::disk::disk_store::LocalDiskWrapper; + use std::sync::mpsc; + + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let bucket = "observed-panic-bucket"; + let object = "prefix/object"; + let tmp_object = "observed-panic-stage"; + let data_dir = Uuid::new_v4(); + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + let staged_part = disk + .get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}/part.1")) + .expect("staged part path should resolve"); + fs::create_dir_all(staged_part.parent().expect("staged part should have a parent")) + .await + .expect("staged data directory should be created"); + fs::write(&staged_part, b"new-payload") + .await + .expect("staged part should be written"); + let destination = disk.io_get_object_path(bucket, object).expect("destination should resolve"); + let published_part = destination.join(data_dir.to_string()).join("part.1"); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { + entered_tx.send(()).expect("signal data publication"); + let _ = release_rx.recv(); + // This hook runs in the owned async mutation, outside spawn_blocking. + panic!("injected observed rename owner panic after publication"); + }); + let external_guard = Arc::new(()); + let guard_probe = Arc::downgrade(&external_guard); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let operation = wrapper.clone(); + let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None); + let rename = tokio::spawn(async move { + operation + .rename_data_observed(RUSTFS_META_TMP_BUCKET, tmp_object, &fi, bucket, object, Some(external_guard)) + .await + }); + tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10))) + .await + .expect("publication waiter should run") + .expect("rename must publish data before the injected owner panic"); + assert!(guard_probe.upgrade().is_some(), "the owned task must retain the publication guard"); + assert_eq!(fs::read(&published_part).await.expect("new data must be published"), b"new-payload"); + assert!(!staged_part.exists(), "the real data rename must have consumed staging"); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists()); + let volume_lock = os::disk_volume_mutation_lock(&disk.root, bucket); + assert!( + Arc::clone(&volume_lock).try_write_owned().is_err(), + "the mutation must retain its volume guard" + ); + + release_tx.send(()).expect("release mutation owner into the injected panic"); + let observed = tokio::time::timeout(Duration::from_secs(5), rename) + .await + .expect("owned task panic must reach the wrapper") + .expect("the wrapper must convert the inner task panic into an error"); + assert!( + !observed.rejected_before_publication(), + "a join failure must carry no local preflight proof" + ); + assert!(matches!(observed.result, Err(DiskError::Io(error)) if error.to_string() == "owned mutation task failed")); + assert!( + guard_probe.upgrade().is_none(), + "the guard must be released after the mutation owner unwinds" + ); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1)); + assert_eq!(snapshot.total_writes, 0); + assert_eq!(snapshot.total_waiting, 0); + let lease = tokio::time::timeout( + Duration::from_secs(5), + os::acquire_rename_data_mutation_lease(&disk.root, bucket, &destination), + ) + .await + .expect("panic must release the namespace lease"); + drop(lease); + let _exclusive = tokio::time::timeout(Duration::from_secs(5), volume_lock.write_owned()) + .await + .expect("panic must release the volume guard"); + assert_eq!( + fs::read(&published_part).await.expect("published recovery data must remain"), + b"new-payload" + ); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists()); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn windows_and_unix_cancelled_staged_metadata_write_serializes_same_object_retry() { use std::sync::{Arc, mpsc}; diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs new file mode 100644 index 000000000..9d51bc1cd --- /dev/null +++ b/crates/ecstore/src/disk/local/commit.rs @@ -0,0 +1,1233 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Single-disk object rename publication and rollback. The shared execution core +//! retains instrumentation, mutation leases, and commit guards through the syscall. + +#[cfg(all(test, windows))] +use super::run_destination_commit_directory_preparation; +use super::{ + EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE, + LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size, + remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature, + run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup, + should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit, + skip_access_checks, +}; +#[cfg(test)] +use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication}; +use crate::crash_inject::{self, CrashPoint}; +use crate::disk::{ + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, + error::{DiskError, Result}, + error_conv::{to_access_error, to_file_error}, + os, + os::{check_path_length, rename_all}, +}; +use bytes::Bytes; +use rustfs_filemeta::{FileInfo, FileMeta}; +use std::{ + io::ErrorKind, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::fs; +use tracing::{info, warn}; +use uuid::Uuid; + +fn rollback_committed_rename_std( + dst_file_path: &Path, + new_data_path: Option<&Path>, + rollback_data_dir: Option, +) -> std::io::Result<()> { + if let Some(old_data_dir) = rollback_data_dir { + let Some(dst_parent) = dst_file_path.parent() else { + return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent")); + }; + let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); + std::fs::rename(backup_path, dst_file_path)?; + } else { + remove_file_if_exists(dst_file_path)?; + } + + if let Some(new_data_path) = new_data_path { + remove_dir_all_if_exists(new_data_path)?; + } + + Ok(()) +} + +fn rollback_inline_metadata_commit_std( + dst_file_path: &Path, + rollback_data_dir: Option, + local_rollback_path: Option<&Path>, +) -> std::io::Result<()> { + if let Some(backup_path) = local_rollback_path { + // The commit immediately before this rollback renamed the staged + // xl.meta from the same directory as `backup_path` onto + // `dst_file_path`, proving both paths are on the same filesystem. + // Unix rename atomically replaces the committed destination; never + // unlink it first or an interrupted rollback could lose xl.meta. + std::fs::rename(backup_path, dst_file_path)?; + } else { + rollback_committed_rename_std(dst_file_path, None, rollback_data_dir)?; + } + Ok(()) +} + +pub(super) fn create_local_inline_rollback_backup( + dst_file_path: &Path, + staging_file_path: &Path, + old_metadata: &[u8], +) -> std::io::Result { + let Some(staging_parent) = staging_file_path.parent() else { + return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent")); + }; + let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP); + remove_file_if_exists(&backup_path)?; + if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err()) + && let Err(err) = std::fs::write(&backup_path, old_metadata) + { + let _ = remove_file_if_exists(&backup_path); + return Err(err); + } + Ok(backup_path) +} + +pub(super) async fn lock_rename_commit_directories( + source_parent: &Path, + destination_parent: &Path, + base_dir: &Path, + publication_root: &os::PublicationRoot, + mutation_lease: Arc, +) -> Result { + #[cfg(windows)] + let result = { + let source_parent = source_parent.to_path_buf(); + let destination_parent = destination_parent.to_path_buf(); + let base_dir = base_dir.to_path_buf(); + let publication_root = publication_root.clone(); + os::run_blocking_namespace_operation(mutation_lease, move || { + let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root); + #[cfg(test)] + if result.is_ok() { + run_destination_commit_directory_preparation(&destination_parent); + } + result + }) + .await + }; + #[cfg(not(windows))] + let result = { + let _ = mutation_lease; + os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) + }; + + let result = result.map_err(|err| match std::fs::symlink_metadata(base_dir) { + Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err, + _ => err, + }); + + result.map_err(to_file_error).map_err(DiskError::from) +} + +async fn read_rename_destination_metadata( + file_path: &Path, + rename_commit_guard: &os::RenameCommitGuard, + mutation_lease: Arc, +) -> Result> { + #[cfg(windows)] + let result = { + let file_path = file_path.to_path_buf(); + let rename_commit_guard = rename_commit_guard.clone(); + os::run_blocking_namespace_operation(mutation_lease, move || { + os::read_destination_file_with_commit_guard(&file_path, &rename_commit_guard) + }) + .await + }; + #[cfg(not(windows))] + let _ = (rename_commit_guard, mutation_lease); + #[cfg(not(windows))] + let result = match super::super::fs::read_file(file_path).await { + Ok(data) => Ok(Some(data)), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + }; + + result + .map(|data| data.map(Bytes::from)) + .map_err(to_file_error) + .map_err(DiskError::from) +} + +async fn restore_renamed_data_source( + src_volume_dir: &Path, + src_data_path: &Path, + dst_data_path: &Path, + publication_root: &os::PublicationRoot, + mutation_lease: Arc, +) -> Result<()> { + if fs::symlink_metadata(src_data_path).await.is_ok() { + return Ok(()); + } + let result = + match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await { + Ok(()) => Ok(()), + Err(DiskError::FileNotFound) => { + let source_exists = fs::symlink_metadata(src_data_path).await.is_ok(); + let destination_missing = matches!( + fs::symlink_metadata(dst_data_path).await, + Err(err) if err.kind() == ErrorKind::NotFound + ); + if source_exists && destination_missing { + Ok(()) + } else { + Err(DiskError::FileNotFound) + } + } + Err(err) => Err(err), + }; + if let Err(err) = &result { + warn!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "restore_staged_data_source_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Failed to restore staged data after a metadata commit was rejected" + ); + } + result +} + +async fn restore_published_data_source( + data_paths: Option<&(PathBuf, PathBuf)>, + src_volume_dir: &Path, + publication_root: &os::PublicationRoot, + mutation_lease: Arc, +) -> Result<()> { + let Some((src_data_path, dst_data_path)) = data_paths else { + return Ok(()); + }; + restore_renamed_data_source(src_volume_dir, src_data_path, dst_data_path, publication_root, mutation_lease).await +} + +/// Proof produced only when the local rename returns at an existing access +/// preflight, before metadata, backups, or object data can be published. +#[derive(Debug)] +pub(in crate::disk) struct LocalRenamePreflightRejection(()); + +impl LocalDisk { + #[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)] + pub(super) async fn rename_data_inner( + &self, + src_volume: &str, + src_path: &str, + fi: FileInfo, + dst_volume: &str, + dst_path: &str, + preflight_rejection: &mut Option, + ) -> Result { + crate::hp_guard!("LocalDisk::rename_data"); + let mut fi = fi; + // A non-force DeleteBucket must not remove a directory while a local + // object commit is publishing into it. The peer's empty scan remains + // optimistic; this lease establishes the local commit/delete order and + // remains owned by any blocking syscall that outlives async cancellation. + let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; + let quota_fence_token = + match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { + Some(value) => { + let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; + Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) + } + None if rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) => + { + return Err(DiskError::FileCorrupt); + } + None => None, + }; + rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); + let quota_fence_claim = match quota_fence_token { + Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), + None => None, + }; + let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + if let Some(claim) = quota_fence_claim { + mutation_lease.attach_external_guard(claim); + } + if fi.is_legacy_indexed_delete_marker() { + fi.erasure.index = 0; + } + fi.validate_for_metadata_read()?; + // Snapshot the destination part paths before `fi` is consumed below. These + // are the descriptors a reader may hold for the version this call is about + // to replace (backlog#1145); readers build the identical string in + // `io_primitives`. An inline-data version has no parts and yields none. + let invalidate_part_paths: Vec = { + let data_dir = fi.data_dir.unwrap_or_default(); + fi.parts + .iter() + .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) + .collect() + }; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; + if !skip_access_checks(src_volume) + && let Err(e) = super::super::fs::access_std(&src_volume_dir) + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?src_volume_dir, + operation = "rename_data_src_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; + if !skip_access_checks(dst_volume) + && let Err(e) = super::super::fs::access_std(&dst_volume_dir) + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?dst_volume_dir, + operation = "rename_data_dst_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + // xl.meta path + let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; + let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; + + // data_dir path + let has_data_dir_path = { + let has_data_dir = { + if !fi.is_remote() { + fi.data_dir + .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) + } else { + None + } + }; + + if let Some(data_dir) = has_data_dir { + let src_data_path = self.io_get_object_path( + src_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), + )?; + let dst_data_path = self.io_get_object_path( + dst_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), + )?; + + Some((src_data_path, dst_data_path)) + } else { + None + } + }; + + check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; + check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; + + let no_inline = fi.data.is_none() && fi.size > 0; + // Captured before `fi` is consumed by add_version; gates the stale + // destination purge below. + let fi_healing = fi.is_healing(); + + // Resolved once for the whole commit so a concurrent configuration + // change can never leave a single rename_data half-synced. The tier is + // keyed on the destination volume: user data staged in scratch + // namespaces follows the configured tier, while commits into + // system-critical namespaces (IAM, config, bucket metadata) stay + // pinned to strict. + let durability = effective_durability(dst_volume); + + let src_file_parent = src_file_path + .parent() + .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; + let dst_file_parent = dst_file_path + .parent() + .ok_or_else(|| DiskError::other("missing object metadata parent"))?; + if !no_inline { + fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; + } + // Acquire the common trees before reading destination metadata. On + // Windows this pins the object directory identity across metadata + // preparation, data publication, rollback backup, and final commit. + let rename_commit_guard = lock_rename_commit_directories( + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; + + if no_inline { + // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta + let mut xlmeta = FileMeta::new(); + // An existing dst xl.meta that fails to parse leaves `xlmeta` empty + // and gets overwritten by the commit below (pre-existing behavior); + // track that so the old-size observation reports unknown instead of + // a false `Absent` (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(dst_buf) = has_dst_buf.as_ref() { + if FileMeta::is_xl2_v1_format(dst_buf) + && let Ok(nmeta) = FileMeta::load(dst_buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let mut skip_parent = dst_volume_dir.clone(); + if has_dst_buf.as_ref().is_some() + && let Some(parent) = dst_file_path.parent() + { + skip_parent = parent.to_path_buf(); + } + + let version_id = fi.version_id.unwrap_or_default(); + let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = has_old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + if let Some(old_data_dir) = has_old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *old_data_dir); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_dst_buf = xlmeta.marshal_msg()?; + + // This tmp xl.meta is renamed onto dst_file_path at the commit + // point below, so only its contents must be durable before the + // rename (SyncMode::FileOnly); the dst parent directory is fsynced + // after the commit rename, and a crash before the rename means the + // PUT was never acknowledged. A metadata commit: relaxed tiers + // leave it to the page cache. + let tmp_meta_sync = if durability.syncs_commit_metadata() { + SyncMode::FileOnly + } else { + SyncMode::None + }; + // The tmp xl.meta write and the shard-file fdatasync are independent + // (disjoint paths) and both only need to be durable before the commit + // renames below, so run them concurrently to drop a blocking + // round-trip from the PUT commit critical path (rustfs/backlog#922 + // step 2). The "contents durable -> rename -> dst dir fsync" ordering + // is unchanged — both futures complete before any rename — which the + // rename_data crash-consistency harness (backlog#935) exercises. + // + // Shard durability: once rename_data succeeds the write is + // acknowledged, so data must not live only in the page cache. + // Multipart parts were already synced during rename_part, so their + // fdatasync here is a cheap no-op. A missing source dir is left for the + // rename below to report through the existing rollback path. Payload + // durability is kept by both strict and relaxed. + let tmp_meta_write = { + let src_file_path = src_file_path.clone(); + let dst_file_path = dst_file_path.clone(); + let rename_commit_guard = rename_commit_guard.clone(); + let mutation_lease = mutation_lease.clone(); + async move { + os::run_blocking_namespace_operation(mutation_lease, move || { + #[cfg(test)] + run_owned_file_write_before_open(&src_file_path); + let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( + &src_file_path, + &dst_file_path, + &rename_commit_guard, + )?; + prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; + Ok(prepared_metadata_source) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from) + } + }; + let shard_sync = async { + if durability.syncs_data_shards() + && let Some((src_data_path, _)) = has_data_dir_path.as_ref() + && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await + && err.kind() != ErrorKind::NotFound + { + return Err::<(), DiskError>(to_file_error(err).into()); + } + Ok(()) + }; + let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); + // Surface a tmp-meta failure first (its prior serial position), then a + // shard-sync failure; either aborts before any rename, exactly as the + // sequential version did. + let prepared_metadata_source = tmp_meta_res?; + shard_sync_res?; + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + std::fs::remove_file(&src_file_path).map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + // Heal reuses the version's data_dir, so for in-place corruption + // the destination dir still exists — and rename(2) cannot replace + // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge + // it first, healing commits only; fresh PUTs mint a new data_dir + // and never collide. Best effort: a real failure surfaces in the + // rename below. + if fi_healing + && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = self.move_to_trash(dst_data_path, true, false).await + { + warn!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + dst_path = ?dst_data_path, + error = ?err, + "Healing commit could not purge the stale destination data dir" + ); + } + if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = os::rename_all_with_commit_guard( + src_data_path, + dst_data_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_data_path_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + #[cfg(test)] + if has_data_dir_path.is_some() { + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + } + + // Crash-consistency injection: hard power loss after the data dir + // is in place but before xl.meta commits. No cleanup — the harness + // reopens the disk and asserts the object still reads as the old + // version (the staged data dir is a harmless orphan for GC). + if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { + return Err(DiskError::Unexpected); + } + + if should_fail_before_old_metadata_backup(dst_path) { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "test_fail_before_old_metadata_backup", + "Disk local rename flow failed before metadata commit" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::Unexpected); + } + + // The rollback backup stays where it is written (no rename) and is + // the sole restore source for a later undo_write, so under strict + // it keeps SyncMode::FileAndDir: contents and directory entry both + // durable. It is part of the metadata commit machinery, so relaxed + // tiers leave it to the page cache like the xl.meta it mirrors. + let backup_sync = if durability.syncs_commit_metadata() { + SyncMode::FileAndDir + } else { + SyncMode::None + }; + if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { + let backup_parent = dst_file_parent.join(old_data_dir.to_string()); + #[cfg(not(windows))] + if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { + Ok(guard) => guard, + Err(err) => { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::from(to_file_error(err))); + } + }; + let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); + if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { + #[cfg(windows)] + drop(backup_path_guard); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_bytes = dst_buf.clone(); + // Keep the volume, commit-tree, and exact destination-path + // guards in this task until the backup write and durability + // sync finish. A detached spawn_blocking writer could survive + // cancellation and later truncate a newer transaction's + // deterministic rollback backup. + let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { + #[cfg(test)] + run_owned_file_write_before_open(&backup_path); + backup_path_guard.write_file_for_path_access( + &backup_path, + backup_bytes.as_ref(), + backup_sync != SyncMode::None, + backup_sync == SyncMode::FileAndDir, + ) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from); + if let Err(err) = write_result { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "write_old_metadata_backup_failed", + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + } + + // Crash-consistency injection: hard power loss after the rollback + // backup is durable but before the xl.meta commit rename. No + // cleanup — the harness asserts the object still reads as the old + // version, since the destination xl.meta is untouched here. + if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + if let Err(err) = os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_metadata_failed", + src_path = ?src_file_path, + dst_path = ?dst_file_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + + let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); + if should_fail_after_metadata_commit(dst_path) { + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + return Err(DiskError::Unexpected); + } + + // Crash-consistency injection: hard power loss immediately after the + // xl.meta commit rename but before the durability fsync. Unlike the + // graceful failpoint above, no rollback runs — the commit rename is + // already on disk, so the harness asserts the object reads back as + // the new version. + if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + // Persist the directory entries for both the data dir and xl.meta renames; + // without this the commit itself can vanish on power loss. Relaxed tiers + // accept that window (documented in docs/operations/durability-modes.md). + if durability.syncs_commit_metadata() + && let Some(parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // The commit rename changed the dst part inodes before this fsync + // failed and rolled them back; drop any fd cached during that + // window so readers re-open the restored inode (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // First PUT of an object creates its directory (and any missing prefix + // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The + // commit fsync above persists the object dir's *contents*, not its own + // entry in the bucket/prefix dir, so on power loss after ack the whole + // object dir could vanish (rustfs/backlog#922 step 4). For a new object + // (no prior xl.meta) fsync the ancestor chain from the object dir's + // parent up to and including the bucket so those new directory entries + // are durable. Overwrites already have a durable object dir. The + // starts_with guard bounds the walk to the bucket subtree. Relaxed/none + // accept the wider window, like the commit fsync above. + if has_dst_buf.is_none() && durability.syncs_commit_metadata() { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(dir) = ancestor { + if !dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dir(dir).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // Same post-commit rollback window as above — drop cached + // dst part fds so readers re-open the restored inode + // (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if dir == dst_volume_dir.as_path() { + break; + } + ancestor = dir.parent(); + } + } + + // Publication and every rollback-capable durability step are now + // complete. Do not retain the Windows object identity guard while + // cleaning staging paths or invalidating cached descriptors. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(src_file_path_parent) = src_file_path.parent() { + if src_volume != super::super::RUSTFS_META_MULTIPART_BUCKET { + let _ = std::fs::remove_dir(src_file_path_parent); + } else { + let _ = self + .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) + .await; + } + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: has_old_data_dir, + rollback_data_dir, + cleanup_data_dir: has_old_data_dir, + sign: version_signature, + old_current_size, + }) + } else { + // Inline metadata preparation is blocking. The transaction lease is + // moved into that work so a timeout can release the async waiter without + // allowing a retry to reuse the deterministic staging path too early. + let src = src_file_path.clone(); + let dst = dst_file_path.clone(); + let cleanup_path = if src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET { + src_file_path.parent().map(|p| p.to_path_buf()) + } else { + None + }; + let dst_path_for_failpoint = dst_path.to_string(); + #[cfg(windows)] + let source_parent = src_file_parent.to_path_buf(); + let rename_commit_guard_for_preparation = rename_commit_guard.clone(); + let sync = durability.syncs_commit_metadata(); + #[cfg(test)] + run_inline_before_file_sync_admission(dst_path); + let mut file_sync_admission = if sync { + Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ) + } else { + None + }; + let prepare_inline_metadata = move || { + let mut prepared_metadata_source = + os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; + #[cfg(windows)] + let source_metadata_guard = + rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; + let mut xlmeta = FileMeta::new(); + // Same as the non-inline branch: an unparsable existing dst + // xl.meta must surface as unknown, not `Absent` + // (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(ref buf) = has_dst_buf { + if FileMeta::is_xl2_v1_format(buf) + && let Ok(nmeta) = FileMeta::load(buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let version_id = fi.version_id.unwrap_or_default(); + let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + let mut staged_rollback_path = None; + if let Some(d) = old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *d); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_buf = xlmeta.marshal_msg()?; + // Write the staged xl.meta. Inline objects carry their data inside + // xl.meta, so this is the durable preparation for the metadata commit: + // relaxed tiers do no per-object fsync here at all (aligned + // with MinIO's default), trading a documented power-loss + // window for latency. + prepared_metadata_source.write_all(&new_buf, sync)?; + run_inline_preparation_before_backup(&dst_path_for_failpoint); + if let Some(ref old_metadata) = has_dst_buf + && (rollback_data_dir.is_some() || sync || cfg!(test)) + { + #[cfg(windows)] + let backup_path = { + let backup_path = src + .parent() + .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? + .join(STORAGE_FORMAT_FILE_BACKUP); + source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; + backup_path + }; + #[cfg(not(windows))] + let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; + #[cfg(not(windows))] + if sync { + std::fs::File::open(&backup_path)?.sync_data()?; + } + staged_rollback_path = Some(backup_path); + } + + Ok::<_, std::io::Error>(( + rollback_data_dir, + old_data_dir, + version_signature, + old_current_size, + staged_rollback_path, + has_dst_buf.is_none(), + prepared_metadata_source, + )) + }; + let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { + os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await + } else { + os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await + } + .map_err(to_file_error) + .map_err(DiskError::from); + + let ( + rollback_data_dir, + cleanup_data_dir, + version_signature, + old_current_size, + mut local_rollback_path, + destination_was_absent, + prepared_metadata_source, + ) = match inline_preparation { + Ok(prepared) => prepared, + Err(err) => { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + }; + + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + let remove_result = std::fs::remove_file(&src_file_path); + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + remove_result.map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { + let Some(dst_parent) = dst_file_path.parent() else { + return Err(DiskError::other("missing object metadata parent")); + }; + let backup_path = dst_parent + .join(rollback_data_dir.to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + // rename_all acquires the backup path's namespace lease. Do not + // hold a disk admission while acquiring another namespace lock. + drop(file_sync_admission.take()); + if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { + let _ = remove_file_if_exists(staged_backup); + return Err(err); + } + #[cfg(test)] + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + if sync { + file_sync_admission = Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ); + } + if let Some(admission) = file_sync_admission.as_ref() + && let Some(backup_parent) = backup_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + return Err(DiskError::from(to_file_error(err))); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + } + local_rollback_path = None; + } + + let commit_result = if should_fail_commit_rename(dst_path) { + Err(DiskError::other("test fail during metadata commit rename")) + } else { + os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &dst_volume_dir, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + }; + if let Err(err) = commit_result { + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + + let post_commit = async { + if should_fail_after_metadata_commit(dst_path) { + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(std::io::Error::other("test fail after metadata commit")); + } + + // Persist the commit rename's directory entry across power loss. + if let Some(admission) = file_sync_admission.as_ref() + && let Some(dst_parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) + .await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // Same power-loss gap as the non-inline path (rustfs/backlog#922 + // step 4): a first PUT creates the object dir (and any missing + // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all + // never fsynced. The fsync above persists the object dir's contents, + // not its own entry, so for a new inline object fsync the ancestor + // chain up to and including the bucket. Overwrites already have a + // durable object dir; the starts_with guard bounds the walk. + if let Some(admission) = file_sync_admission.as_ref() + && destination_was_absent + { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(ancestor_dir) = ancestor { + if !ancestor_dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std( + &dst_file_path, + rollback_data_dir, + local_rollback_path.as_deref(), + )?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if ancestor_dir == dst_volume_dir.as_path() { + break; + } + ancestor = ancestor_dir.parent(); + } + } + + Ok::<(), std::io::Error>(()) + } + .await; + + // The disk admission protects the durability chain, not staging + // cleanup or cache invalidation after that chain has completed. + drop(file_sync_admission.take()); + + // A post-commit rollback (for example, a commit-metadata fsync + // failure under strict durability) restores the old metadata; drop any + // descriptors cached during the committed window before propagating the + // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so + // this is mostly defensive and keeps both commit branches consistent. + if let Err(err) = post_commit { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(DiskError::from(err)); + } + + // The commit no longer has a rollback path. Release the Windows + // object identity guard before best-effort staging cleanup. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + + // Cleanup + if let Some(ref cleanup) = cleanup_path { + let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; + } else if let Some(parent) = src_file_path.parent() { + let _ = std::fs::remove_dir(parent); + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: cleanup_data_dir, + rollback_data_dir, + cleanup_data_dir, + sign: version_signature, + old_current_size, + }) + } + } + + pub(in crate::disk) async fn rename_data_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> super::super::RenameDataObservation { + let mut preflight_rejection = None; + let result = self + .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection) + .await; + super::super::RenameDataObservation { + result, + preflight_rejection, + } + } +} diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 7801274ef..c2f2c52b4 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -75,6 +75,25 @@ use time::OffsetDateTime; use tokio::io::{AsyncRead, AsyncWrite}; use uuid::Uuid; +/// Local preflight evidence stays outside DiskAPI and the RPC response format. +pub(crate) struct RenameDataObservation { + pub(crate) result: Result, + preflight_rejection: Option, +} + +impl RenameDataObservation { + fn unknown(result: Result) -> Self { + Self { + result, + preflight_rejection: None, + } + } + + pub(crate) fn rejected_before_publication(&self) -> bool { + self.result.is_err() && self.preflight_rejection.is_some() + } +} + const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/"; pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token"; @@ -711,6 +730,36 @@ impl Disk { .await } + pub(crate) async fn rename_data_borrowed_with_fence_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + scanner_publication_lease_token: Option, + ) -> RenameDataObservation { + match self { + Disk::Local(local_disk) => { + local_disk + .rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None) + .await + } + Disk::Remote(remote_disk) => RenameDataObservation::unknown( + remote_disk + .rename_data_borrowed_with_fence( + src_volume, + src_path, + fi, + dst_volume, + dst_path, + scanner_publication_lease_token, + ) + .await, + ), + } + } + pub(crate) async fn rename_data_borrowed_with_fence( &self, src_volume: &str, diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 85e270679..701596cbf 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -870,6 +870,18 @@ impl TierFreeVersionReceiptSink { } } +/// Internal PUT completion boundary; this does not change fsync or write quorum. +#[doc(hidden)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum WriteCompletion { + /// Return at write quorum when the commit owner can retain its guards. + #[default] + Quorum, + /// Drain the rename fan-out before returning. Minority failures still heal + /// after a successful quorum commit; this does not require every disk to succeed. + TailDrained, +} + #[derive(Default, Clone)] pub struct ObjectOptions { // Use the maximum parity (N/2), used when saving server configuration files @@ -896,6 +908,10 @@ pub struct ObjectOptions { /// Persisted bucket incarnation observed before authorization. pub expected_bucket_incarnation_id: Option, pub no_lock: bool, + /// Control-plane writers that immediately read or CAS the same namespace + /// key use TailDrained without changing namespace lock ownership. + #[doc(hidden)] + pub write_completion: WriteCompletion, /// True when an upper layer already holds the object read lock before /// forwarding a no_lock read to the set layer. pub metadata_cache_safe: bool, diff --git a/crates/ecstore/src/services/tier/tier_mutation_intent.rs b/crates/ecstore/src/services/tier/tier_mutation_intent.rs index 288b02969..eef2dcf7a 100644 --- a/crates/ecstore/src/services/tier/tier_mutation_intent.rs +++ b/crates/ecstore/src/services/tier/tier_mutation_intent.rs @@ -460,6 +460,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -556,6 +557,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current_etag.to_string()), ..Default::default() diff --git a/crates/ecstore/src/services/tier/tier_probe_intent.rs b/crates/ecstore/src/services/tier/tier_probe_intent.rs index 3d11402a5..b3d96dc05 100644 --- a/crates/ecstore/src/services/tier/tier_probe_intent.rs +++ b/crates/ecstore/src/services/tier/tier_probe_intent.rs @@ -494,6 +494,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -549,6 +550,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current.record_etag.clone()), ..Default::default() diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index c07a013de..a7c52549d 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -1492,6 +1492,7 @@ pub(in crate::set_disk) fn record_read_repair_dedup(reason: &'static str) { counter!("rustfs_heal_read_repair_dedup_total", "reason" => reason).increment(1); } +#[derive(Debug)] pub(in crate::set_disk) enum ReadRepairAdmissionOutcome { Response(HealAdmissionResult), Failed(String), @@ -3838,6 +3839,286 @@ pub(in crate::set_disk) struct RenameTailOutcome { pub(in crate::set_disk) cleanup: Vec, } +const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback"; + +#[derive(Clone, Copy)] +enum RenameDispatchState { + NotDispatched, + RejectedBeforePublication, + MayHavePublished, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum RenameRollbackOutcome { + NotAttempted(DiskError), + RejectedBeforePublication(DiskError), + Indeterminate(DiskError), + Succeeded, + Failed(DiskError), + Panicked, + Cancelled, +} + +impl RenameRollbackOutcome { + fn stage(&self) -> &'static str { + match self { + Self::NotAttempted(_) => "rename_not_dispatched", + Self::RejectedBeforePublication(_) => "rename_rejected_before_publication", + Self::Indeterminate(_) => "rename_indeterminate", + Self::Succeeded => "undo_succeeded", + Self::Failed(_) => "undo_failed", + Self::Panicked => "undo_panicked", + Self::Cancelled => "undo_cancelled", + } + } + + fn undo_attempted(&self) -> bool { + matches!(self, Self::Succeeded | Self::Failed(_) | Self::Panicked | Self::Cancelled) + } + + fn needs_recovery(&self) -> bool { + matches!(self, Self::Indeterminate(_) | Self::Failed(_) | Self::Panicked | Self::Cancelled) + } +} + +#[derive(Debug, Clone)] +struct RenameRollbackDiskOutcome { + disk_index: usize, + rollback_dir: Option, + outcome: RenameRollbackOutcome, +} + +#[derive(Debug)] +struct RenameRollbackReport { + disks: Vec, +} + +/// Shares rollback completion with the staging owner without replacing the +/// original disk/quorum error returned by the rename operation. +#[derive(Clone, Default)] +pub(in crate::set_disk) struct RenameRollbackReceipt(Arc>); + +impl RenameRollbackReceipt { + pub(in crate::set_disk) fn is_incomplete(&self) -> bool { + self.0 + .get() + .is_some_and(|report| report.disks.iter().any(|disk| disk.outcome.needs_recovery())) + } +} + +async fn inspect_incomplete_rename_rollback( + disks: &[Option], + bucket: &str, + object: &str, + submitter: ReadRepairAdmissionSubmitter, +) -> ReadRepairAdmissionOutcome { + let location = disks.iter().flatten().next().map(|disk| disk.get_disk_location()); + let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options( + bucket.to_string(), + Some(object.to_string()), + false, + Some(HealChannelPriority::High), + location.as_ref().and_then(|location| location.pool_idx), + location.as_ref().and_then(|location| location.set_idx), + ); + // A failed write's surviving minority is not an authoritative heal source. + // Request inspection only: MRF PartialWrite would schedule mutating repair. + request.dry_run = Some(true); + request.remove_corrupted = Some(false); + request.recreate_missing = Some(false); + request.update_parity = Some(false); + request.recursive = Some(false); + match tokio::time::timeout(Duration::from_secs(1), submitter(request)).await { + Ok(result) => result, + Err(_) => ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()), + } +} + +fn rename_rollback_task_outcome( + result: std::result::Result, tokio::task::JoinError>, +) -> RenameRollbackOutcome { + match result { + Ok(Ok(())) => RenameRollbackOutcome::Succeeded, + Ok(Err(err)) => RenameRollbackOutcome::Failed(err), + Err(err) if err.is_panic() => RenameRollbackOutcome::Panicked, + Err(_) => RenameRollbackOutcome::Cancelled, + } +} + +async fn rollback_failed_rename( + disks: &[Option], + file_infos: Vec, + errs: &[Option], + dispatch_states: &[RenameDispatchState], + rollback_dirs: &[Option], + dst: (&str, &str), + receipt: Option, +) { + let owned_disks = disks.to_vec(); + let owned_errs = errs.to_vec(); + let owned_dispatch_states = dispatch_states.to_vec(); + let owned_dirs = rollback_dirs.to_vec(); + let owned_dst = (dst.0.to_string(), dst.1.to_string()); + let coordinator_failure_receipt = receipt.clone(); + // Own both undo mutations and their accounting: a cancelled requester must + // not leave detached disk tasks without the recovery evidence they produce. + let rollback = tokio::spawn(async move { + let disks = owned_disks.as_slice(); + let errs = owned_errs.as_slice(); + let dispatch_states = owned_dispatch_states.as_slice(); + let rollback_dirs = owned_dirs.as_slice(); + let dst = (owned_dst.0.as_str(), owned_dst.1.as_str()); + let mut file_infos = file_infos; + + let (bucket, object) = dst; + let mut outcomes = Vec::with_capacity(disks.len()); + let mut tasks = Vec::with_capacity(disks.len()); + for (disk_index, disk) in disks.iter().enumerate() { + let rollback_dir = rollback_dirs[disk_index]; + let outcome = match &errs[disk_index] { + Some(err) => match dispatch_states[disk_index] { + RenameDispatchState::NotDispatched => RenameRollbackOutcome::NotAttempted(err.clone()), + RenameDispatchState::RejectedBeforePublication => { + RenameRollbackOutcome::RejectedBeforePublication(err.clone()) + } + RenameDispatchState::MayHavePublished => RenameRollbackOutcome::Indeterminate(err.clone()), + }, + None => RenameRollbackOutcome::Failed(DiskError::DiskNotFound), + }; + outcomes.push(RenameRollbackDiskOutcome { + disk_index, + rollback_dir, + outcome, + }); + if errs[disk_index].is_some() { + continue; + } + let Some(disk) = disk.clone() else { + continue; + }; + let fi = std::mem::take(&mut file_infos[disk_index]); + let bucket = bucket.to_string(); + let object = object.to_string(); + let task = tokio::spawn(async move { + #[allow(clippy::let_unit_value)] + let _task_guard = SetDisks::rename_fanout_task_guard(&object); + SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await; + #[cfg(test)] + rollback_fault_injection::before_undo(&object, disk_index)?; + disk.delete_version( + &bucket, + &object, + fi, + false, + DeleteOptions { + undo_write: true, + old_data_dir: rollback_dir, + ..Default::default() + }, + ) + .await + }); + tasks.push(async move { (disk_index, task.await) }); + } + for (disk_index, result) in join_all(tasks).await { + outcomes[disk_index].outcome = rename_rollback_task_outcome(result); + } + + record_rename_rollback_outcomes(disks, outcomes, dst, receipt).await; + }); + if rollback.await.is_err() { + record_indeterminate_rename(disks, dst, coordinator_failure_receipt).await; + } +} + +async fn record_indeterminate_rename(disks: &[Option], dst: (&str, &str), receipt: Option) { + let outcomes = disks + .iter() + .enumerate() + .map(|(disk_index, disk)| RenameRollbackDiskOutcome { + disk_index, + rollback_dir: None, + outcome: if disk.is_some() { + RenameRollbackOutcome::Indeterminate(DiskError::Unexpected) + } else { + RenameRollbackOutcome::NotAttempted(DiskError::DiskNotFound) + }, + }) + .collect(); + record_rename_rollback_outcomes(disks, outcomes, dst, receipt).await; +} + +async fn record_rename_rollback_outcomes( + disks: &[Option], + outcomes: Vec, + dst: (&str, &str), + receipt: Option, +) { + let (bucket, object) = dst; + let attempted = outcomes.iter().filter(|disk| disk.outcome.undo_attempted()).count(); + let failed = outcomes + .iter() + .filter(|disk| disk.outcome.undo_attempted() && disk.outcome.needs_recovery()) + .count(); + let indeterminate = outcomes + .iter() + .filter(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(_))) + .count(); + let succeeded = attempted - failed; + for disk in &outcomes { + counter!("rustfs_rename_rollback_disks_total", "stage" => disk.outcome.stage()).increment(1); + if disk.outcome.needs_recovery() { + let location = disks[disk.disk_index].as_ref().map(|disk| disk.get_disk_location()); + warn!( + event = EVENT_SET_DISK_RENAME_ROLLBACK, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + state = "recovery_required", + stage = disk.outcome.stage(), + pool_index = ?location.as_ref().and_then(|location| location.pool_idx), + set_index = ?location.as_ref().and_then(|location| location.set_idx), + disk_index = disk.disk_index, + bucket, + object, + rollback_dir = ?disk.rollback_dir, + outcome = ?disk.outcome, + attempted, + succeeded, + failed, + indeterminate, + "rename rollback incomplete; preserve recovery material" + ); + } + } + if let Some(receipt) = receipt { + let _ = receipt.0.set(RenameRollbackReport { disks: outcomes }); + } + if failed > 0 || indeterminate > 0 { + let result = inspect_incomplete_rename_rollback(disks, bucket, object, send_read_repair_heal_request).await; + let admission = match &result { + ReadRepairAdmissionOutcome::Response(response) => response.result_label(), + ReadRepairAdmissionOutcome::Failed(_) => "failed", + }; + counter!("rustfs_rename_rollback_inspection_total", "admission" => admission).increment(1); + warn!( + event = EVENT_SET_DISK_RENAME_ROLLBACK, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + state = "recovery_required", + stage = "inspection_admission", + bucket, + object, + attempted, + succeeded, + failed, + indeterminate, + admission, + outcome = ?result, + "rename rollback inspection requested; recovery remains incomplete" + ); + } +} + /// Options shared by the normal and early-ack rename fanouts. Keeping the /// quorum and optional scanner lease map together avoids widening either /// fanout helper's argument list while preserving the fence semantics. @@ -3845,6 +4126,7 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> { write_quorum: usize, scanner_publication_lease_tokens: Option<&'a HashMap>, scanner_publication_commit_scope: Option, + rollback_receipt: Option, } impl<'a> RenameDataFenceOptions<'a> { @@ -3856,9 +4138,15 @@ impl<'a> RenameDataFenceOptions<'a> { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope: None, + rollback_receipt: None, } } + pub(in crate::set_disk) fn with_rollback_receipt(mut self, receipt: RenameRollbackReceipt) -> Self { + self.rollback_receipt = Some(receipt); + self + } + pub(in crate::set_disk) fn with_publication_scope( mut self, scanner_publication_commit_scope: Option, @@ -4224,6 +4512,7 @@ impl SetDisks { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope: _scanner_publication_commit_scope, + rollback_receipt, } = fence_options; if let Some(file_info) = disks .iter() @@ -4247,6 +4536,7 @@ impl SetDisks { let dst_object = Arc::new(dst_object.to_string()); let (commit_tx, commit_rx) = tokio::sync::oneshot::channel(); + let coordinator_failure_receipt = rollback_receipt.clone(); let tail_drain = tokio::spawn({ let fanout_src_bucket = src_bucket.clone(); let fanout_src_object = src_object.clone(); @@ -4269,7 +4559,8 @@ impl SetDisks { let file_info = file_info.clone(); let successful_rename_completion_rank = successful_rename_completion_rank.clone(); tasks.spawn(async move { - let result = std::panic::AssertUnwindSafe(async move { + let mut dispatch_state = RenameDispatchState::NotDispatched; + let result = std::panic::AssertUnwindSafe(async { #[allow(clippy::let_unit_value)] let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); @@ -4294,8 +4585,9 @@ impl SetDisks { } let disk_wait_started = rustfs_io_metrics::put_stage_timer(); - let result = disk - .rename_data_borrowed_with_fence( + dispatch_state = RenameDispatchState::MayHavePublished; + let observed = disk + .rename_data_borrowed_with_fence_observed( &src_bucket, &src_object, &file_info, @@ -4304,6 +4596,12 @@ impl SetDisks { scanner_publication_lease_token, ) .await; + let rejected_before_publication = observed.rejected_before_publication(); + let result = observed.result; + #[cfg(test)] + if result.is_ok() { + rollback_fault_injection::after_rename(&dst_object, i)?; + } if let Some(disk_wait_started) = disk_wait_started { let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0; rustfs_io_metrics::record_put_object_stage_duration( @@ -4325,11 +4623,14 @@ impl SetDisks { }; rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); } + if rejected_before_publication { + dispatch_state = RenameDispatchState::RejectedBeforePublication; + } result }) .catch_unwind() .await; - (i, result) + (i, dispatch_state, result) }); } @@ -4339,6 +4640,8 @@ impl SetDisks { let mut fanout_panic = 0usize; let mut results_seen = 0usize; let mut errs = vec![Some(DiskError::DiskNotFound); disk_count]; + // Missing task results cannot prove that a disk mutation never ran. + let mut dispatch_states = vec![RenameDispatchState::MayHavePublished; disk_count]; let mut disk_versions = vec![None; disk_count]; let mut data_dirs = vec![None; disk_count]; let mut cleanup_data_dirs = vec![None; disk_count]; @@ -4349,7 +4652,8 @@ impl SetDisks { while let Some(joined) = tasks.join_next().await { results_seen += 1; match joined { - Ok((idx, Ok(Ok(res)))) => { + Ok((idx, dispatch_state, Ok(Ok(res)))) => { + dispatch_states[idx] = dispatch_state; data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir); cleanup_data_dirs[idx] = res.cleanup_data_dir; disk_versions[idx] = res.sign; @@ -4357,10 +4661,12 @@ impl SetDisks { errs[idx] = None; success_count += 1; } - Ok((idx, Ok(Err(err)))) => { + Ok((idx, dispatch_state, Ok(Err(err)))) => { + dispatch_states[idx] = dispatch_state; errs[idx] = Some(err); } - Ok((idx, Err(_))) => { + Ok((idx, dispatch_state, Err(_))) => { + dispatch_states[idx] = dispatch_state; errs[idx] = Some(DiskError::Unexpected); fanout_panic += 1; } @@ -4390,6 +4696,8 @@ impl SetDisks { } } + #[cfg(test)] + rollback_fault_injection::after_fanout(&fanout_dst_object); if rustfs_io_metrics::put_stage_metrics_enabled() { let fanout_success = errs.iter().filter(|err| err.is_none()).count(); let fanout_error = errs.len().saturating_sub(fanout_success + fanout_panic); @@ -4405,36 +4713,16 @@ impl SetDisks { if !sent_commit { let ret_err = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum).unwrap_or(DiskError::Unexpected); - let mut rollbacks = Vec::new(); - let mut rollback_file_infos = file_infos; - for (i, err) in errs.iter().enumerate() { - if err.is_some() { - continue; - } - - if let Some(disk) = coordinator_disks[i].as_ref() { - let fi = std::mem::take(&mut rollback_file_infos[i]); - let old_data_dir = data_dirs[i]; - let disk = disk.clone(); - let dst_bucket = fanout_dst_bucket.clone(); - let dst_object = fanout_dst_object.clone(); - rollbacks.push(tokio::spawn(async move { - disk.delete_version( - &dst_bucket, - &dst_object, - fi, - false, - DeleteOptions { - undo_write: true, - old_data_dir, - ..Default::default() - }, - ) - .await - })); - } - } - let _ = join_all(rollbacks).await; + rollback_failed_rename( + &coordinator_disks, + file_infos, + &errs, + &dispatch_states, + &data_dirs, + (&fanout_dst_bucket, &fanout_dst_object), + rollback_receipt, + ) + .await; if let Some(commit_tx) = commit_tx.take() { let _ = commit_tx.send(Err(ret_err)); } @@ -4524,7 +4812,13 @@ impl SetDisks { }); let quorum_wait_started = rustfs_io_metrics::put_stage_timer(); - let commit = commit_rx.await.map_err(|_| DiskError::Unexpected)?; + let commit = match commit_rx.await { + Ok(commit) => commit, + Err(_) => { + record_indeterminate_rename(disks, (&dst_bucket, &dst_object), coordinator_failure_receipt).await; + return Err(DiskError::Unexpected); + } + }; rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, @@ -4582,6 +4876,7 @@ impl SetDisks { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope, + rollback_receipt, } = fence_options; if let Some(file_info) = disks .iter() @@ -4637,81 +4932,100 @@ impl SetDisks { let successful_rename_completion_rank = successful_rename_completion_rank.clone(); let publication_scope = scanner_publication_commit_scope.clone(); - std::panic::AssertUnwindSafe(async move { - // Test-only introspection guard: counts this operation as - // in-flight for the whole body. Compiles to `()` in production. - #[allow(clippy::let_unit_value)] - let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); + async move { + let mut dispatch_state = RenameDispatchState::NotDispatched; + let result = std::panic::AssertUnwindSafe(async { + // Test-only introspection guard: counts this operation as + // in-flight for the whole body. Compiles to `()` in production. + #[allow(clippy::let_unit_value)] + let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); - let Some(disk) = disk else { - return Err(DiskError::DiskNotFound); - }; - - let is_delete_marker = file_info.is_canonical_delete_marker(); - let mut local_file_info; - let file_info = if file_info.erasure.index == 0 { - local_file_info = file_info.clone(); - local_file_info.erasure.index = i + 1; - &local_file_info - } else { - file_info - }; - if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) { - return Err(DiskError::FileCorrupt); - } - - // Test-only awaitable pause point right before the disk rename. - // A no-op immediately-ready future in production. - Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; - - if let Some(err) = Self::rename_injected_error(&dst_object, i) { - return Err(err); - } - - if let Some(scope) = publication_scope.as_ref() - && !scope.can_commit() - { - let _ = scope.mark_indeterminate(); - return Err(DiskError::other("scanner publication commit scope deadline or cancellation reached")); - } - - let disk_wait_started = rustfs_io_metrics::put_stage_timer(); - let result = disk - .rename_data_borrowed_with_fence( - &src_bucket, - &src_object, - file_info, - &dst_bucket, - &dst_object, - scanner_publication_lease_token, - ) - .await; - if let Some(disk_wait_started) = disk_wait_started { - let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0; - rustfs_io_metrics::record_put_object_stage_duration( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, - duration_ms, - ); - let position = if result.is_ok() { - let rank = successful_rename_completion_rank - .as_ref() - .map(|rank| rank.fetch_add(1, Ordering::Relaxed) + 1) - .unwrap_or(1); - if rank <= write_quorum { - rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST - } else { - rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL - } - } else { - rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR + let Some(disk) = disk else { + return Err(DiskError::DiskNotFound); }; - rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); - } - result - }) - .catch_unwind() + + let is_delete_marker = file_info.is_canonical_delete_marker(); + let mut local_file_info; + let file_info = if file_info.erasure.index == 0 { + local_file_info = file_info.clone(); + local_file_info.erasure.index = i + 1; + &local_file_info + } else { + file_info + }; + if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) { + return Err(DiskError::FileCorrupt); + } + + // Test-only awaitable pause point right before the disk rename. + // A no-op immediately-ready future in production. + Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; + + if let Some(err) = Self::rename_injected_error(&dst_object, i) { + return Err(err); + } + + if let Some(scope) = publication_scope.as_ref() + && !scope.can_commit() + { + let _ = scope.mark_indeterminate(); + return Err(DiskError::other( + "scanner publication commit scope deadline or cancellation reached", + )); + } + + let disk_wait_started = rustfs_io_metrics::put_stage_timer(); + dispatch_state = RenameDispatchState::MayHavePublished; + let observed = disk + .rename_data_borrowed_with_fence_observed( + &src_bucket, + &src_object, + file_info, + &dst_bucket, + &dst_object, + scanner_publication_lease_token, + ) + .await; + let rejected_before_publication = observed.rejected_before_publication(); + let result = observed.result; + #[cfg(test)] + if result.is_ok() { + rollback_fault_injection::after_rename(&dst_object, i)?; + } + if let Some(disk_wait_started) = disk_wait_started { + let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, + duration_ms, + ); + let position = if result.is_ok() { + let rank = successful_rename_completion_rank + .as_ref() + .map(|rank| rank.fetch_add(1, Ordering::Relaxed) + 1) + .unwrap_or(1); + if rank <= write_quorum { + rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST + } else { + rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL + } + } else { + rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR + }; + rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); + } + if rejected_before_publication { + dispatch_state = RenameDispatchState::RejectedBeforePublication; + } + result + }) + .catch_unwind() + .await; + (dispatch_state, result) + } }); let results = join_all(futures).await; + #[cfg(test)] + rollback_fault_injection::after_fanout(&fanout_dst_object); (results, fanout_file_infos) }); @@ -4726,12 +5040,18 @@ impl SetDisks { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, ); - let (results, mut file_infos) = fanout_result.map_err(|_| DiskError::Unexpected)?; + let (results, mut file_infos) = match fanout_result { + Ok(result) => result, + Err(_) => { + record_indeterminate_rename(disks, (&dst_bucket, &dst_object), rollback_receipt).await; + return Err(DiskError::Unexpected); + } + }; if rustfs_io_metrics::put_stage_metrics_enabled() { let mut fanout_success = 0; let mut fanout_error = 0; let mut fanout_panic = 0; - for result in &results { + for (_, result) in &results { match result { Ok(Ok(_)) => fanout_success += 1, Ok(Err(_)) => fanout_error += 1, @@ -4747,7 +5067,9 @@ impl SetDisks { ); } - for (idx, result) in results.iter().enumerate() { + let mut dispatch_states = Vec::with_capacity(results.len()); + for (idx, (dispatch_state, result)) in results.iter().enumerate() { + dispatch_states.push(*dispatch_state); match result { Ok(Ok(res)) => { data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir); @@ -4793,36 +5115,7 @@ impl SetDisks { ); } - let mut futures = Vec::with_capacity(disks.len()); if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - for (i, err) in errs.iter().enumerate() { - if err.is_some() { - continue; - } - - if let Some(disk) = disks[i].as_ref() { - let fi = std::mem::take(&mut file_infos[i]); - let old_data_dir = data_dirs[i]; - let disk = disk.clone(); - let dst_bucket = dst_bucket.clone(); - let dst_object = dst_object.clone(); - futures.push(tokio::spawn(async move { - disk.delete_version( - &dst_bucket, - &dst_object, - fi, - false, - DeleteOptions { - undo_write: true, - old_data_dir, - ..Default::default() - }, - ) - .await - })); - } - } - if issue3031_diag_enabled() { warn!( target: "rustfs_ecstore::set_disk", @@ -4838,23 +5131,16 @@ impl SetDisks { ); } - let undo_results = join_all(futures).await; - let undo_error_count = undo_results - .iter() - .filter(|result| match result { - Err(_) | Ok(Err(_)) => true, - Ok(Ok(_)) => false, - }) - .count(); - if undo_error_count > 0 { - warn!( - target: "rustfs_ecstore::set_disk", - dst_bucket = %dst_bucket, - dst_object = %dst_object, - undo_error_count, - "rename_data quorum rollback reported errors" - ); - } + rollback_failed_rename( + disks, + file_infos, + &errs, + &dispatch_states, + &data_dirs, + (&dst_bucket, &dst_object), + rollback_receipt, + ) + .await; return Err(ret_err); } @@ -6800,6 +7086,86 @@ pub(in crate::set_disk) mod rename_fault_injection { } } +#[cfg(test)] +pub(in crate::set_disk) mod rollback_fault_injection { + use super::DiskError; + use std::{ + collections::HashMap, + sync::{Mutex, OnceLock}, + }; + + #[derive(Clone, Copy, Debug)] + pub(in crate::set_disk) enum Fault { + Io, + Panic, + IoAfterRename, + VolumeNotFoundAfterRename, + PanicAfterRename, + CoordinatorPanic, + } + + fn registry() -> &'static Mutex> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(Mutex::default) + } + + pub(in crate::set_disk) struct Guard(String); + + impl Drop for Guard { + fn drop(&mut self) { + if let Ok(mut registry) = registry().lock() { + registry.remove(&self.0); + } + } + } + + pub(in crate::set_disk) fn arm(object: &str, disk_index: usize, fault: Fault) -> Guard { + registry() + .lock() + .expect("rollback registry should not poison") + .insert(object.to_string(), (disk_index, fault)); + Guard(object.to_string()) + } + + pub(super) fn before_undo(object: &str, disk_index: usize) -> Result<(), DiskError> { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + match fault { + Some((target, Fault::Io)) if target == disk_index => Err(DiskError::FaultyDisk), + Some((target, Fault::Panic)) if target == disk_index => panic!("injected rollback panic"), + _ => Ok(()), + } + } + + pub(super) fn after_rename(object: &str, disk_index: usize) -> Result<(), DiskError> { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + match fault { + Some((target, Fault::IoAfterRename)) if target == disk_index => Err(DiskError::FaultyDisk), + Some((target, Fault::VolumeNotFoundAfterRename)) if target == disk_index => Err(DiskError::VolumeNotFound), + Some((target, Fault::PanicAfterRename)) if target == disk_index => panic!("injected panic after rename mutation"), + _ => Ok(()), + } + } + + pub(super) fn after_fanout(object: &str) { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + if matches!(fault, Some((_, Fault::CoordinatorPanic))) { + panic!("injected rename coordinator panic"); + } + } +} + /// Test-only per-disk call counters for the metadata fan-out (backlog#1325, /// serving the RPC-count assertions of #1309 / #1314 / #1315). /// @@ -6911,6 +7277,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase { pub const RENAME: &str = "rename"; /// The per-disk old-data-dir cleanup phase of the commit fan-out. pub const CLEANUP: &str = "cleanup"; + pub const ROLLBACK: &str = "rollback"; /// The per-disk `read_version` phase of metadata read fan-out. #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub const READ_VERSION: &str = "read_version"; @@ -9668,6 +10035,7 @@ mod tests { file_info.mod_time = Some(OffsetDateTime::now_utc()); file_info.erasure.index = idx + 1; file_info.data = Some(Bytes::from_static(b"inline-body")); + file_info.set_inline_data(); file_info.metadata.insert("etag".to_string(), etag.to_string()); file_info }) @@ -10422,6 +10790,453 @@ mod tests { .await; } + #[tokio::test] + async fn rename_rollback_incomplete_inspection_rejection_is_not_recovery() { + fn reject_inspection(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture { + assert_eq!(request.bucket, "rollback-inspection"); + assert_eq!(request.object_prefix.as_deref(), Some("object")); + assert_eq!(request.dry_run, Some(true), "failed minority must never become a mutating heal source"); + assert_eq!(request.remove_corrupted, Some(false)); + assert_eq!(request.recreate_missing, Some(false)); + assert_eq!(request.recursive, Some(false)); + Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Full) }) + } + let result = inspect_incomplete_rename_rollback(&[], "rollback-inspection", "object", reject_inspection).await; + assert!(matches!(result, ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Full))); + } + + #[tokio::test] + async fn rename_rollback_incomplete_cancelled_task_is_not_success() { + let task = tokio::spawn(std::future::pending::>()); + task.abort(); + assert_eq!(rename_rollback_task_outcome(task.await), RenameRollbackOutcome::Cancelled); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_matches_early_ack_and_full_wait_after_reopen() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + const DISKS: usize = 4; + const WRITE_QUORUM: usize = 3; + for overwrite in [false, true] { + for success_count in [0, WRITE_QUORUM - 1, WRITE_QUORUM] { + for fault in [ + None, + Some(rollback_fault_injection::Fault::Io), + Some(rollback_fault_injection::Fault::Panic), + ] { + let mut previous = None; + for early_ack in [false, true] { + let bucket = "rename-rollback-matrix"; + let object = format!("object-{overwrite}-{success_count}-{fault:?}-{early_ack}"); + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + if overwrite { + let mut old = metadata_test_fileinfo(&object); + old.mod_time = Some(OffsetDateTime::now_utc()); + old.data = Some(Bytes::from_static(b"old-inline-body")); + old.set_inline_data(); + old.metadata.insert("etag".to_string(), "old-etag".to_string()); + for disk in disks.iter().flatten() { + disk.write_metadata(bucket, bucket, &object, old.clone()) + .await + .expect("old version must be staged"); + } + } + let _rename_fault = + rename_fault_injection::fail_rename_on(&object, &(success_count..DISKS).collect::>()); + let _undo_fault = fault.map(|fault| rollback_fault_injection::arm(&object, 0, fault)); + let receipt = RenameRollbackReceipt::default(); + let result = SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(&object, DISKS, "new-etag"), + (bucket, &object), + early_ack, + RenameDataFenceOptions::new(WRITE_QUORUM, None).with_rollback_receipt(receipt.clone()), + ) + .await; + let actual_error = match result { + Ok(commit) => { + assert_eq!(success_count, WRITE_QUORUM); + if let Some(tail) = commit.tail_drain { + tail.await + .expect("committed tail should join") + .expect("committed tail should converge"); + } + assert!( + receipt.0.get().is_none(), + "a quorum commit must not enter rollback even when undo faults are armed" + ); + None + } + Err(err) => { + assert!(success_count < WRITE_QUORUM); + let report = receipt + .0 + .get() + .expect("both failure paths must publish per-disk rollback evidence"); + assert_eq!(report.disks.len(), DISKS); + for (idx, outcome) in report.disks.iter().enumerate() { + assert_eq!(outcome.disk_index, idx); + let expected = if idx >= success_count { + RenameRollbackOutcome::NotAttempted(DiskError::other( + "injected rename failure (test-only)", + )) + } else if idx == 0 { + match fault { + Some(rollback_fault_injection::Fault::Io) => { + RenameRollbackOutcome::Failed(DiskError::FaultyDisk) + } + Some(rollback_fault_injection::Fault::Panic) => RenameRollbackOutcome::Panicked, + None => RenameRollbackOutcome::Succeeded, + Some(_) => unreachable!("matrix only injects undo faults"), + } + } else { + RenameRollbackOutcome::Succeeded + }; + assert_eq!(outcome.outcome, expected); + if overwrite && idx < success_count { + let backup = dirs[idx] + .path() + .join(bucket) + .join(&object) + .join(outcome.rollback_dir.expect("overwrite needs rollback dir").to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + assert_eq!( + backup.exists(), + outcome.outcome.needs_recovery(), + "failed undo must retain its only old-version backup" + ); + } + } + assert_eq!(receipt.is_incomplete(), success_count > 0 && fault.is_some()); + Some(err) + } + }; + if let Some(expected_error) = previous.as_ref() { + assert_eq!( + &actual_error, expected_error, + "early ACK must preserve the original full-wait quorum error" + ); + } + previous = Some(actual_error); + for (idx, dir) in dirs.iter().enumerate() { + let reopened = reopen_local_disk(dir).await; + let read = reopened + .read_version( + "", + bucket, + &object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await; + let keeps_new = + idx < success_count && (success_count == WRITE_QUORUM || (idx == 0 && fault.is_some())); + if keeps_new || overwrite { + let stored = read.expect("old or committed version must survive reopen"); + assert_eq!( + stored.metadata.get("etag").map(String::as_str), + Some(if keeps_new { "new-etag" } else { "old-etag" }) + ); + assert_eq!( + stored.data.as_deref(), + Some(if keeps_new { + b"inline-body".as_slice() + } else { + b"old-inline-body".as_slice() + }), + "object={object}, disk={idx}, keeps_new={keeps_new}" + ); + } else { + assert!( + matches!(read, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)), + "fresh rollback must not expose data: {read:?}" + ); + } + } + } + } + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_data_early_ack_post_mutation_tail_error_never_rolls_back_commit() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for fault in [ + rollback_fault_injection::Fault::IoAfterRename, + rollback_fault_injection::Fault::VolumeNotFoundAfterRename, + rollback_fault_injection::Fault::PanicAfterRename, + ] { + let bucket = "rename-tail-unknown"; + let object = format!("tail-{fault:?}"); + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let receipt = RenameRollbackReceipt::default(); + let _fault = rollback_fault_injection::arm(&object, 0, fault); + let barrier = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier_phase::RENAME); + let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(&object, 4, "new-etag"), + (bucket, &object), + true, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + )); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + tokio::select! { + () = barrier.wait_until_paused() => {} + _ = rename.as_mut() => panic!("tail barrier must precede quorum ACK"), + } + }) + .await + .expect("tail reaches the barrier"); + let commit = tokio::time::timeout(BARRIER_PAUSE_GUARD, rename) + .await + .expect("quorum must ACK before tail release") + .expect("three disks commit"); + barrier.release(); + let tail = commit + .tail_drain + .expect("early ACK owns a tail") + .await + .expect("tail coordinator joins") + .expect("committed tail reports convergence"); + assert_eq!(tail.convergence, RenameConvergence::PartialCommit); + assert!(receipt.0.get().is_none(), "post-ACK errors must never start rollback"); + for dir in &dirs { + let reopened = reopen_local_disk(dir).await; + let stored = reopened + .read_version( + "", + bucket, + &object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("all actual writes survive despite a lost tail acknowledgement"); + assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice())); + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for fault in [ + rollback_fault_injection::Fault::Io, + rollback_fault_injection::Fault::IoAfterRename, + rollback_fault_injection::Fault::VolumeNotFoundAfterRename, + rollback_fault_injection::Fault::PanicAfterRename, + rollback_fault_injection::Fault::CoordinatorPanic, + ] { + for early_ack in [false, true] { + let bucket = "rollback-data-dirs"; + let object = format!("object-{early_ack}-{fault:?}"); + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let old_data_dir = Uuid::new_v4(); + let new_data_dir = Uuid::new_v4(); + let mut old = metadata_test_fileinfo(&object); + old.data_dir = Some(old_data_dir); + old.mod_time = Some(OffsetDateTime::now_utc()); + let mut infos = Vec::new(); + for (idx, disk) in disks.iter().enumerate() { + let disk = disk.as_ref().expect("fixture disk should be present"); + disk.write_metadata(bucket, bucket, &object, old.clone()) + .await + .expect("old metadata should be staged"); + let old_dir = dirs[idx].path().join(bucket).join(&object).join(old_data_dir.to_string()); + tokio::fs::create_dir_all(&old_dir) + .await + .expect("old data directory should exist"); + tokio::fs::write(old_dir.join("part.1"), b"old-data") + .await + .expect("old shard should exist"); + let source = dirs[idx] + .path() + .join(RUSTFS_META_TMP_BUCKET) + .join("source") + .join(new_data_dir.to_string()); + tokio::fs::create_dir_all(&source) + .await + .expect("new data directory should be staged"); + tokio::fs::write(source.join("part.1"), b"new-data") + .await + .expect("new shard should be staged"); + let mut fi = metadata_test_fileinfo(&object); + fi.data_dir = Some(new_data_dir); + fi.erasure.index = idx + 1; + fi.mod_time = Some(OffsetDateTime::now_utc()); + infos.push(fi); + } + let _rename_fault = rename_fault_injection::fail_rename_on(&object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(&object, 0, fault); + let receipt = RenameRollbackReceipt::default(); + assert!( + SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + infos, + (bucket, &object), + early_ack, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + ) + .await + .is_err() + ); + assert!(receipt.is_incomplete()); + if !matches!(fault, rollback_fault_injection::Fault::Io) { + assert!( + matches!( + receipt.0.get().expect("indeterminate report").disks[0].outcome, + RenameRollbackOutcome::Indeterminate(_) + ), + "post-mutation failure must not be classified as unattempted" + ); + } + let coordinator_failed = matches!(fault, rollback_fault_injection::Fault::CoordinatorPanic); + for (idx, dir) in dirs.iter().enumerate() { + let root = dir.path().join(bucket).join(&object); + assert_eq!( + tokio::fs::read(root.join(old_data_dir.to_string()).join("part.1")) + .await + .expect("old data must survive failed overwrite"), + b"old-data" + ); + let backup = root.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); + assert_eq!( + backup.exists(), + idx == 0 || (coordinator_failed && idx == 1), + "unknown mutations must retain the old-version backup" + ); + if idx >= 2 { + let staged = dir + .path() + .join(RUSTFS_META_TMP_BUCKET) + .join("source") + .join(new_data_dir.to_string()) + .join("part.1"); + assert_eq!( + tokio::fs::read(staged) + .await + .expect("failed-write staging must remain available"), + b"new-data" + ); + } + let reopened = reopen_local_disk(dir).await; + let stored = reopened + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("metadata should survive reopen"); + assert_eq!( + stored.data_dir, + Some(if idx == 0 || (coordinator_failed && idx == 1) { + new_data_dir + } else { + old_data_dir + }) + ); + } + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() { + for cancel_caller in [false, true] { + let bucket = "rename-rollback-barrier"; + let object = if cancel_caller { + "rollback-barrier-cancelled" + } else { + "rollback-barrier-object" + }; + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let mut old = metadata_test_fileinfo(object); + old.mod_time = Some(OffsetDateTime::now_utc()); + old.data = Some(Bytes::from_static(b"old-inline-body")); + old.set_inline_data(); + old.metadata.insert("etag".to_string(), "old-etag".to_string()); + for disk in disks.iter().flatten() { + disk.write_metadata(bucket, bucket, object, old.clone()) + .await + .expect("old metadata should be staged"); + } + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); + let receipt = RenameRollbackReceipt::default(); + let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(object, 4, "new-etag"), + (bucket, object), + false, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + )); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + tokio::select! { + () = barrier.wait_until_paused() => {} + _ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"), + } + }) + .await + .expect("undo must reach its disk barrier"); + assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success"); + if cancel_caller { + drop(rename); + barrier.release(); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + while receipt.0.get().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled caller must not cancel rollback accounting"); + } else { + barrier.release(); + assert!(rename.await.is_err()); + } + assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt"); + for dir in dirs.iter().skip(1) { + let reopened = reopen_local_disk(dir).await; + let restored = reopened + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("old version must remain readable after caller cancellation"); + assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice())); + } + } + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn rename_data_early_ack_strict_quorum_failure_rolls_back_fresh_after_reopen() { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 5a220e432..c76daf707 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -299,11 +299,11 @@ use crate::error::is_err_invalid_upload_id; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; use crate::object_api::{ NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, - SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, + SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion, }; use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; -use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal}; +use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal}; #[cfg(test)] use crate::storage_api_contracts::namespace::NamespaceLocking; #[cfg(test)] @@ -3548,6 +3548,7 @@ impl SetDisks { (None, None, None) }; let mut tmp_cleanup_owned = false; + let rollback_receipt = RenameRollbackReceipt::default(); let operation = async { let erasure = Arc::new(erasure_from_file_info(&fi, false)?); @@ -4256,6 +4257,7 @@ impl SetDisks { let commit_bucket = bucket.to_owned(); let commit_object = object.to_owned(); let commit_tmp_dir = tmp_dir.clone(); + let commit_rollback_receipt = rollback_receipt.clone(); let commit_object_lock_guard = object_lock_guard.take(); let commit_decommission_object_lock_guard = decommission_object_lock_guard.take(); let commit_publication_guard = publication_commit_guard.take(); @@ -4266,13 +4268,17 @@ impl SetDisks { // complete rename fan-out drains. Keep this path synchronous so // its terminal state is known before the coordinator releases // remote leases. - let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation()) - && (commit_object_lock_guard.is_some() - || commit_decommission_object_lock_guard.is_some() - || commit_publication_guard.is_some()) + let commit_owns_namespace_guard = commit_object_lock_guard.is_some() + || commit_decommission_object_lock_guard.is_some() + || commit_publication_guard.is_some(); + let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum + && !(opts.data_movement && opts.has_decommission_capacity_reservation()) + && commit_owns_namespace_guard && commit_scanner_publication_scope.is_none(); + // Full-tail callers also transfer owned guards to the coordinator: + // cancelling their ACK waiter must not cancel an in-flight rename. let detach_commit_owner = commit_scanner_publication_scope.is_some() - || commit_allows_early_ack + || commit_owns_namespace_guard || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence; let commit_write_path_label = write_path.metric_label(); @@ -4452,7 +4458,8 @@ impl SetDisks { write_quorum, commit_scanner_publication_lease_tokens.as_ref(), ) - .with_publication_scope(commit_scanner_publication_scope.clone()), + .with_publication_scope(commit_scanner_publication_scope.clone()) + .with_rollback_receipt(commit_rollback_receipt.clone()), ) .await; if let Some(scope) = commit_scanner_publication_scope.as_ref() { @@ -4585,6 +4592,11 @@ impl SetDisks { let rename_commit = match rename_result { Ok(commit) => commit, Err(err) => { + if commit_rollback_receipt.is_incomplete() { + // Incomplete undo retains the staging source and + // rollback backup for recovery; cleanup is unsafe. + return Err(err.into()); + } if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { @@ -4617,9 +4629,8 @@ impl SetDisks { request.object_version_id = committed_version_id .or_else(|| commit_version_suspended.then(Uuid::nil)) .map(|version_id| version_id.to_string()); - tokio::spawn(async move { - let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await; - }); + let heal_set = commit_set.clone(); + tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await }); } let rename_stage_elapsed = rename_stage_start.elapsed(); @@ -4885,7 +4896,7 @@ impl SetDisks { ); } }); - } else { + } else if !rollback_receipt.is_incomplete() { // Failure path (quorum loss / rollback): keep the cleanup inline so // a failed PUT never returns while its tmp shards are still on disk // (state-residue hardening tracked by backlog#864 / backlog#898). @@ -17494,27 +17505,69 @@ mod put_object_tmp_cleanup_tests { } #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] async fn put_object_failure_cleans_tmp_workspace_inline() { - let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await; + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] { + let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "tmp-clean-missing-bucket"; + let object = "orphan-object"; + let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace); + let writer = Arc::clone(&set_disks); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("missing-bucket PUT must stage before rename"); + let staged = non_trash_tmp_entries(&temp_dirs).await; + assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection"); + for workspace in staged { + let mut entries = tokio::fs::read_dir(&workspace) + .await + .expect("staged workspace should be readable"); + let mut shards = 0; + while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") { + if entry.file_type().await.expect("staged entry type").is_dir() { + let part = tokio::fs::metadata(entry.path().join("part.1")) + .await + .expect("staging must contain an actual erasure shard"); + assert!(part.len() > 0, "the shard must be written before the missing-bucket failure"); + shards += 1; + } + } + assert_eq!(shards, 1); + } + assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists())); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("missing-bucket PUT must finish") + .expect("PUT task should join") + .expect_err("put_object into a missing bucket volume must fail"); + assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}"); - // The bucket volume is never created, so the shards are written into - // the tmp workspace and the commit fails at rename_data with a quorum - // error — exercising the failure-path cleanup. - let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]); - let err = set_disks - .put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default()) - .await - .expect_err("put_object into a missing bucket volume must fail"); - - // No polling: the failure path must clean the tmp workspace inline, - // before put_object returns (backlog#864 / backlog#898 hardening). - let leftovers = non_trash_tmp_entries(&temp_dirs).await; - assert!( - leftovers.is_empty(), - "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}" - ); - - drop(temp_dirs); + // No polling: known pre-publication rejection must clean staging + // inline, before PUT returns (backlog#864 / backlog#898). + let leftovers = non_trash_tmp_entries(&temp_dirs).await; + assert!( + leftovers.is_empty(), + "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}" + ); + } + }) + .await; } #[tokio::test] @@ -18157,6 +18210,354 @@ mod put_object_tmp_cleanup_tests { .await; } + async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) { + for disk in disks { + disk.make_volume(bucket) + .await + .expect("completion test bucket should be created"); + } + } + + /// Observe the actual metadata quorum while the remaining rename is parked. + /// A completed task count alone can race tasks that have not started yet. + async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let mut committed = 0; + for disk in disks { + match disk.read_version("", bucket, object, "", &ReadOptions::default()).await { + Ok(_) => committed += 1, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {} + Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"), + } + } + if committed == 3 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("three disks must publish metadata while the fourth rename remains paused"); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for size in [4096, 1024 * 1024] { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-full-tail-cas"; + let object = "full-tail-cas-object"; + make_completion_test_bucket(&disks, bucket).await; + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; size]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("full-tail PUT must reach the rename barrier"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum"); + let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object)); + assert!( + futures::poll!(lock_probe.as_mut()).is_pending(), + "the owned namespace guard must remain held" + ); + barrier.release(); + let written = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("full-tail PUT should finish after release") + .expect("full-tail PUT task should join") + .expect("full-tail PUT must commit"); + assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task"); + drop( + tokio::time::timeout(Duration::from_secs(5), lock_probe) + .await + .expect("same-key lock should be available on return") + .expect("same-key lock probe should succeed"), + ); + for disk in &disks { + disk.read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("successful full-tail PUT must publish on every healthy disk"); + } + drop(barrier); + let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec()); + set.put_object( + bucket, + object, + &mut replacement, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_match: written.etag, + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .expect("immediate same-key CAS must acquire the namespace guard"); + let mut read = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("CAS successor must be immediately readable"); + let mut body = Vec::new(); + read.stream.read_to_end(&mut body).await.expect("successor body must drain"); + assert_eq!(body, b"cas successor"); + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-full-tail-heal"; + let object = "full-tail-heal-object"; + make_completion_test_bucket(&disks, bucket).await; + let mut heals = set.capture_test_rename_tail_heals(); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let _fault = rename_fault_injection::fail_rename_on(object, &[0]); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("failed tail must first reach the rename barrier"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + assert!(!put.is_finished(), "committed quorum must still wait for the failing tail"); + barrier.release(); + tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("failed tail should drain") + .expect("PUT task should join") + .expect("a minority tail error must not negate committed quorum"); + assert_eq!(tasks.running(), 0); + let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv()) + .await + .expect("failed tail must schedule heal") + .expect("heal capture must remain connected"); + assert_eq!(heal.bucket, bucket); + assert_eq!(heal.object_prefix.as_deref(), Some(object)); + let info = set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("committed object must remain readable despite the failed tail"); + assert_eq!(info.size, TEST_OBJECT_SIZE as i64); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_rejects_quorum_minus_one() { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-full-tail-no-quorum"; + let object = "full-tail-no-quorum-object"; + make_completion_test_bucket(&disks, bucket).await; + let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]); + let tasks = rename_fanout_barrier::observe_tasks(object); + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + let err = set + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect_err("draining two successful disks cannot satisfy write quorum three"); + assert!( + matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), + "original quorum error expected: {err}" + ); + assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return"); + assert!( + set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(), + "failed fresh write must not become visible" + ); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() { + use crate::set_disk::core::io_primitives::rollback_fault_injection; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] { + for fault in [ + rollback_fault_injection::Fault::Io, + rollback_fault_injection::Fault::VolumeNotFoundAfterRename, + ] { + let (dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-incomplete-undo"; + let object = "incomplete-undo-object"; + make_completion_test_bucket(&disks, bucket).await; + let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]); + set.put_object( + bucket, + object, + &mut old_reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("old generation should be completely committed"); + wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await; + let old = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("old metadata must be readable"); + let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory"); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(object, 0, fault); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("overwrite must enter the actual rename fan-out before failure injection"); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("incomplete undo must return without hanging") + .expect("PUT task should join") + .expect_err("two renamed disks cannot satisfy write quorum three"); + assert!( + matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), + "original quorum error expected: {err}" + ); + assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return"); + let leftovers = non_trash_tmp_entries(&dirs).await; + assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery"); + let backups = dirs + .iter() + .filter(|dir| { + dir.path() + .join(bucket) + .join(object) + .join(old_data_dir.to_string()) + .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP) + .exists() + }) + .count(); + assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup"); + // The remaining three disks still serve the old generation; + // the failed minority must never become an acknowledged write. + let mut read = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("old generation must remain readable after incomplete rollback"); + let mut body = Vec::new(); + read.stream + .read_to_end(&mut body) + .await + .expect("old generation should stream"); + assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]); + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_owned_commit_survives_waiter_cancellation() { + let (dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = RUSTFS_META_BUCKET; + let object = "full-tail-cancelled-receipt"; + // Internal config writes do not own a bucket lifecycle guard. The object + // guard alone must keep the full-tail coordinator alive after cancellation. + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("cancelled receipt must first reach the rename barrier"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + put.abort(); + assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled()); + let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object)); + assert!( + futures::poll!(lock_probe.as_mut()).is_pending(), + "owned coordinator must retain the namespace guard after waiter cancellation" + ); + barrier.release(); + drop( + tokio::time::timeout(Duration::from_secs(30), lock_probe) + .await + .expect("cancelled coordinator must eventually release its guard") + .expect("post-commit lock probe should succeed"), + ); + assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task"); + for disk in &disks { + disk.read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("caller cancellation must not interrupt committed receipt materialization"); + } + wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await; + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn no_lock_put_waits_for_rename_tail_under_outer_guard() { @@ -18184,6 +18585,7 @@ mod put_object_tmp_cleanup_tests { &mut reader, &ObjectOptions { no_lock: true, + write_completion: WriteCompletion::TailDrained, ..Default::default() }, ) @@ -18209,7 +18611,18 @@ mod put_object_tmp_cleanup_tests { put.await .expect("no-lock PUT task should join") .expect("no-lock PUT should commit after the rename tail releases"); + let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object)); + assert!( + futures::poll!(lock_probe.as_mut()).is_pending(), + "full-tail PUT must not release the caller's outer guard" + ); drop(outer_guard); + drop( + tokio::time::timeout(Duration::from_secs(5), lock_probe) + .await + .expect("outer owner releasing its guard should unblock the probe") + .expect("post-outer-guard probe should succeed"), + ); }) .await; } diff --git a/crates/ecstore/src/set_disk/transition_matrix_tests.rs b/crates/ecstore/src/set_disk/transition_matrix_tests.rs index 8924b2a4a..9b5912ce7 100644 --- a/crates/ecstore/src/set_disk/transition_matrix_tests.rs +++ b/crates/ecstore/src/set_disk/transition_matrix_tests.rs @@ -18,6 +18,7 @@ use super::{ }; use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time}; use crate::ecstore_validation_blackbox::make_local_set_disks; +use crate::object_api::WriteCompletion; use crate::services::tier::test_util::register_mock_tier; use crate::storage_api_contracts::bucket::BucketOperations; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; @@ -72,7 +73,7 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() { object, &mut reader, &ObjectOptions { - no_lock: true, + write_completion: WriteCompletion::TailDrained, ..Default::default() }, ) @@ -185,7 +186,7 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot object, &mut reader, &ObjectOptions { - no_lock: true, + write_completion: WriteCompletion::TailDrained, ..Default::default() }, ) diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 463f50a82..d0b426b68 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -2979,6 +2979,33 @@ mod tests { #[cfg(feature = "test-util")] const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object"; + fn decommission_retry_fault_hook( + bucket: &str, + object: &str, + faults: Arc, + ) -> crate::core::pools::DecommissionTestFaultDecision { + let target_bucket = bucket.to_string(); + let target_object = object.to_string(); + Arc::new(move |stage, bucket, object, _attempt, succeeded| { + if !succeeded + || stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT + || bucket != target_bucket + || object != target_object + { + return false; + } + + // Entry retries reset the local attempt; real copy errors can skip + // successful attempts. Only injected faults spend this global budget. + faults + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| { + (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1)) + .then_some(faults.saturating_add(1)) + }) + .is_ok() + }) + } + async fn seed_decommission_source( store: &Arc, bucket: &str, @@ -5120,6 +5147,33 @@ mod tests { shutdown.cancel(); } + #[test] + fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() { + for attempts in [[1, 2, 3], [1, 1, 2], [1, 3, 3]] { + let faults = Arc::new(AtomicUsize::new(0)); + let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults)); + + for (stage, bucket, object, succeeded) in [ + ("other-stage", "bucket", "object", true), + (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true), + (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true), + (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false), + ] { + assert!(!hook(stage, bucket, object, 1, succeeded)); + } + assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults"); + + for (index, attempt) in attempts.into_iter().enumerate() { + assert_eq!( + hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, true), + index < 2, + "attempts={attempts:?}, index={index}" + ); + } + assert_eq!(faults.load(Ordering::SeqCst), 2, "attempts={attempts:?}"); + } + } + #[test] #[serial_test::serial(storage_class_env)] fn decommission_entry_retries_source_changed_without_canceling_other_bucket() { @@ -5214,31 +5268,8 @@ mod tests { )); let ordinary_faults = Arc::new(AtomicUsize::new(0)); - let ordinary_faults_for_hook = Arc::clone(&ordinary_faults); - let fault_bucket = other_bucket.clone(); - let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new( - move |stage, bucket, object, attempt, succeeded| { - let candidate = succeeded - && stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT - && bucket == fault_bucket.as_str() - && object == other_object; - if !candidate { - return false; - } - - // Keep the fault budget global across any - // entry-level re-list; its inner attempt counter - // restarts after SourceChanged. - ordinary_faults_for_hook - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| { - let next_fault = faults.saturating_add(1); - (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1) - && attempt == next_fault) - .then_some(next_fault) - }) - .is_ok() - }, - )); + let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults)); + let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook); let rx = CancellationToken::new(); let source_changed_exhaustions = Arc::new(AtomicUsize::new(0)); @@ -8045,10 +8076,15 @@ mod tests { ); assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok()); - com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone()) + let full_tail = ObjectOptions { + max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }; + com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail) .await .expect("second page receipt should restore"); - com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec()) + com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail) .await .expect("second page receipt should corrupt deterministically"); let corrupt = store diff --git a/docs/testing/ecstore-validation-suite-design.md b/docs/testing/ecstore-validation-suite-design.md index e9bec1fe4..de717d1b9 100644 --- a/docs/testing/ecstore-validation-suite-design.md +++ b/docs/testing/ecstore-validation-suite-design.md @@ -54,6 +54,22 @@ Fail-closed invariants every row enforces: Fault injection is explicit and deterministic: local disk mocks for unit tests, process-level disk manipulation (`crates/e2e_test/src/chaos.rs`) for e2e tests. Property tests replay a fixed seed for payload, range, and missing-shard selection. +### PUT completion fixtures + +`ObjectOptions::default()` uses `WriteCompletion::Quorum`: a namespace-lock-owning PUT may acknowledge write quorum while its rename tail retains the lock. A fixture that immediately inspects every disk or primes a metadata generation must set `write_completion: WriteCompletion::TailDrained` and keep normal locking. TailDrained waits for the existing rename fan-out; it does not require every disk to succeed or change fsync policy. Codec-only `no_lock` fixtures do not cover namespace locking. + +The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` and `observe_tasks(object)`. Wait for the barrier with a deadline, observe actual metadata quorum with `wait_for_paused_tail_metadata_quorum`, then release or cancel. The metadata check distinguishes a real quorum from disk tasks that have not started. Assert zero remaining rename tasks after the owned coordinator releases its lock; cancellation tests also wait for staging cleanup. + +| Fixture | Completion boundary | +|---|---| +| `early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes` | Default PUT returns before the parked tail; a second writer remains blocked. | +| `tail_drained_put_*` | Explicit full-tail PUT retains its guard, preserves quorum success with a failed minority, rejects quorum-minus-one, and survives ACK waiter cancellation. | +| `transition_and_restore_reclaim_prior_metadata_generations` | Both source fixtures use TailDrained before cache priming, with normal namespace locks. | +| `object_transaction_fencing_persists_epoch_on_multipart_commit` | Multipart completion already always drains rename before inspecting all per-disk transaction UUIDs. | +| `decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page`, `dispatch_completion_cas_is_bounded_and_reaches_the_tail` | Durable receipt, journal, and manifest writers choose TailDrained; the pagination fixture also drains deliberate receipt replacement writes. | + +Select these checks with `cargo nextest list -p rustfs-ecstore --features test-util -E 'test(tail_drained_put) | test(early_ack_tail_drain) | test(no_lock_put_waits_for_rename_tail) | test(object_transaction_fencing_persists_epoch_on_multipart_commit) | test(transition_and_restore_reclaim) | test(decommission_durable_ilm_receipt_pagination) | test(dispatch_completion_cas)'`, then run the same expression under the default and CI profiles without retries. Remaining crash, reopen, rollback, and lock-loss schedules use the existing domain tests; this completion fixture is not a replacement for those checks. + ### Coverage gate `full` and `destructive` run `cargo llvm-cov -p rustfs-ecstore --lib` and fail when line coverage of the gate scope is below `--unit-coverage-min`. The default minimum and the 100% target for EC read, write, decode, heal, metadata-quorum, and rollback paths are the `UNIT_COVERAGE_*` constants at the top of the runner. `cargo-llvm-cov` must be installed unless `--skip-coverage` is passed explicitly. The default scope `ec-critical` is: From 2e4ab045b661a58ed0732645768fe9b910f09318 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 15:34:29 +0800 Subject: [PATCH 11/40] test(scanner): add durable checkpoint diagnostics (#7175) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(scanner): add durable checkpoint diagnostics Refs rustfs/backlog#2260 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- Cargo.lock | 207 ++++----- Cargo.toml | 11 +- crates/ecstore/Cargo.toml | 1 + crates/ecstore/src/bucket/remote_s3_client.rs | 171 +++++++- crates/scanner/src/remote_scanner.rs | 3 + crates/scanner/src/remote_scanner/stream.rs | 42 ++ crates/scanner/src/scanner_folder/tests.rs | 2 + .../tests/checkpoint_fixture.rs | 410 ++++++++++++++++++ crates/scanner/src/scanner_io.rs | 8 + crates/scanner/src/scanner_io/tests.rs | 21 + docs/testing/README.md | 2 + docs/testing/scanner-checkpoint-fixture.md | 22 + 12 files changed, 791 insertions(+), 109 deletions(-) create mode 100644 crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs create mode 100644 docs/testing/scanner-checkpoint-fixture.md diff --git a/Cargo.lock b/Cargo.lock index e9e7d9fe5..1258d7e85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,7 +506,7 @@ dependencies = [ "arrow-select", "chrono", "half", - "indexmap 2.14.1", + "indexmap 2.14.2", "itoa", "lexical-core", "memchr", @@ -809,7 +809,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -891,9 +891,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -958,9 +958,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" +checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -986,9 +986,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.117.0" +version = "1.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b602641be84ebe5f96606cfefe4b96efaae1fd947c1b34ea8513b8ac0d2d8d" +checksum = "c243864bc3754be9f0001414e0162fdf1370fa62c70b0cfbe1da6a6221d553c7" dependencies = [ "arc-swap", "aws-credential-types", @@ -1012,9 +1012,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.144.0" +version = "1.145.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30dc8bf6baaf7d46336a0ca2c69f223d9b90d7a801fb3e28f7ea17b00dc6b1de" +checksum = "f0e6320417a37c8a62f78b443d0b4cf628b57cd340a09b0eb56173d47cc94e93" dependencies = [ "arc-swap", "aws-credential-types", @@ -1049,9 +1049,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.108.0" +version = "1.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15301b04372832947916607983b114b3374b9db0be058a00fb7513800de1f05" +checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" dependencies = [ "arc-swap", "aws-credential-types", @@ -1075,9 +1075,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.110.0" +version = "1.111.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72cc2c205cb27108183cf1856333f7d584c2ba0f505421b4209ca5828f9ea899" +checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" dependencies = [ "arc-swap", "aws-credential-types", @@ -1101,9 +1101,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.113.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68182ecb449f7537db0f4d5d25917789cf41e32074a9fe47b6a0b847fe1d2032" +checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" dependencies = [ "arc-swap", "aws-credential-types", @@ -1235,7 +1235,7 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "indexmap 2.14.1", + "indexmap 2.14.2", "pin-project-lite", "rustls", "rustls-native-certs", @@ -1406,9 +1406,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1660,7 +1660,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1679,7 +1679,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1734,7 +1734,7 @@ dependencies = [ "prettyplease 0.3.0", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1951,9 +1951,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.4" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", "jobserver", @@ -2061,7 +2061,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.6", + "crypto-common 0.1.7", "inout 0.1.4", ] @@ -2108,7 +2108,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2531,7 +2531,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", @@ -2556,11 +2556,11 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "typenum", ] @@ -2759,7 +2759,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2803,7 +2803,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2868,7 +2868,7 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "log", "object_store", @@ -2943,7 +2943,7 @@ dependencies = [ "foldhash 0.2.0", "half", "hashbrown 0.17.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "libc", "log", @@ -3148,7 +3148,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "recursive", "serde_json", @@ -3163,7 +3163,7 @@ checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", ] @@ -3304,7 +3304,7 @@ checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" dependencies = [ "datafusion-doc", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -3319,7 +3319,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "log", "recursive", @@ -3341,7 +3341,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.17.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "parking_lot", "petgraph 0.8.3", @@ -3375,7 +3375,7 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "hashbrown 0.17.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "parking_lot", "pin-project", @@ -3426,7 +3426,7 @@ dependencies = [ "futures", "half", "hashbrown 0.17.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "log", "num-traits", @@ -3479,7 +3479,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-functions-nested", - "indexmap 2.14.1", + "indexmap 2.14.2", "log", "recursive", "regex", @@ -3852,7 +3852,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.6", + "crypto-common 0.1.7", "subtle", ] @@ -3929,7 +3929,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -4117,7 +4117,7 @@ dependencies = [ "crypto-bigint 0.5.5", "digest 0.10.7", "ff 0.13.1", - "generic-array 0.14.9", + "generic-array 0.14.7", "group 0.13.0", "hkdf 0.12.4", "pem-rfc7468 0.7.0", @@ -4341,9 +4341,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "findshlibs" @@ -4517,7 +4517,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -4562,9 +4562,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -4577,7 +4577,7 @@ version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rustversion", "typenum", ] @@ -4656,7 +4656,7 @@ checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" dependencies = [ "fnv", "hashbrown 0.16.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "stable_deref_trait", ] @@ -4934,7 +4934,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.14.1", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -5583,9 +5583,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.1" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -5600,7 +5600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ "block-padding 0.3.3", - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -5847,9 +5847,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -5885,7 +5885,7 @@ dependencies = [ "crc", "crc32c", "flate2", - "indexmap 2.14.1", + "indexmap 2.14.2", "lz4", "snap", "uuid", @@ -5918,7 +5918,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", ] [[package]] @@ -6398,7 +6398,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.16.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "metrics", "ordered-float 5.5.0", "quanta", @@ -7027,7 +7027,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.22.1", + "base64 0.21.7", "chrono", "getrandom 0.2.17", "http 1.5.0", @@ -7703,7 +7703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.14.1", + "indexmap 2.14.2", ] [[package]] @@ -7714,7 +7714,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.14.1", + "indexmap 2.14.2", "serde", ] @@ -7989,9 +7989,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ "portable-atomic", ] @@ -8095,7 +8095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -8943,7 +8943,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -9662,6 +9662,7 @@ dependencies = [ "async-trait", "aws-credential-types", "aws-sdk-s3", + "aws-smithy-async", "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-smithy-types", @@ -9956,7 +9957,7 @@ dependencies = [ "bytes", "fnv", "hmac 0.13.0", - "indexmap 2.14.1", + "indexmap 2.14.2", "kafka-protocol", "metrics", "pbkdf2 0.13.0", @@ -11293,7 +11294,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", "der 0.7.10", - "generic-array 0.14.9", + "generic-array 0.14.7", "pkcs8 0.10.2", "subtle", "zeroize", @@ -11405,7 +11406,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -11424,7 +11425,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "itoa", "memchr", "serde", @@ -11469,7 +11470,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -11495,7 +11496,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.1", + "indexmap 2.14.2", "jiff", "schemars 0.9.0", "schemars 1.2.2", @@ -11549,7 +11550,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -12146,9 +12147,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -12271,7 +12272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -12367,7 +12368,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -12513,7 +12514,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -12558,9 +12559,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls", "tokio", @@ -12642,7 +12643,7 @@ version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "toml_datetime", "toml_parser", "winnow", @@ -12736,7 +12737,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.1", + "indexmap 2.14.2", "pin-project-lite", "slab", "sync_wrapper", @@ -13239,9 +13240,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -13252,9 +13253,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -13262,9 +13263,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -13272,22 +13273,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] @@ -13307,9 +13308,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -13856,7 +13857,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -13873,7 +13874,7 @@ dependencies = [ "flate2", "getrandom 0.4.3", "hmac 0.13.0", - "indexmap 2.14.1", + "indexmap 2.14.2", "lzma-rust2", "memchr", "pbkdf2 0.13.0", @@ -13921,18 +13922,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 05ddc1110..ea784a350 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -168,7 +168,7 @@ reqwest = "0.13.4" rustfs-kafka-async = { version = "1.3.1" } socket2 = { version = "0.6.5" } tokio = { version = "1.53.1" } -tokio-rustls = { default-features = false, version = "0.26.4" } +tokio-rustls = { default-features = false, version = "0.26.5" } tokio-stream = { version = "0.1.19" } tokio-test = "0.4.5" tokio-util = { version = "0.7.19" } @@ -238,11 +238,12 @@ arc-swap = "1.9.2" astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" } atoi = "3.1.0" atomic_enum = "0.3.0" -aws-config = { version = "1.11.0" } +aws-config = { version = "1.12.0" } aws-credential-types = { version = "1.3.0" } -aws-sdk-kms = { default-features = false, version = "1.117.0" } -aws-sdk-s3 = { default-features = false, version = "1.144.0" } -aws-sdk-sts = { default-features = false, version = "1.113.0" } +aws-sdk-kms = { default-features = false, version = "1.118.0" } +aws-sdk-s3 = { default-features = false, version = "1.145.0" } +aws-sdk-sts = { default-features = false, version = "1.114.0" } +aws-smithy-async = { version = "1.3.0" } aws-smithy-http-client = { default-features = false, version = "1.4.0" } aws-smithy-runtime-api = { version = "1.16.0" } aws-smithy-types = { version = "1.6.3" } diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 50aa7c1d3..d36c9d85b 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -244,6 +244,7 @@ windows-sys = { workspace = true, features = [ windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] } [dev-dependencies] +aws-smithy-async.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] } criterion = { workspace = true, features = ["html_reports"] } temp-env = { workspace = true, features = ["async_closure"] } diff --git a/crates/ecstore/src/bucket/remote_s3_client.rs b/crates/ecstore/src/bucket/remote_s3_client.rs index 20d3d0365..a434e09ac 100644 --- a/crates/ecstore/src/bucket/remote_s3_client.rs +++ b/crates/ecstore/src/bucket/remote_s3_client.rs @@ -652,9 +652,10 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option { #[cfg(test)] mod tests { use super::*; + use aws_smithy_async::time::TimeSource; use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode; use std::sync::Mutex; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec { RemoteS3EndpointSpec { @@ -824,6 +825,174 @@ mod tests { ); } + #[derive(Clone, Debug)] + struct ClockSkewTimeSource(Arc); + + impl TimeSource for ClockSkewTimeSource { + fn now(&self) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst)) + } + } + + #[derive(Clone, Debug)] + struct ClockSkewConnector { + request_headers: RecordedHeaders, + error_code: &'static str, + skew_seconds: i64, + clock: ClockSkewTimeSource, + } + + fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str { + headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + .unwrap_or_else(|| panic!("signed request must contain {name}")) + } + + fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime { + chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ") + .expect("SDK signing timestamp must use the SigV4 format") + } + + impl SmithyHttpConnector for ClockSkewConnector { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let mut headers = self.request_headers.lock().expect("clock skew request capture lock"); + assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD"); + headers.push( + request + .headers() + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ); + let server_time = chrono::DateTime::::from(self.clock.now()).naive_utc() + + chrono::Duration::seconds(self.skew_seconds); + let (status, body) = if headers.len() == 1 { + ( + 403, + format!("{}Clock skew fixture", self.error_code), + ) + } else { + (200, String::new()) + }; + let response = http::Response::builder() + .status(status) + .header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string()) + .header("content-type", "application/xml") + .header("content-length", body.len()) + .body(SdkBody::from(body)) + .expect("clock skew fixture response"); + HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response"))) + } + } + + async fn clock_skew_client( + error_code: &'static str, + skew_seconds: i64, + retry: RemoteS3RetryPolicy, + ) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) { + let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new())); + let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000))); + let connector = SharedHttpConnector::new(ClockSkewConnector { + request_headers: Arc::clone(&headers), + error_code, + skew_seconds, + clock: clock.clone(), + }); + let mut spec = spec("s3.example.com", true); + spec.retry = retry; + let config = build_remote_s3_config(&spec) + .await + .expect("clock skew fixture uses the production outbound configuration") + .http_client(http_client_fn(move |_settings, _components| connector.clone())) + .time_source(clock.clone()) + .build(); + (S3Client::from_conf(config), headers, clock) + } + + #[tokio::test(start_paused = true)] + async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() { + for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] { + for skew_seconds in [-600, 600] { + let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await; + let initial = chrono::DateTime::::from(clock.now()).naive_utc(); + client + .get_object() + .bucket("bucket") + .key("object") + .send() + .await + .expect("clock skew GET must retry successfully"); + assert_eq!( + headers.lock().expect("captured requests").len(), + 2, + "{error_code}: GET needs exactly one retry" + ); + clock.0.fetch_add(17, Ordering::SeqCst); + // SDK signing time is independent of Tokio's retry/scheduler clock. + tokio::time::advance(Duration::from_secs(61)).await; + client + .head_bucket() + .bucket("bucket") + .send() + .await + .expect("subsequent HEAD must use the client's cached skew"); + let headers = headers.lock().expect("captured signed requests"); + assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt"); + assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock"); + assert_eq!( + signing_time(&headers[1]), + initial + chrono::Duration::seconds(skew_seconds), + "{error_code}: retry must apply the measured offset exactly" + ); + assert_eq!( + signing_time(&headers[2]), + initial + chrono::Duration::seconds(skew_seconds + 17), + "{error_code}: the next operation must apply cached skew to the advanced signing clock" + ); + let signature = |index: usize| { + recorded_header(&headers[index], "authorization") + .rsplit_once("Signature=") + .expect("SigV4 authorization contains a signature") + .1 + }; + assert_ne!( + signature(0), + signature(1), + "{error_code}: retry must be signed again after adjusting its date" + ); + } + } + } + + #[tokio::test(start_paused = true)] + async fn remote_s3_clock_skew_respects_one_attempt_policy() { + use aws_smithy_types::error::metadata::ProvideErrorMetadata; + + for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] { + for retry in [ + RemoteS3RetryPolicy::Disabled, + RemoteS3RetryPolicy::Standard { max_attempts: 1 }, + ] { + let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await; + let error = client + .get_object() + .bucket("bucket") + .key("object") + .send() + .await + .expect_err("clock skew must not override the caller's one-attempt budget"); + assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code)); + assert_eq!( + headers.lock().expect("captured requests").len(), + 1, + "{error_code}: {retry:?} must send exactly one request" + ); + } + } + } + #[test] fn path_style_auto_and_path_force_path_style() { assert!(PathStyle::Auto.force_path_style()); diff --git a/crates/scanner/src/remote_scanner.rs b/crates/scanner/src/remote_scanner.rs index 42ee47c52..47ba5eacd 100644 --- a/crates/scanner/src/remote_scanner.rs +++ b/crates/scanner/src/remote_scanner.rs @@ -52,6 +52,9 @@ static REMOTE_SCANNER_CYCLE_REFRESH: LazyLock> = LazyLock::new(|| mod stream; +#[cfg(test)] +pub(crate) use stream::checkpoint_fixture_partial_return; + pub use stream::{RemoteScannerAdmission, RemoteScannerRequest, serve_remote_scanner_request}; pub(crate) use stream::{RemoteScannerOutcome, RemoteScannerScanSpec, scan_remote_bucket}; use stream::{RemoteScannerReplayCache, RemoteScannerRequestWire, RemoteScannerValidatedCycle}; diff --git a/crates/scanner/src/remote_scanner/stream.rs b/crates/scanner/src/remote_scanner/stream.rs index 7d128cdd4..26f3610b6 100644 --- a/crates/scanner/src/remote_scanner/stream.rs +++ b/crates/scanner/src/remote_scanner/stream.rs @@ -1017,6 +1017,48 @@ fn finish_remote_scanner_stream( #[cfg(test)] const TEST_NEXT_CYCLE: u64 = 11; +#[cfg(test)] +pub(crate) async fn checkpoint_fixture_partial_return(progress: (u64, u64), entries_visited: u64) { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let mut bytes = Vec::new(); + write_frame( + &mut bytes, + &writer_auth, + &mut 0, + &RemoteScannerFrame::terminal( + RemoteScannerProgress { + objects_scanned: progress.0, + directories_started: progress.1, + entries_visited, + }, + RemoteScannerFrameResult::Partial, + ), + ) + .await + .expect("checkpoint partial frame must encode"); + let frame = read_frame(&mut std::io::Cursor::new(bytes.as_slice()), &reader_auth, &mut 0) + .await + .expect("checkpoint progress frame must authenticate"); + assert_eq!(frame.progress.entries_visited, entries_visited); + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default()); + let result = consume_remote_scanner_stream( + std::io::Cursor::new(bytes), + parent, + budget.clone(), + "bucket", + DataUsageCacheSource::new(0, 0), + DataUsageScanPlanDigest([17; 32]), + reader_auth, + ) + .await + .expect("checkpoint partial frame must decode"); + assert!(matches!(result, RemoteScannerOutcome::Partial)); + assert_eq!(budget.progress(), progress); +} + #[cfg(test)] async fn consume_remote_scanner_stream( reader: R, diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index ade37c88c..650ccaab3 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -24,6 +24,8 @@ use std::io::Write; use std::os::unix::fs::{PermissionsExt, symlink}; use std::sync::Mutex; +mod checkpoint_fixture; + /// Reset the process-global alert cooldown map; test-only. fn reset_alert_cooldowns() { *SCANNER_ALERT_EMISSION_COOLDOWN diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs new file mode 100644 index 000000000..466383993 --- /dev/null +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs @@ -0,0 +1,410 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::scanner_budget::ScannerCycleBudgetConfig; +use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk}; +use crate::storage_api::scanner_io::ObjectIO; +use crate::{DataUsageCacheSource, DataUsageScanPlanDigest}; +use std::io::Cursor; +use tokio::io::AsyncReadExt; + +const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin"; +const STATIC_OBJECTS: u64 = 24; +const MAX_CACHE_BYTES: u64 = 1024 * 1024; +const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0); +const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]); + +/// Real cache persistence codec and CAS calls, backed by two bounded local files. +#[derive(Debug)] +struct FixtureStore { + root: tempfile::TempDir, + reject_save: AtomicBool, +} + +impl FixtureStore { + fn new() -> Arc { + Arc::new(Self { + root: tempfile::tempdir().expect("checkpoint fixture storage directory"), + reject_save: AtomicBool::new(false), + }) + } + + fn path(&self, object: &str) -> std::path::PathBuf { + assert!(object.ends_with(CACHE_NAME) || object.ends_with(&format!("{CACHE_NAME}.bkp"))); + self.root + .path() + .join(if object.ends_with(".bkp") { "backup" } else { "main" }) + } + + async fn strict_load(&self) -> DataUsageCache { + let bytes = tokio::fs::read(self.root.path().join("main")) + .await + .expect("saved checkpoint fixture must exist"); + decode_fixture(&bytes).expect("saved checkpoint fixture must contain a valid bucket root") + } +} + +#[async_trait::async_trait] +impl ObjectIO for FixtureStore { + type Error = crate::EcstoreError; + type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = crate::ScannerObjectOptions; + type ObjectInfo = crate::ScannerObjectInfo; + type GetObjectReader = crate::ScannerGetObjectReader; + type PutObjectReader = crate::ScannerPutObjReader; + + async fn get_object_reader( + &self, + _bucket: &str, + object: &str, + _range: Option, + _headers: Self::HeaderMap, + _options: &Self::ObjectOptions, + ) -> crate::EcstoreResult { + let bytes = tokio::fs::read(self.path(object)).await.map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + crate::EcstoreError::FileNotFound + } else { + crate::EcstoreError::from(error) + } + })?; + assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES); + Ok(crate::ScannerGetObjectReader { + stream: Box::new(Cursor::new(bytes)), + object_info: crate::ScannerObjectInfo { + etag: Some("fixture".into()), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }) + } + + async fn put_object( + &self, + _bucket: &str, + object: &str, + data: &mut Self::PutObjectReader, + options: &Self::ObjectOptions, + ) -> crate::EcstoreResult { + if self.reject_save.load(Ordering::SeqCst) { + return Err(crate::EcstoreError::PreconditionFailed); + } + let path = self.path(object); + let exists = tokio::fs::try_exists(&path).await?; + let preconditions = options.http_preconditions.as_ref().expect("checkpoint writes must use CAS"); + if (exists && preconditions.if_none_match_value() == Some("*")) + || (!exists && preconditions.if_match_value().is_some()) + || (exists && preconditions.if_match_value() != Some("fixture")) + { + return Err(crate::EcstoreError::PreconditionFailed); + } + let mut bytes = Vec::new(); + (&mut data.stream).take(MAX_CACHE_BYTES + 1).read_to_end(&mut bytes).await?; + assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES); + tokio::fs::write(path, bytes).await?; + Ok(crate::ScannerObjectInfo { + etag: Some("fixture".into()), + ..Default::default() + }) + } +} + +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for FixtureStore { + async fn delete_config_object( + &self, + _bucket: &str, + _object: &str, + _options: crate::ScannerObjectOptions, + ) -> crate::EcstoreResult { + Err(crate::EcstoreError::NotImplemented) + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) + } +} + +fn decode_fixture(bytes: &[u8]) -> Result { + if bytes.is_empty() || bytes.len() > usize::try_from(MAX_CACHE_BYTES).expect("fixture bound") { + return Err("missing or oversized checkpoint fixture"); + } + let cache = DataUsageCache::unmarshal(bytes).map_err(|_| "corrupt checkpoint fixture")?; + if cache.info.name != "bucket" || cache.checked_flatten("bucket").is_none() { + return Err("checkpoint fixture has no valid bucket root"); + } + Ok(cache) +} + +fn retained(cache: &DataUsageCache) -> u64 { + assert!( + !cache.root().is_some_and(|root| root.compacted), + "a compacted bucket root cannot prove static-prefix coverage" + ); + cache + .checked_flatten("bucket/static") + .map_or(0, |entry| u64::try_from(entry.objects).expect("fixture object count fits u64")) +} + +#[derive(Debug, PartialEq, Eq)] +enum CoverageDiagnosis { + Progress, + NoNewWork, + LostAtPrepare, + LostAtReload, + WalkWithoutRetention, +} + +fn diagnose(previous: u64, prepared: u64, walked: u64, scanned: u64, reloaded: u64) -> CoverageDiagnosis { + if reloaded < scanned { + CoverageDiagnosis::LostAtReload + } else if prepared < previous { + CoverageDiagnosis::LostAtPrepare + } else if walked > 0 && reloaded <= previous { + CoverageDiagnosis::WalkWithoutRetention + } else if reloaded > previous { + CoverageDiagnosis::Progress + } else { + CoverageDiagnosis::NoNewWork + } +} + +#[test] +fn checkpoint_fixture_diagnosis_rejects_walk_without_retention() { + assert_eq!(diagnose(4, 4, 9, 8, 8), CoverageDiagnosis::Progress); + assert_eq!(diagnose(4, 4, 9, 4, 4), CoverageDiagnosis::WalkWithoutRetention); + assert_eq!(diagnose(4, 0, 9, 4, 4), CoverageDiagnosis::LostAtPrepare); + assert_eq!(diagnose(4, 4, 9, 8, 4), CoverageDiagnosis::LostAtReload); + assert_eq!(diagnose(4, 4, 0, 4, 4), CoverageDiagnosis::NoNewWork); +} + +#[test] +fn checkpoint_fixture_missing_and_corrupt_inputs_fail() { + for bytes in [ + vec![], + vec![0xc1], + DataUsageCache::default().marshal_msg().expect("empty cache encoding"), + vec![0; usize::try_from(MAX_CACHE_BYTES + 1).expect("oversized fixture")], + ] { + assert!(decode_fixture(&bytes).is_err(), "invalid fixture must not become an empty complete root"); + } +} + +#[test] +fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() { + let mut cache = DataUsageCache::default(); + cache.info.name = "bucket".to_string(); + cache.replace("bucket", "", DataUsageEntry::default()); + cache.replace("bucket/static", "bucket", DataUsageEntry::default()); + for index in 0..4 { + cache.replace( + &format!("bucket/static/{index}"), + "bucket/static", + DataUsageEntry { + objects: 1, + ..Default::default() + }, + ); + } + cache.reduce_children_of(&hash_path("bucket/static"), 1, true); + let decoded = decode_fixture(&cache.marshal_msg().expect("encode compacted cache")).expect("decode compacted fixture"); + let entry = decoded + .find("bucket/static") + .expect("compaction must retain the static subtree root"); + assert!(entry.compacted); + assert!(entry.children.is_empty()); + assert_eq!( + retained(&decoded), + 4, + "compaction retains aggregate coverage even when leaf keys are absent" + ); +} + +#[tokio::test] +#[serial] +async fn checkpoint_fixture_save_reload_resume() { + run_checkpoint_fixture(false).await; +} + +#[tokio::test] +#[serial] +async fn checkpoint_fixture_hot_digest_diagnostic() { + run_checkpoint_fixture(true).await; +} + +async fn run_checkpoint_fixture(change_digest: bool) { + let (scanner, root) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(root.clone()), + }; + for index in 0..STATIC_OBJECTS { + write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await; + } + let store = FixtureStore::new(); + let mut previous = 0; + let mut visited = 0; + for round in 0..3_u8 { + write_test_object_metadata(&root, "bucket", "hot/current").await; + let mut cache = DataUsageCache::default(); + let revisions = cache + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("load checkpoint revisions"); + if round > 0 { + assert_eq!(retained(&store.strict_load().await), previous); + } + let plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, change_digest.then_some(u64::from(round))); + crate::scanner_io::current_cache_root_or_prepare_with_generation( + &mut cache, + "bucket", + SOURCE, + 11, + 7, + plan, + crate::scanner_io::DataUsageCacheReuseOptions { + require_source: true, + tier_registry_generation: None, + }, + ); + let prepared = retained(&cache); + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(4), + ..Default::default() + }, + ); + let outcome = scanner + .local_disk + .clone() + .nsscanner_disk( + budget.token(), + budget.clone(), + vec![scanner.local_disk.clone()], + cache, + None, + HealScanMode::Normal, + ) + .await + .expect("budgeted local disk scan returns partial cache"); + let ScannerDiskScanOutcome::Partial(cache) = outcome else { + panic!("budgeted fixture must remain partial") + }; + assert!(!cache.info.snapshot_complete, "partial must never publish a complete root"); + assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects)); + let scanned = retained(&cache); + cache + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect("persist partial checkpoint"); + let mut loaded = DataUsageCache::default(); + loaded + .load(store.clone(), CACHE_NAME) + .await + .expect("reload persisted partial checkpoint"); + let reloaded = retained(&loaded); + assert_eq!(reloaded, retained(&store.strict_load().await)); + assert_eq!(scanned, reloaded, "save/load must retain static subtree coverage"); + assert!(!loaded.info.snapshot_complete); + visited += budget.entries_visited(); + let diagnosis = diagnose(previous, prepared, budget.entries_visited(), scanned, reloaded); + eprintln!( + "checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}" + ); + if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() { + assert_eq!( + diagnosis, + CoverageDiagnosis::Progress, + "visited growth must produce durable static coverage" + ); + } + crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await; + previous = reloaded; + } + assert!(visited > 0, "fixture must exercise the directory walk"); + assert!(previous > 0, "fixture must retain and enumerate static subtree entries"); + + let mut loaded = DataUsageCache::default(); + let revisions = loaded + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("load final checkpoint"); + let before = tokio::fs::read(store.root.path().join("main")) + .await + .expect("read durable checkpoint bytes"); + let epoch_error = loaded + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1) + .await + .expect_err("stale publication epoch must reject persistence"); + assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED)); + store.reject_save.store(true, Ordering::SeqCst); + loaded.info.next_cycle += 1; + loaded + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect_err("injected save failure must not report durable progress"); + assert_eq!( + tokio::fs::read(store.root.path().join("main")) + .await + .expect("read unchanged checkpoint bytes"), + before + ); + + let parent = CancellationToken::new(); + parent.cancel(); + let budget = ScannerCycleBudget::new(&parent, Default::default()); + let result = scanner + .local_disk + .clone() + .nsscanner_disk( + budget.token(), + budget.clone(), + vec![scanner.local_disk.clone()], + loaded.clone(), + None, + HealScanMode::Normal, + ) + .await; + assert!(result.is_err(), "pre-scan cancellation must not produce a complete root"); + assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion"); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, Default::default()); + let result = scanner + .local_disk + .clone() + .nsscanner_disk( + budget.token(), + budget, + vec![scanner.local_disk.clone()], + loaded, + None, + HealScanMode::Normal, + ) + .await + .expect("unbounded scan must complete after durable partial progress"); + let ScannerDiskScanOutcome::Complete(cache) = result else { + panic!("unbounded fixture must produce a complete disk cache"); + }; + assert!(cache.info.snapshot_complete); + assert!(cache.info.scan_checkpoint.is_none()); + assert_eq!( + cache.checked_flatten("bucket").expect("complete bucket root").objects, + usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize") + ); +} diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index f568050d8..4807e11b8 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -209,6 +209,14 @@ fn scanner_bucket_cache_digest( DataUsageScanPlanDigest(hasher.finalize().into()) } +#[cfg(test)] +pub(crate) fn checkpoint_fixture_bucket_digest( + scan_plan_digest: DataUsageScanPlanDigest, + dirty_generation: Option, +) -> DataUsageScanPlanDigest { + scanner_bucket_cache_digest(scan_plan_digest, dirty_generation) +} + fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option) -> Result<()> { if results.iter().any(|result| result.info.last_update.is_some()) { return Ok(()); diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index a59d620fc..6fddafdfe 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -1048,6 +1048,27 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() { } } +#[test] +fn checkpoint_fixture_superseded_is_distinct_from_partial_and_cancel() { + for (budget, cancelled, bucket, expected) in [ + (false, false, ScannerBucketScanStatus::Complete, ScannerCycleStatus::Superseded), + (true, false, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete), + (false, true, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete), + ] { + assert_eq!( + classify_nsscanner_cycle( + true, + budget, + cancelled, + bucket, + DirtyUsageSnapshotStatus::Changed, + ScannerCycleActivityStatus::Unchanged + ), + expected, + ); + } +} + #[test] fn unverified_activity_defers_partial_and_floor_cycles() { let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable); diff --git a/docs/testing/README.md b/docs/testing/README.md index 3589d8ee1..60343b9e2 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -21,6 +21,8 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md). +The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation. + ## Naming conventions ### Reserved test-name substrings (migration gate) diff --git a/docs/testing/scanner-checkpoint-fixture.md b/docs/testing/scanner-checkpoint-fixture.md new file mode 100644 index 000000000..bc1992006 --- /dev/null +++ b/docs/testing/scanner-checkpoint-fixture.md @@ -0,0 +1,22 @@ +# Scanner Checkpoint Fixture + +The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark. + +Run the fixture and confirm the test filter selects a nonzero number of tests: + +```sh +cargo test -p rustfs-scanner --lib checkpoint_fixture -- --list +RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib checkpoint_fixture -- --nocapture +``` + +The unchanged-plan case requires durable static coverage to increase each round. The hot-plan diagnostic changes the bucket plan digest between rounds and reports where coverage is lost without asserting that a particular defect must remain present. To require progress in this diagnostic as well: + +```sh +RUST_MIN_STACK=4194304 RUSTFS_CHECKPOINT_REQUIRE_PROGRESS=1 cargo test -p rustfs-scanner --lib checkpoint_fixture_hot_digest_diagnostic -- --nocapture +``` + +A nonzero exit from the strict command means that walked work did not become additional retained static coverage. `LostAtPrepare` identifies invalidation before traversal; `LostAtReload` identifies loss between the returned cache and persisted data; `WalkWithoutRetention` identifies visited growth without durable coverage growth. Missing, corrupt, empty-root, and oversized checkpoint inputs are rejected by the strict fixture reader. Save failure and publication-epoch rejection must preserve the preceding file bytes. Parent cancellation is checked separately from object-budget exhaustion. Superseded classification is tested separately from either incomplete outcome. + +For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store. + +The synthetic namespace contains no customer data. Temporary files are removed with their owning fixture. Production scan semantics and persistent formats are unchanged, so rollback consists of removing these tests and this guide. A passing fixture alone does not establish that the field report in [issue #7108](https://github.com/rustfs/rustfs/issues/7108) has been independently reproduced or fixed. A field diagnosis must separately identify the source capture, cycle and leader identity, and decoded bucket/set caches. From 19a29a70270507a0df931083d85ba1ab6f124cd7 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 5 Sep 2026 15:41:48 +0800 Subject: [PATCH 12/40] test(s3): add Snowball tar-codec compatibility fixtures (#7157) Co-authored-by: Zhengchao An --- .gitignore | 1 + Cargo.lock | 229 +++++++- Cargo.toml | 5 +- crates/zip/Cargo.toml | 9 + .../snowball/minio-go-v7.3.0/README.md | 24 + .../snowball/minio-go-v7.3.0/generate/go.mod | 26 + .../snowball/minio-go-v7.3.0/generate/go.sum | 59 ++ .../snowball/minio-go-v7.3.0/generate/main.go | 193 ++++++ .../snowball/minio-go-v7.3.0/manifest.json | 51 ++ .../snowball/minio-go-v7.3.0/snowball.tar | Bin 0 -> 4096 bytes .../snowball/minio-go-v7.3.0/snowball.tar.s2 | Bin 0 -> 528 bytes crates/zip/tests/snowball_tar_codec_compat.rs | 548 ++++++++++++++++++ deny.toml | 4 +- docs/architecture/compat-cleanup-register.md | 2 +- 14 files changed, 1138 insertions(+), 13 deletions(-) create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar create mode 100644 crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2 create mode 100644 crates/zip/tests/snowball_tar_codec_compat.rs diff --git a/.gitignore b/.gitignore index b8d2eecbf..60325653d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ profile.json *.zst .secrets *.go +!crates/zip/tests/fixtures/snowball/**/generate/*.go *.pb *.svg deploy/logs/*.log.* diff --git a/Cargo.lock b/Cargo.lock index 1258d7e85..8206b1951 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "amq-protocol" version = "10.6.3" @@ -330,6 +336,19 @@ dependencies = [ "rustversion", ] +[[package]] +name = "archive-trait" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6080ea14ccf9019d7ce572c319e581c20a805c9bdc6dbfb9b988da003cbd1a3" +dependencies = [ + "cap-std", + "thiserror 2.0.20", + "tokio", + "walkdir", + "windows-sys 0.60.2", +] + [[package]] name = "arcstr" version = "1.2.0" @@ -1873,6 +1892,36 @@ dependencies = [ "serde_core", ] +[[package]] +name = "cap-primitives" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", +] + +[[package]] +name = "cap-std" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes 3.0.1", + "rustix", +] + [[package]] name = "cargo-platform" version = "0.3.3" @@ -4442,6 +4491,17 @@ dependencies = [ "pe-unwind-info", ] +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -5626,6 +5686,28 @@ dependencies = [ "tempfile", ] +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.52.0", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + [[package]] name = "io-uring" version = "0.7.14" @@ -6328,6 +6410,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "md-5" version = "0.10.6" @@ -10904,9 +10992,16 @@ dependencies = [ name = "rustfs-zip" version = "1.0.0-rc.5" dependencies = [ + "astral-tokio-tar", "async-compression", + "futures", "hotpath", "rustfs-rio", + "serde", + "serde_json", + "sha2 0.11.0", + "tar-codec", + "tar-framing", "thiserror 2.0.20", "tokio", ] @@ -10967,6 +11062,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", +] + [[package]] name = "rustls" version = "0.23.43" @@ -12242,6 +12347,28 @@ dependencies = [ "xattr", ] +[[package]] +name = "tar-codec" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea42eb144d30fcbf32c26dfea8959175bb8335bfb7b18d20c6ed32edecf0551" +dependencies = [ + "archive-trait", + "tar-framing", + "thiserror 2.0.20", + "tokio", +] + +[[package]] +name = "tar-framing" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "783223a7a6590be4227cb821e7ce80372575511f67c824921aba0752d8ad5573" +dependencies = [ + "thiserror 2.0.20", + "tokio", +] + [[package]] name = "tcp-stream" version = "0.34.14" @@ -13521,7 +13648,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -13539,14 +13675,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -13564,48 +13717,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "1.0.4" @@ -13615,6 +13816,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags 2.13.1", + "windows-sys 0.52.0", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/Cargo.toml b/Cargo.toml index ea784a350..a7ecf889b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -234,8 +234,11 @@ tokio-postgres-rustls = "0.14.0" # Utilities and Tools anyhow = "1.0.104" arc-swap = "1.9.2" -# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams. +# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork. astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" } +# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures. +tar-codec = "0.0.14" +tar-framing = "0.0.14" atoi = "3.1.0" atomic_enum = "0.3.0" aws-config = { version = "1.12.0" } diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 5558ffa44..312d2928d 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -49,5 +49,14 @@ rustfs-rio.workspace = true tokio = { workspace = true, features = ["io-util", "macros", "rt"] } thiserror = { workspace = true } +[dev-dependencies] +astral-tokio-tar = { workspace = true } +futures = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sha2 = { workspace = true } +tar-codec = { workspace = true } +tar-framing = { workspace = true } + [lints] workspace = true diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md new file mode 100644 index 000000000..d9b558d81 --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md @@ -0,0 +1,24 @@ +# minio-go Snowball fixtures + +These request bodies are generated by +`github.com/minio/minio-go/v7.Client.PutObjectsSnowball` at the version pinned +in `generate/go.mod`. They cover the raw TAR and S2-compressed forms accepted by +RustFS Snowball extraction. + +The decoded TAR intentionally ends immediately after the final padded member +body because minio-go flushes, rather than closes, its TAR writer. The +compatibility test permits that shape only when the authenticated request body +is complete at the exact member boundary; it does not make incomplete TAR +terminators generally valid. + +Regenerate them from this directory with Go 1.25: + +```console +cd generate +go mod download +go run . -out .. +``` + +`manifest.json` records the input objects and SHA-256 digest of each captured +request body. Review changes to the manifest and binary fixtures together when +updating minio-go. diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod new file mode 100644 index 000000000..e2fd8f481 --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod @@ -0,0 +1,26 @@ +module rustfs.local/snowball-fixture + +go 1.25.0 + +require github.com/minio/minio-go/v7 v7.3.0 + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect +) diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum new file mode 100644 index 000000000..0027f37cd --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum @@ -0,0 +1,59 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU= +github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go new file mode 100644 index 000000000..b03052943 --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go @@ -0,0 +1,193 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const minioGoVersion = "v7.3.0" + +type fixtureManifest struct { + Generator string `json:"generator"` + MinioGo string `json:"minio_go"` + GeneratedAt string `json:"generated_at"` + Objects []fixtureObject `json:"objects"` + Archives []fixtureArchive `json:"archives"` +} + +type fixtureObject struct { + Key string `json:"key"` + Body string `json:"body"` + ModTime string `json:"mod_time"` + VersionID string `json:"version_id,omitempty"` + Headers map[string][]string `json:"headers,omitempty"` +} + +type fixtureArchive struct { + File string `json:"file"` + Compressed bool `json:"compressed"` + Length int `json:"length"` + SHA256 string `json:"sha256"` +} + +func objects() []fixtureObject { + return []fixtureObject{ + { + Key: "alpha.txt", + Body: "alpha-body", + ModTime: "2024-01-02T03:04:05Z", + VersionID: "018cc251-f400-7c22-9e8d-8b1800000001", + Headers: map[string][]string{ + "Content-Type": {"text/plain"}, + "X-Amz-Meta-Owner": {"snowball-fixture"}, + "X-Amz-Tagging": {"project=rustfs&source=minio-go"}, + }, + }, + { + Key: "nested/世界.txt", + Body: "bravo-body", + ModTime: "2024-01-02T03:05:05Z", + Headers: map[string][]string{ + "Content-Language": {"zh-CN"}, + "X-Amz-Meta-Note": {"unicode-path"}, + }, + }, + } +} + +func captureSnowball(compressed bool, specs []fixtureObject) ([]byte, error) { + body := make(chan []byte, 1) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + payload, err := io.ReadAll(request.Body) + if err != nil { + http.Error(writer, err.Error(), http.StatusInternalServerError) + return + } + body <- payload + writer.Header().Set("ETag", `"snowball-fixture"`) + writer.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{ + // The S3 authentication layer removes AWS streaming-signature framing + // before Snowball extraction sees the request body. Anonymous signing + // captures those decoded archive bytes directly. + Creds: credentials.NewStatic("", "", "", credentials.SignatureAnonymous), + Secure: false, + Region: "us-east-1", + }) + if err != nil { + return nil, fmt.Errorf("construct minio client: %w", err) + } + + input := make(chan minio.SnowballObject, len(specs)) + for _, spec := range specs { + modTime, err := time.Parse(time.RFC3339, spec.ModTime) + if err != nil { + return nil, fmt.Errorf("parse mod time for %q: %w", spec.Key, err) + } + headers := make(http.Header, len(spec.Headers)) + for name, values := range spec.Headers { + headers[name] = append([]string(nil), values...) + } + input <- minio.SnowballObject{ + Key: spec.Key, + Size: int64(len(spec.Body)), + ModTime: modTime, + Content: bytes.NewReader([]byte(spec.Body)), + VersionID: spec.VersionID, + Headers: headers, + } + } + close(input) + + err = client.PutObjectsSnowball(context.Background(), "fixture-bucket", minio.SnowballOptions{ + Opts: minio.PutObjectOptions{ + ContentType: "application/octet-stream", + }, + InMemory: true, + Compress: compressed, + }, input) + if err != nil { + return nil, fmt.Errorf("generate snowball request: %w", err) + } + return <-body, nil +} + +func main() { + outDir := flag.String("out", "..", "fixture output directory") + flag.Parse() + + specs := objects() + archives := make([]fixtureArchive, 0, 2) + for _, fixture := range []struct { + name string + compressed bool + }{ + {name: "snowball.tar"}, + {name: "snowball.tar.s2", compressed: true}, + } { + payload, err := captureSnowball(fixture.compressed, specs) + if err != nil { + panic(err) + } + path := filepath.Join(*outDir, fixture.name) + if err := os.WriteFile(path, payload, 0o644); err != nil { + panic(fmt.Errorf("write %s: %w", path, err)) + } + digest := sha256.Sum256(payload) + archives = append(archives, fixtureArchive{ + File: fixture.name, + Compressed: fixture.compressed, + Length: len(payload), + SHA256: hex.EncodeToString(digest[:]), + }) + } + + manifest := fixtureManifest{ + Generator: "github.com/minio/minio-go/v7.Client.PutObjectsSnowball", + MinioGo: minioGoVersion, + GeneratedAt: "2026-09-05T00:00:00Z", + Objects: specs, + Archives: archives, + } + payload, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + panic(err) + } + payload = append(payload, '\n') + path := filepath.Join(*outDir, "manifest.json") + if err := os.WriteFile(path, payload, 0o644); err != nil { + panic(fmt.Errorf("write %s: %w", path, err)) + } +} diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json new file mode 100644 index 000000000..85191f09c --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json @@ -0,0 +1,51 @@ +{ + "generator": "github.com/minio/minio-go/v7.Client.PutObjectsSnowball", + "minio_go": "v7.3.0", + "generated_at": "2026-09-05T00:00:00Z", + "objects": [ + { + "key": "alpha.txt", + "body": "alpha-body", + "mod_time": "2024-01-02T03:04:05Z", + "version_id": "018cc251-f400-7c22-9e8d-8b1800000001", + "headers": { + "Content-Type": [ + "text/plain" + ], + "X-Amz-Meta-Owner": [ + "snowball-fixture" + ], + "X-Amz-Tagging": [ + "project=rustfs\u0026source=minio-go" + ] + } + }, + { + "key": "nested/世界.txt", + "body": "bravo-body", + "mod_time": "2024-01-02T03:05:05Z", + "headers": { + "Content-Language": [ + "zh-CN" + ], + "X-Amz-Meta-Note": [ + "unicode-path" + ] + } + } + ], + "archives": [ + { + "file": "snowball.tar", + "compressed": false, + "length": 4096, + "sha256": "f00f2789dcb65b567f722f49cfdac9705e7bdac6c0badae75194327c32193d2e" + }, + { + "file": "snowball.tar.s2", + "compressed": true, + "length": 528, + "sha256": "f8a9d9aa9b9ccdfae24ded1bff3741aacb935f1457a252efc9266674ff13c992" + } + ] +} diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar new file mode 100644 index 0000000000000000000000000000000000000000..ea427f547ce3c6a1a9a3b518b2056bf14b60bc9a GIT binary patch literal 4096 zcmeHJJ#K?A6lUfWOx;XuV;~(oR9RZ7O*3TZ>VQM2h>gG`BxmT_o}~w>)N2$PXcDSy zP^A%G9>#vp@9*>H3CnLe^Ldh*aKNH?!AzAat@}>VK*jR=ll;)Bv<}=02kravx1*V&rQ^j*Nh$9vX-Tt-n3EV8V;Gs|i zk88mbnhLp|u_yxbFjrZ^b;oL2-|0+Q5QHM2aU!3&r)W}CVV+*4GD|#8za0XB)NQ9y zUlsisN^$2?h=!hrZ3N~7Lg0EBgKIwY!ElC#RevG<(u>w&d>ujl$iox3-(wzO4H26V z$8`uP?gE6M3pL|UFxeLVn?W9B;Cv_F^#?QQZ_d}p-m~s9Ab6^{KbRao1KNH8G@;R$ zL*(H7U9jVS3*(;uyQ})%GC@s-uKE9-i6CPEr`rV>jdkKSHy+UcSSn645qi?+AZBWz zW5*~Zt@`WZ{q6JhrEIL8ocGB;gRiPSZA1PjvIgF3tk)72@BJVCw+Slw-zr4U|3_Ku RW-?3IO7{HUO>+N%)h|U}$ol{Q literal 0 HcmV?d00001 diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2 b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2 new file mode 100644 index 0000000000000000000000000000000000000000..b9337a76637692871095feba8ac926b594131a31 GIT binary patch literal 528 zcmW;JL2DC17zW@kA<=b7p^I5CBC-cj6egLO%x>agYhw@D_>Y;RYcd}VYW|Q4X zljh_P=tZc-iy(sH#k)VjgI5Lb1wl{{5l>RLy}utX&-Wgk0@$EwZ?*>cJiNJ&i%+f1 zbw0l%cp#Frifep09`LG~D==~SGDnwM^d+MUW2_&)EX^NC(EL> znG}he%6R7S;SlwsxtbiJu5xxk*;sPcNtOH1UO749h5tb=z6G zvA>f^qFCMyTuhq2Pi=zw3}e*rDMi;sGeAv`G&`*V(Mj8cEu(0{`^j$;Zqovw7n8#k z4YqOS1j1~_oQA`gow*5aV6Z0pCOmisCN!5m8J`bqI6hBq!7}pVVAtrHz>Go#-oiO4 zQYC^KJ+1xBYbKN?XCEJbk~q{Xod->;Q(6HZfc)_Pe7OkP>OC&QDG!Ca7@&4nXN{Gs z@b7578!O>XW#q?!Kx3{3I(4j^25x7&eE8wu+w;BFqhBv8V4V!GfnFV3Rx``!>xqSC WT{a^*(CUm-z=LHk;X82!{=$E|_>r3c literal 0 HcmV?d00001 diff --git a/crates/zip/tests/snowball_tar_codec_compat.rs b/crates/zip/tests/snowball_tar_codec_compat.rs new file mode 100644 index 000000000..53e37afa8 --- /dev/null +++ b/crates/zip/tests/snowball_tar_codec_compat.rs @@ -0,0 +1,548 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::io::Cursor; + +use futures::StreamExt; +use rustfs_zip::CompressionFormat; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use tar_codec::{Archive as _, DecodePolicy, Member, MemberPayload as _, PaxDecodePolicy, PaxVendorExtensionPolicy, TarArchive}; +use tar_framing::{ + FrameError, FrameErrorInner, PaxKeyword, PaxRecord, PaxValue, StreamPolicy, UstarKind, + logical::{MemberExtensions, PaxState, TarReader}, +}; +use tokio::io::AsyncReadExt; + +const FIXTURE_ROOT: &str = "fixtures/snowball/minio-go-v7.3.0"; +const RAW_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar"); +const S2_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2"); +const MANIFEST: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/manifest.json"); + +#[derive(Debug, Deserialize)] +struct FixtureManifest { + generator: String, + minio_go: String, + generated_at: String, + objects: Vec, + archives: Vec, +} + +#[derive(Debug, Deserialize)] +struct FixtureObject { + key: String, + body: String, + mod_time: String, + #[serde(default)] + version_id: String, + #[serde(default)] + headers: BTreeMap>, +} + +#[derive(Debug, Deserialize)] +struct FixtureArchive { + file: String, + compressed: bool, + length: usize, + sha256: String, +} + +#[derive(Debug, Eq, PartialEq)] +struct ParsedMember { + path: String, + size: u64, + mtime: Option, + body: Vec, + minio_pax: BTreeMap>>, +} + +fn sha256_hex(bytes: &[u8]) -> String { + let mut encoded = String::with_capacity(64); + for byte in Sha256::digest(bytes) { + write!(&mut encoded, "{byte:02x}").expect("writing to a String should not fail"); + } + encoded +} + +async fn decode_s2(bytes: &[u8]) -> Vec { + let mut decoder = CompressionFormat::S2 + .get_decoder(Cursor::new(bytes.to_vec())) + .expect("S2 fixture decoder should be available"); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).await.expect("S2 fixture should decode"); + decoded +} + +async fn parse_with_tokio_tar(bytes: &[u8]) -> Vec { + let mut archive = tokio_tar::Archive::new(Cursor::new(bytes.to_vec())); + let mut entries = archive.entries().expect("tokio-tar should create an entry stream"); + let mut parsed = Vec::new(); + + while let Some(entry) = entries.next().await { + let mut entry = entry.expect("tokio-tar should parse the fixture member"); + let kind = entry.header().entry_type(); + if kind == tokio_tar::EntryType::XGlobalHeader { + continue; + } + + let path_bytes = entry.path_bytes().expect("tokio-tar should resolve the fixture path"); + let path = std::str::from_utf8(path_bytes.as_ref()) + .expect("fixture paths should be UTF-8") + .to_owned(); + let size = entry.effective_size(); + let mtime = entry.header().mtime().ok(); + let mut minio_pax = BTreeMap::new(); + if let Some(extensions) = entry + .pax_extensions() + .await + .expect("tokio-tar should parse local PAX records") + { + for extension in extensions { + let extension = extension.expect("fixture PAX record should be valid"); + let key = extension.key().expect("fixture PAX keys should be UTF-8"); + if key.starts_with("minio.") { + minio_pax.insert(key.to_owned(), Some(extension.value_bytes().to_vec())); + } + } + } + let mut body = Vec::new(); + entry + .read_to_end(&mut body) + .await + .expect("tokio-tar should read the fixture body"); + parsed.push(ParsedMember { + path, + size, + mtime, + body, + minio_pax, + }); + } + parsed +} + +fn effective_minio_pax(state: &PaxState<'_>, known_keywords: &mut Vec) -> BTreeMap>> { + for extension in state.extensions() { + for record in extension.records() { + let keyword = record.keyword(); + if matches!(&keyword, PaxKeyword::Vendor { vendor, .. } if vendor.as_ref() == "minio") + && !known_keywords.contains(&keyword) + { + known_keywords.push(keyword); + } + } + } + + known_keywords + .iter() + .filter_map(|keyword| { + let record = state.effective_record(keyword)?; + let PaxRecord::Vendor { vendor, name, value } = record else { + return None; + }; + let key = format!("{vendor}.{name}"); + let value = match value { + PaxValue::Value(value) => Some(value.to_vec()), + PaxValue::Deleted => None, + }; + Some((key, value)) + }) + .collect() +} + +fn effective_mtime(header_mtime: Option, extensions: &MemberExtensions<'_>) -> Option { + let MemberExtensions::Pax(state) = extensions else { + return header_mtime; + }; + match state.effective_record(&PaxKeyword::Mtime) { + Some(PaxRecord::Mtime(PaxValue::Value(value))) => Some(*value), + Some(PaxRecord::Mtime(PaxValue::Deleted)) => None, + _ => header_mtime, + } +} + +fn padded_member_end(position: u64, size: u64) -> u64 { + let padded_size = size.checked_add(511).expect("fixture member size should not overflow") / 512 * 512; + position + .checked_add(512) + .and_then(|position| position.checked_add(padded_size)) + .expect("fixture member end should not overflow") +} + +fn is_authenticated_footerless_end(error: &FrameError, last_member_end: Option, request_body_complete: bool) -> bool { + // The production gate must source `request_body_complete` from RustFS's + // length, checksum, and trailing-header validation state. + request_body_complete && matches!(&error.inner, FrameErrorInner::MissingEndMarker) && last_member_end == Some(error.position) +} + +fn candidate_snowball_decode_policy() -> DecodePolicy { + DecodePolicy::default() + .allow_gnu(true) + .allow_all_nul_numeric_fields(true) + .max_gnu_extension_size(1_048_576) + .pax_policy( + PaxDecodePolicy::default() + .max_extension_size(1_048_576) + .max_global_extensions_size(67_108_864) + .allow_global_pax_extensions(false) + .allow_non_utf8_pax_vendor_values(false) + .allow_duplicate_pax_records(false) + .allow_global_pax_member_metadata(false) + .vendor_extension_policy(PaxVendorExtensionPolicy::ignore(["minio"])), + ) +} + +async fn parse_with_tar_framing(bytes: &[u8]) -> (Vec, Option, Option) { + let policy = StreamPolicy::default() + .max_pax_extension_size(1024 * 1024) + .max_global_pax_extensions_size(4 * 1024 * 1024) + .max_gnu_extension_size(128 * 1024); + let mut reader = TarReader::new(Cursor::new(bytes.to_vec())).with_policy(policy); + let mut parsed = Vec::new(); + let mut known_minio_keywords = Vec::new(); + let mut last_member_end = None; + + loop { + let mut frame = match reader.next_frame().await { + Ok(Some(frame)) => frame, + Ok(None) => return (parsed, None, last_member_end), + Err(error) => return (parsed, Some(error), last_member_end), + }; + assert_eq!(frame.header.kind, UstarKind::Regular); + let path = String::from_utf8( + frame + .effective_path() + .expect("tar-framing should resolve the fixture path") + .into_owned(), + ) + .expect("fixture paths should be UTF-8"); + let size = frame.header.effective_size; + let mtime = effective_mtime(frame.header.mtime, &frame.extensions); + let minio_pax = match &frame.extensions { + MemberExtensions::Pax(state) => effective_minio_pax(state, &mut known_minio_keywords), + MemberExtensions::Gnu { .. } => BTreeMap::new(), + }; + let mut body = Vec::new(); + let mut chunk = Vec::new(); + while frame + .payload + .next_chunk(&mut chunk, 64 * 1024) + .await + .expect("tar-framing should read the fixture body") + { + body.extend_from_slice(&chunk); + } + last_member_end = Some(padded_member_end(frame.header.position, size)); + parsed.push(ParsedMember { + path, + size, + mtime, + body, + minio_pax, + }); + } +} + +#[test] +fn checked_in_fixtures_match_the_minio_go_manifest() { + let manifest: FixtureManifest = serde_json::from_slice(MANIFEST).expect("fixture manifest should be valid JSON"); + assert_eq!(manifest.generator, "github.com/minio/minio-go/v7.Client.PutObjectsSnowball"); + assert_eq!(manifest.minio_go, "v7.3.0"); + assert_eq!(manifest.generated_at, "2026-09-05T00:00:00Z"); + assert_eq!(manifest.objects.len(), 2); + assert_eq!(manifest.objects[0].key, "alpha.txt"); + assert_eq!(manifest.objects[0].body, "alpha-body"); + assert_eq!(manifest.objects[0].mod_time, "2024-01-02T03:04:05Z"); + assert_eq!(manifest.objects[0].version_id, "018cc251-f400-7c22-9e8d-8b1800000001"); + assert_eq!( + manifest.objects[0].headers.get("X-Amz-Meta-Owner"), + Some(&vec!["snowball-fixture".to_owned()]) + ); + + for archive in &manifest.archives { + let bytes = match archive.file.as_str() { + "snowball.tar" => RAW_FIXTURE, + "snowball.tar.s2" => S2_FIXTURE, + file => panic!("unexpected archive in {FIXTURE_ROOT}/manifest.json: {file}"), + }; + assert_eq!(bytes.len(), archive.length); + assert_eq!(sha256_hex(bytes), archive.sha256); + assert_eq!(archive.compressed, archive.file.ends_with(".s2")); + } +} + +#[tokio::test] +async fn minio_go_raw_and_s2_fixtures_have_identical_footerless_tar_data() { + assert_eq!(decode_s2(S2_FIXTURE).await, RAW_FIXTURE); + assert_eq!(RAW_FIXTURE.len() % 512, 0); + assert!(RAW_FIXTURE.len() >= 1024); + assert!( + !RAW_FIXTURE[RAW_FIXTURE.len() - 1024..].iter().all(|byte| *byte == 0), + "minio-go Flush output should not contain the standard two-block terminator" + ); +} + +#[tokio::test] +async fn tar_framing_matches_tokio_tar_before_rejecting_the_missing_terminator() { + let expected = parse_with_tokio_tar(RAW_FIXTURE).await; + let (actual, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await; + let error = error.expect("footerless minio-go fixture should fail strict termination"); + + assert_eq!(actual, expected); + assert_eq!( + actual, + [ + ParsedMember { + path: "alpha.txt".to_owned(), + size: 10, + mtime: Some(1_704_164_645), + body: b"alpha-body".to_vec(), + minio_pax: BTreeMap::from([ + ("minio.metadata.Content-Type".to_owned(), Some(b"text/plain".to_vec()),), + ("minio.metadata.X-Amz-Meta-Owner".to_owned(), Some(b"snowball-fixture".to_vec()),), + ( + "minio.metadata.X-Amz-Tagging".to_owned(), + Some(b"project=rustfs&source=minio-go".to_vec()), + ), + ("minio.versionId".to_owned(), Some(b"018cc251-f400-7c22-9e8d-8b1800000001".to_vec()),), + ]), + }, + ParsedMember { + path: "nested/世界.txt".to_owned(), + size: 10, + mtime: Some(1_704_164_705), + body: b"bravo-body".to_vec(), + minio_pax: BTreeMap::from([ + ("minio.metadata.Content-Language".to_owned(), Some(b"zh-CN".to_vec()),), + ("minio.metadata.X-Amz-Meta-Note".to_owned(), Some(b"unicode-path".to_vec()),), + ]), + }, + ] + ); + assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker)); + assert_eq!( + error.position, + u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64") + ); + assert_eq!(last_member_end, Some(error.position)); +} + +#[tokio::test] +async fn footerless_compatibility_requires_authenticated_eof_at_the_member_boundary() { + let (_, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await; + let error = error.expect("the real fixture should be footerless"); + assert!(is_authenticated_footerless_end(&error, last_member_end, true)); + assert!(!is_authenticated_footerless_end(&error, last_member_end, false)); + + let mut one_zero_block = RAW_FIXTURE.to_vec(); + one_zero_block.extend([0; 512]); + let (_, error, last_member_end) = parse_with_tar_framing(&one_zero_block).await; + let error = error.expect("one zero block is not a valid TAR terminator"); + assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker)); + assert_eq!( + last_member_end, + Some(u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64")) + ); + assert_eq!( + error.position, + u64::try_from(one_zero_block.len()).expect("fixture length should fit in u64") + ); + assert!(!is_authenticated_footerless_end(&error, last_member_end, true)); +} + +#[tokio::test] +async fn tar_codec_policy_accepts_only_the_explicit_minio_vendor_namespace() { + let default_error = match TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())).members().next().await { + Err(error) => error, + Ok(_) => panic!("the default policy should reject minio vendor records"), + }; + assert!(default_error.to_string().contains("pax vendor extension minio.")); + + let mut members = TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())) + .with_policy(candidate_snowball_decode_policy()) + .members(); + let mut bodies = Vec::new(); + loop { + let member = match members.next().await { + Ok(Some(member)) => member, + Ok(None) => panic!("footerless minio-go fixture should not report a valid archive end"), + Err(error) => { + assert!(error.to_string().contains("missing two-block end-of-archive marker")); + break; + } + }; + let Member::File { mut payload, .. } = member else { + panic!("fixture should contain only regular files"); + }; + let mut body = Vec::new(); + let mut chunk = Vec::new(); + while payload + .next_chunk(&mut chunk, 64 * 1024) + .await + .expect("tar-codec should read the fixture body") + { + body.extend_from_slice(&chunk); + } + bodies.push(body); + } + assert_eq!(bodies, [b"alpha-body".to_vec(), b"bravo-body".to_vec()]); + assert!( + members + .next() + .await + .expect("the member cursor should be fused after an error") + .is_none() + ); +} + +fn pax_record(key: &str, value: &str) -> Vec { + let payload = format!("{key}={value}\n"); + let mut len = payload.len() + 3; + loop { + let record = format!("{len} {payload}"); + if record.len() == len { + return record.into_bytes(); + } + len = record.len(); + } +} + +async fn append_pax_header( + builder: &mut tokio_tar::Builder>>, + entry_type: tokio_tar::EntryType, + records: &[(&str, &str)], +) { + let mut payload = Vec::new(); + for (key, value) in records { + payload.extend(pax_record(key, value)); + } + let mut header = tokio_tar::Header::new_ustar(); + header.set_entry_type(entry_type); + header.set_size(u64::try_from(payload.len()).expect("PAX test payload should fit in u64")); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload)) + .await + .expect("PAX test header should be written"); +} + +async fn append_regular(builder: &mut tokio_tar::Builder>>, path: &str) { + let body = path.as_bytes(); + let mut header = tokio_tar::Header::new_ustar(); + header.set_entry_type(tokio_tar::EntryType::Regular); + header.set_size(u64::try_from(body.len()).expect("test member body should fit in u64")); + header.set_mode(0o644); + header.set_mtime(1_704_164_645); + header.set_cksum(); + builder + .append_data(&mut header, path, Cursor::new(body)) + .await + .expect("ordinary test member should be written"); +} + +async fn archive_with_local_pax(records: &[(&str, &str)]) -> Vec { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, records).await; + append_regular(&mut builder, "member.txt").await; + builder.into_inner().await.expect("policy archive should finish").into_inner() +} + +#[tokio::test] +async fn candidate_policy_rejects_unknown_vendor_and_duplicate_pax_records() { + let unknown_vendor = archive_with_local_pax(&[("acme.metadata.owner", "mallory")]).await; + let error = match TarArchive::new(Cursor::new(unknown_vendor)) + .with_policy(candidate_snowball_decode_policy()) + .members() + .next() + .await + { + Err(error) => error, + Ok(_) => panic!("the candidate Snowball policy should reject unknown vendors"), + }; + assert!( + error + .to_string() + .contains("pax vendor extension acme.metadata.owner is not allowed") + ); + + let duplicate = archive_with_local_pax(&[ + ("minio.metadata.x-amz-meta-owner", "first"), + ("minio.metadata.x-amz-meta-owner", "second"), + ]) + .await; + let error = match TarArchive::new(Cursor::new(duplicate)) + .with_policy(candidate_snowball_decode_policy()) + .members() + .next() + .await + { + Err(error) => error, + Ok(_) => panic!("the candidate Snowball policy should reject duplicate PAX records"), + }; + assert!( + error + .to_string() + .contains("pax extended header contains duplicate record minio.metadata.x-amz-meta-owner") + ); +} + +#[tokio::test] +async fn global_minio_pax_inheritance_is_an_explicit_migration_difference() { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + append_pax_header( + &mut builder, + tokio_tar::EntryType::XGlobalHeader, + &[("minio.metadata.x-amz-meta-owner", "global")], + ) + .await; + append_pax_header( + &mut builder, + tokio_tar::EntryType::XHeader, + &[("minio.metadata.x-amz-meta-owner", "local")], + ) + .await; + append_regular(&mut builder, "local.txt").await; + append_regular(&mut builder, "inherited.txt").await; + let archive = builder + .into_inner() + .await + .expect("precedence archive should finish") + .into_inner(); + + let legacy = parse_with_tokio_tar(&archive).await; + let (framing, error, _) = parse_with_tar_framing(&archive).await; + assert!(error.is_none()); + assert_eq!(legacy.len(), 2); + assert_eq!(framing.len(), 2); + + let owner_key = "minio.metadata.x-amz-meta-owner"; + assert_eq!(legacy[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec()))); + assert!(!legacy[1].minio_pax.contains_key(owner_key)); + assert_eq!(framing[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec()))); + assert_eq!(framing[1].minio_pax.get(owner_key), Some(&Some(b"global".to_vec()))); + + let error = match TarArchive::new(Cursor::new(archive)) + .with_policy(candidate_snowball_decode_policy()) + .members() + .next() + .await + { + Err(error) => error, + Ok(_) => panic!("the candidate Snowball policy should reject global PAX state"), + }; + assert!(error.to_string().contains("global pax extended headers are not allowed")); +} diff --git a/deny.toml b/deny.toml index fc5d25916..cfecab82f 100644 --- a/deny.toml +++ b/deny.toml @@ -37,8 +37,8 @@ unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] allow-git = [ # Temporary tokio-tar fork pinned to the reviewed parser limits, - # cancellation safety, and error-fusing change while - # astral-sh/tokio-tar#118 awaits an upstream release. + # cancellation safety, and error-fusing change while Snowball is + # prototyped against tar-codec and Swift retains its current reader. # owner: cxymds review: 2026-10 "https://github.com/cxymds/tokio-tar.git", # Official s3s repository. Temporarily pinned to the merged generic REST diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 6472481a0..924193b6e 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -13,7 +13,7 @@ - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation. - `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains. -- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release. +- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on precedence-resolved MinIO PAX metadata; per-entry and cumulative extension limits; a physical-entry limit; cancellation-safe parsing and ownership of large streamed members; fused streams after errors; and compatibility with minio-go streams that omit the two-block terminator. Swift bulk extraction also uses the same fork. Keep the reviewed pin while the Snowball path is prototyped against tar-codec/tar-framing. Remove it only after a released API exposes the effective allowed vendor records, RustFS provides a cancellation-safe handoff for borrowed member payloads, footerless input is accepted solely when authenticated request framing proves EOF immediately after a complete member, the existing resource-limit, cancellation, error-fuse, and real minio-go fixtures pass against the replacement, and Swift no longer depends on the fork. - `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources. - `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources. - `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive. From 882d9ca8a4fbc0bc22d670e6d33ce758fd1eaf4b Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 15:46:13 +0800 Subject: [PATCH 13/40] refactor(ecstore): isolate metadata quorum decisions (#7165) * refactor(ecstore): isolate metadata quorum decisions * test(ecstore): match sealed context fixture map type * test(ecstore): count decommission faults across retry restarts --- .../src/set_disk/core/io_primitives.rs | 390 +----------------- .../src/set_disk/core/metadata_quorum.rs | 385 +++++++++++++++++ crates/ecstore/src/set_disk/core/mod.rs | 1 + 3 files changed, 407 insertions(+), 369 deletions(-) create mode 100644 crates/ecstore/src/set_disk/core/metadata_quorum.rs diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index a7c52549d..18b04c781 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -59,27 +59,34 @@ use super::super::{ send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn, }; #[cfg(test)] +pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision; +pub(in crate::set_disk) use super::metadata_quorum::{ + MetadataQuorumAccumulator, is_metadata_fanout_ignored_error, metadata_early_stop_candidate_matches, +}; +#[cfg(test)] use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; #[cfg(test)] use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH; #[cfg(test)] use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD; use crate::diagnostics::get::{ - GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, + GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, + GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, + GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, + GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, + GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, + GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, + record_get_stage_duration_if_enabled, +}; +#[cfg(test)] +use crate::diagnostics::get::{ GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, - GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, - GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, - GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, - GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR, - GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID, - GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, - GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, - GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, - GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, record_get_stage_duration_if_enabled, + GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, }; #[cfg(test)] use crate::disk::CHECK_PART_FILE_NOT_FOUND; @@ -690,324 +697,6 @@ impl MetadataFanoutDiagnostics { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(in crate::set_disk) struct MetadataEarlyStopDecision { - pub(in crate::set_disk) reason: &'static str, -} - -#[derive(Clone, Debug)] -pub(in crate::set_disk) struct MetadataQuorumAccumulator { - pub(in crate::set_disk) total_disks: usize, - pub(in crate::set_disk) default_parity_count: usize, - pub(in crate::set_disk) allow_early_stop: bool, - pub(in crate::set_disk) valid_responses: usize, - pub(in crate::set_disk) not_found_responses: usize, - pub(in crate::set_disk) version_not_found_responses: usize, - pub(in crate::set_disk) ignored_errors: usize, - pub(in crate::set_disk) hard_errors: usize, - pub(in crate::set_disk) candidate: Option, - pub(in crate::set_disk) candidate_votes: usize, - // Bitset of shard indexes whose metadata matches the candidate. Erasure - // layouts are capped at 16 shards, so this stays allocation-free on the - // GET metadata hot path. - candidate_shard_mask: u16, - pub(in crate::set_disk) conflicting_metadata: bool, - pub(in crate::set_disk) delete_marker_seen: bool, - pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>, - pub(in crate::set_disk) delete_marker_votes: usize, - pub(in crate::set_disk) requested_version_id: String, - pub(in crate::set_disk) matching_version_votes: usize, -} - -impl MetadataQuorumAccumulator { - pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self { - Self { - total_disks, - default_parity_count, - allow_early_stop, - valid_responses: 0, - not_found_responses: 0, - version_not_found_responses: 0, - ignored_errors: 0, - hard_errors: 0, - candidate: None, - candidate_votes: 0, - candidate_shard_mask: 0, - conflicting_metadata: false, - delete_marker_seen: false, - delete_marker_candidates: Vec::new(), - delete_marker_votes: 0, - requested_version_id: String::new(), - matching_version_votes: 0, - } - } - - pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self { - self.requested_version_id = version_id.to_string(); - self - } - - pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) { - self.observe_file_info_with_index(None, file_info); - } - - pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) { - self.observe_file_info_with_index(Some(disk_index), file_info); - } - - fn observe_file_info_with_index(&mut self, disk_index: Option, file_info: &FileInfo) { - if !file_info_is_valid_for_metadata(file_info) { - self.hard_errors = self.hard_errors.saturating_add(1); - return; - } - - self.valid_responses = self.valid_responses.saturating_add(1); - - // Track version match for versioned requests - if !self.requested_version_id.is_empty() - && let Some(ref vid) = file_info.version_id - && vid.to_string() == self.requested_version_id - { - self.matching_version_votes = self.matching_version_votes.saturating_add(1); - } - - if file_info.is_canonical_delete_marker() { - self.delete_marker_seen = true; - if let Some((_, votes)) = self - .delete_marker_candidates - .iter_mut() - .find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info)) - { - *votes = votes.saturating_add(1); - } else { - self.delete_marker_candidates.push((file_info.clone(), 1)); - } - self.delete_marker_votes = self - .delete_marker_candidates - .iter() - .map(|(_, votes)| *votes) - .max() - .unwrap_or_default(); - self.conflicting_metadata |= self.delete_marker_candidates.len() > 1; - return; - } - - match &self.candidate { - Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => { - self.candidate_votes = self.candidate_votes.saturating_add(1); - if let Some(disk_index) = disk_index - && let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index) - { - self.candidate_shard_mask |= bit; - } - } - Some(_) => { - self.conflicting_metadata = true; - } - None => { - self.candidate = Some(file_info.clone()); - self.candidate_votes = 1; - if let Some(disk_index) = disk_index - && let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index) - { - self.candidate_shard_mask |= bit; - } - } - } - } - - fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option { - let &erasure_index = candidate.erasure.distribution.get(disk_index)?; - if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index { - return None; - } - Some(1u16 << (erasure_index - 1)) - } - - pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool { - self.candidate_read_reserve_target() - .is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required) - } - - pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option { - let candidate = self.candidate.as_ref()?; - Some( - candidate - .erasure - .data_blocks - .saturating_add(usize::from(candidate.erasure.parity_blocks > 0)), - ) - } - - pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) { - match err { - DiskError::FileNotFound | DiskError::VolumeNotFound => { - self.not_found_responses = self.not_found_responses.saturating_add(1); - } - DiskError::FileVersionNotFound => { - self.version_not_found_responses = self.version_not_found_responses.saturating_add(1); - } - _ if is_metadata_fanout_ignored_error(err) => { - self.ignored_errors = self.ignored_errors.saturating_add(1); - } - _ => { - self.hard_errors = self.hard_errors.saturating_add(1); - } - } - } - - pub(in crate::set_disk) fn early_stop_decision(&self) -> Option { - if !self.allow_early_stop { - return None; - } - if self.delete_marker_votes >= self.default_write_quorum() { - return Some(MetadataEarlyStopDecision { - reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, - }); - } - if self.conflicting_metadata - || self.delete_marker_seen - || self.not_found_responses > 0 - || self.version_not_found_responses > 0 - || self.hard_errors > 0 - { - return None; - } - if self - .candidate - .as_ref() - .and_then(|candidate| self.candidate_latest_quorum(candidate)) - .is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum) - { - return Some(MetadataEarlyStopDecision { - reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, - }); - } - None - } - - /// Check if a versioned request can early-stop because the requested - /// version_id has reached quorum across disks. - pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option { - if !self.allow_early_stop { - return None; - } - if self.requested_version_id.is_empty() { - return None; - } - if self.conflicting_metadata - || self.delete_marker_seen - || self.not_found_responses > 0 - || self.version_not_found_responses > 0 - || self.hard_errors > 0 - { - return None; - } - if self.matching_version_votes >= self.read_quorum_for_version() { - return Some(MetadataEarlyStopDecision { - reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, - }); - } - None - } - - pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool { - if !self.allow_early_stop { - return false; - } - if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() { - return true; - } - if self.conflicting_metadata - || self.delete_marker_seen - || self.not_found_responses > 0 - || self.version_not_found_responses > 0 - || self.hard_errors > 0 - { - return false; - } - if !self.requested_version_id.is_empty() - && self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version() - { - return true; - } - match &self.candidate { - Some(candidate) => self - .candidate_latest_quorum(candidate) - .is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum), - None => pending >= self.default_write_quorum(), - } - } - - /// Compute the read quorum threshold for version-aware early-stop. - /// Uses `total_disks / 2` (like `missing_response_quorum`) when - /// `default_parity_count` is set, otherwise requires all disks. - pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize { - self.missing_response_quorum() - } - - pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str { - if !self.allow_early_stop { - return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST; - } - if self.conflicting_metadata { - return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA; - } - if self.delete_marker_seen { - return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER; - } - let missing_response_quorum = self.missing_response_quorum(); - if self.version_not_found_responses >= missing_response_quorum { - return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND; - } - if self.not_found_responses >= missing_response_quorum { - return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND; - } - if self.hard_errors > 0 { - return GET_METADATA_EARLY_STOP_REASON_ERROR; - } - if self.ignored_errors > 0 { - return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM; - } - GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM - } - - pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option { - if self.default_parity_count == 0 { - return Some(self.total_disks); - } - if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks { - return None; - } - let data_blocks = candidate.erasure.data_blocks; - Some(if data_blocks == candidate.erasure.parity_blocks { - data_blocks.saturating_add(1) - } else { - data_blocks - }) - } - - pub(crate) fn default_write_quorum(&self) -> usize { - if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { - return self.total_disks; - } - let data_blocks = self.total_disks.saturating_sub(self.default_parity_count); - if data_blocks == self.default_parity_count { - data_blocks.saturating_add(1) - } else { - data_blocks - } - } - - pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize { - if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { - self.total_disks - } else { - self.total_disks / 2 - } - } -} - #[derive(Clone, Debug)] pub(in crate::set_disk) enum MetadataCacheLookup { Hit(Arc), @@ -1015,39 +704,6 @@ pub(in crate::set_disk) enum MetadataCacheLookup { RejectedInsufficientQuorum, } -pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool { - left.volume == right.volume - && left.name == right.name - && left.version_id == right.version_id - && left.is_latest == right.is_latest - && left.deleted == right.deleted - && left.mark_deleted == right.mark_deleted - && left.transition_status == right.transition_status - && left.transitioned_objname == right.transitioned_objname - && left.transition_tier == right.transition_tier - && left.transition_version_id == right.transition_version_id - && left.transition_version == right.transition_version - && left.transition_version_state == right.transition_version_state - && left.expire_restored == right.expire_restored - && left.size == right.size - && left.mod_time == right.mod_time - && left.mode == right.mode - && left.written_by_version == right.written_by_version - && left.metadata == right.metadata - && left.replication_state_internal == right.replication_state_internal - && left.parts == right.parts - && left.checksum == right.checksum - && left.versioned == right.versioned - && left.num_versions == right.num_versions - && left.successor_mod_time == right.successor_mod_time - && left.data_dir == right.data_dir - && left.erasure.algorithm == right.erasure.algorithm - && left.erasure.data_blocks == right.erasure.data_blocks - && left.erasure.parity_blocks == right.erasure.parity_blocks - && left.erasure.block_size == right.erasure.block_size - && left.erasure.distribution == right.erasure.distribution -} - pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( bucket: &str, object: &str, @@ -1247,10 +903,6 @@ pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> } } -pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool { - OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err) -} - pub(in crate::set_disk) fn is_confirmed_missing_part_error(err: Option<&str>) -> bool { let Some(err) = err else { return false; diff --git a/crates/ecstore/src/set_disk/core/metadata_quorum.rs b/crates/ecstore/src/set_disk/core/metadata_quorum.rs new file mode 100644 index 000000000..2922b6fc6 --- /dev/null +++ b/crates/ecstore/src/set_disk/core/metadata_quorum.rs @@ -0,0 +1,385 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pure metadata quorum and early-stop decisions for `SetDisks` reads. +//! +//! Disk scheduling, coalescing, cancellation, and late shard materialization +//! remain with their existing owners; this module only classifies observations. + +use crate::diagnostics::get::{ + GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, + GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, + GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, + GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, + GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, +}; +use crate::disk::error::DiskError; +use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS; +use crate::set_disk::file_info_is_valid_for_metadata; +use rustfs_filemeta::FileInfo; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::set_disk) struct MetadataEarlyStopDecision { + pub(in crate::set_disk) reason: &'static str, +} + +#[derive(Clone, Debug)] +pub(in crate::set_disk) struct MetadataQuorumAccumulator { + pub(in crate::set_disk) total_disks: usize, + pub(in crate::set_disk) default_parity_count: usize, + pub(in crate::set_disk) allow_early_stop: bool, + pub(in crate::set_disk) valid_responses: usize, + pub(in crate::set_disk) not_found_responses: usize, + pub(in crate::set_disk) version_not_found_responses: usize, + pub(in crate::set_disk) ignored_errors: usize, + pub(in crate::set_disk) hard_errors: usize, + pub(in crate::set_disk) candidate: Option, + pub(in crate::set_disk) candidate_votes: usize, + // Bitset of shard indexes whose metadata matches the candidate. Erasure + // layouts are capped at 16 shards, so this stays allocation-free on the + // GET metadata hot path. + candidate_shard_mask: u16, + pub(in crate::set_disk) conflicting_metadata: bool, + pub(in crate::set_disk) delete_marker_seen: bool, + pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>, + pub(in crate::set_disk) delete_marker_votes: usize, + pub(in crate::set_disk) requested_version_id: String, + pub(in crate::set_disk) matching_version_votes: usize, +} + +impl MetadataQuorumAccumulator { + pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self { + Self { + total_disks, + default_parity_count, + allow_early_stop, + valid_responses: 0, + not_found_responses: 0, + version_not_found_responses: 0, + ignored_errors: 0, + hard_errors: 0, + candidate: None, + candidate_votes: 0, + candidate_shard_mask: 0, + conflicting_metadata: false, + delete_marker_seen: false, + delete_marker_candidates: Vec::new(), + delete_marker_votes: 0, + requested_version_id: String::new(), + matching_version_votes: 0, + } + } + + pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self { + self.requested_version_id = version_id.to_string(); + self + } + + pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) { + self.observe_file_info_with_index(None, file_info); + } + + pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) { + self.observe_file_info_with_index(Some(disk_index), file_info); + } + + fn observe_file_info_with_index(&mut self, disk_index: Option, file_info: &FileInfo) { + if !file_info_is_valid_for_metadata(file_info) { + self.hard_errors = self.hard_errors.saturating_add(1); + return; + } + + self.valid_responses = self.valid_responses.saturating_add(1); + + // Track version match for versioned requests + if !self.requested_version_id.is_empty() + && let Some(ref vid) = file_info.version_id + && vid.to_string() == self.requested_version_id + { + self.matching_version_votes = self.matching_version_votes.saturating_add(1); + } + + if file_info.is_canonical_delete_marker() { + self.delete_marker_seen = true; + if let Some((_, votes)) = self + .delete_marker_candidates + .iter_mut() + .find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info)) + { + *votes = votes.saturating_add(1); + } else { + self.delete_marker_candidates.push((file_info.clone(), 1)); + } + self.delete_marker_votes = self + .delete_marker_candidates + .iter() + .map(|(_, votes)| *votes) + .max() + .unwrap_or_default(); + self.conflicting_metadata |= self.delete_marker_candidates.len() > 1; + return; + } + + match &self.candidate { + Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => { + self.candidate_votes = self.candidate_votes.saturating_add(1); + if let Some(disk_index) = disk_index + && let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index) + { + self.candidate_shard_mask |= bit; + } + } + Some(_) => { + self.conflicting_metadata = true; + } + None => { + self.candidate = Some(file_info.clone()); + self.candidate_votes = 1; + if let Some(disk_index) = disk_index + && let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index) + { + self.candidate_shard_mask |= bit; + } + } + } + } + + fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option { + let &erasure_index = candidate.erasure.distribution.get(disk_index)?; + if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index { + return None; + } + Some(1u16 << (erasure_index - 1)) + } + + pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool { + self.candidate_read_reserve_target() + .is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required) + } + + pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option { + let candidate = self.candidate.as_ref()?; + Some( + candidate + .erasure + .data_blocks + .saturating_add(usize::from(candidate.erasure.parity_blocks > 0)), + ) + } + + pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) { + match err { + DiskError::FileNotFound | DiskError::VolumeNotFound => { + self.not_found_responses = self.not_found_responses.saturating_add(1); + } + DiskError::FileVersionNotFound => { + self.version_not_found_responses = self.version_not_found_responses.saturating_add(1); + } + _ if is_metadata_fanout_ignored_error(err) => { + self.ignored_errors = self.ignored_errors.saturating_add(1); + } + _ => { + self.hard_errors = self.hard_errors.saturating_add(1); + } + } + } + + pub(in crate::set_disk) fn early_stop_decision(&self) -> Option { + if !self.allow_early_stop { + return None; + } + if self.delete_marker_votes >= self.default_write_quorum() { + return Some(MetadataEarlyStopDecision { + reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, + }); + } + if self.conflicting_metadata + || self.delete_marker_seen + || self.not_found_responses > 0 + || self.version_not_found_responses > 0 + || self.hard_errors > 0 + { + return None; + } + if self + .candidate + .as_ref() + .and_then(|candidate| self.candidate_latest_quorum(candidate)) + .is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum) + { + return Some(MetadataEarlyStopDecision { + reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, + }); + } + None + } + + /// Check if a versioned request can early-stop because the requested + /// version_id has reached quorum across disks. + pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option { + if !self.allow_early_stop { + return None; + } + if self.requested_version_id.is_empty() { + return None; + } + if self.conflicting_metadata + || self.delete_marker_seen + || self.not_found_responses > 0 + || self.version_not_found_responses > 0 + || self.hard_errors > 0 + { + return None; + } + if self.matching_version_votes >= self.read_quorum_for_version() { + return Some(MetadataEarlyStopDecision { + reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, + }); + } + None + } + + pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool { + if !self.allow_early_stop { + return false; + } + if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() { + return true; + } + if self.conflicting_metadata + || self.delete_marker_seen + || self.not_found_responses > 0 + || self.version_not_found_responses > 0 + || self.hard_errors > 0 + { + return false; + } + if !self.requested_version_id.is_empty() + && self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version() + { + return true; + } + match &self.candidate { + Some(candidate) => self + .candidate_latest_quorum(candidate) + .is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum), + None => pending >= self.default_write_quorum(), + } + } + + /// Compute the read quorum threshold for version-aware early-stop. + /// Uses `total_disks / 2` (like `missing_response_quorum`) when + /// `default_parity_count` is set, otherwise requires all disks. + pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize { + self.missing_response_quorum() + } + + pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str { + if !self.allow_early_stop { + return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST; + } + if self.conflicting_metadata { + return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA; + } + if self.delete_marker_seen { + return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER; + } + let missing_response_quorum = self.missing_response_quorum(); + if self.version_not_found_responses >= missing_response_quorum { + return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND; + } + if self.not_found_responses >= missing_response_quorum { + return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND; + } + if self.hard_errors > 0 { + return GET_METADATA_EARLY_STOP_REASON_ERROR; + } + if self.ignored_errors > 0 { + return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM; + } + GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM + } + + pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option { + if self.default_parity_count == 0 { + return Some(self.total_disks); + } + if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks { + return None; + } + let data_blocks = candidate.erasure.data_blocks; + Some(if data_blocks == candidate.erasure.parity_blocks { + data_blocks.saturating_add(1) + } else { + data_blocks + }) + } + + pub(crate) fn default_write_quorum(&self) -> usize { + if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { + return self.total_disks; + } + let data_blocks = self.total_disks.saturating_sub(self.default_parity_count); + if data_blocks == self.default_parity_count { + data_blocks.saturating_add(1) + } else { + data_blocks + } + } + + pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize { + if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { + self.total_disks + } else { + self.total_disks / 2 + } + } +} + +pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool { + left.volume == right.volume + && left.name == right.name + && left.version_id == right.version_id + && left.is_latest == right.is_latest + && left.deleted == right.deleted + && left.mark_deleted == right.mark_deleted + && left.transition_status == right.transition_status + && left.transitioned_objname == right.transitioned_objname + && left.transition_tier == right.transition_tier + && left.transition_version_id == right.transition_version_id + && left.transition_version == right.transition_version + && left.transition_version_state == right.transition_version_state + && left.expire_restored == right.expire_restored + && left.size == right.size + && left.mod_time == right.mod_time + && left.mode == right.mode + && left.written_by_version == right.written_by_version + && left.metadata == right.metadata + && left.replication_state_internal == right.replication_state_internal + && left.parts == right.parts + && left.checksum == right.checksum + && left.versioned == right.versioned + && left.num_versions == right.num_versions + && left.successor_mod_time == right.successor_mod_time + && left.data_dir == right.data_dir + && left.erasure.algorithm == right.erasure.algorithm + && left.erasure.data_blocks == right.erasure.data_blocks + && left.erasure.parity_blocks == right.erasure.parity_blocks + && left.erasure.block_size == right.erasure.block_size + && left.erasure.distribution == right.erasure.distribution +} + +pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool { + OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err) +} diff --git a/crates/ecstore/src/set_disk/core/mod.rs b/crates/ecstore/src/set_disk/core/mod.rs index 903487e64..b8507603e 100644 --- a/crates/ecstore/src/set_disk/core/mod.rs +++ b/crates/ecstore/src/set_disk/core/mod.rs @@ -18,3 +18,4 @@ //! duplicating read/write/erasure logic. pub(crate) mod io_primitives; +mod metadata_quorum; From 15e9bc5ed0f8558ed6bcdce559eeb1419c479753 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 15:54:56 +0800 Subject: [PATCH 14/40] test(ecstore): count injected faults across retry restarts (#7170) test(ecstore): count decommission faults across retry restarts From bbd7b9ef17b5b9bce2dd2134fcfc18510bb8c5f8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 5 Sep 2026 15:55:54 +0800 Subject: [PATCH 15/40] fix(site-replication): bound and order outage recovery (#7148) * fix(site-replication): wake retry drain after peer recovery * fix(site-replication): replay configure after bucket make * fix(site-replication): serialize retry replay state * fix(site-replication): persist destructive retry intents * fix(site-replication): bound retry recovery rounds * fix(site-replication): keep recovery replay live * fix(site-replication): preserve retry ordering * fix(site-replication): bound retry coordination * fix(site-replication): serialize topology replay * fix(site-replication): fence distributed retry state * fix(site-replication): bound outage retry drain * fix(site-replication): drop unsafe delete retry intents * fix(site-replication): order bucket mutation replay * fix(site-replication): harden outage retry replay * fix(site-replication): fence destructive peer delivery * fix(site-replication): avoid peer edit retry deadlock * fix(site-replication): fence retry error classification * fix(site-replication): classify connect timeouts * fix(site-replication): close recovery review races * test(site-replication): cover timeout endpoint text * fix(site-replication): close destructive recovery gaps * fix(site-replication): fence recovery revisions * fix(site-replication): replay bucket metadata on recovery * fix(site-replication): preserve s3gate boundary --------- Co-authored-by: overtrue --- .../src/replication_extension_test.rs | 93 ++ rustfs/src/admin/handlers/site_replication.rs | 662 +++++++++----- rustfs/src/app/bucket_usecase.rs | 100 ++- rustfs/src/site_replication/hooks.rs | 462 +++++++++- rustfs/src/site_replication/mod.rs | 13 +- rustfs/src/site_replication/repair.rs | 4 +- rustfs/src/site_replication/retry.rs | 849 +++++++++++++++--- rustfs/src/site_replication/state_lock.rs | 37 +- rustfs/src/site_replication/tests.rs | 609 ++++++++++++- rustfs/src/site_replication/transport.rs | 30 +- rustfs/src/site_replication_reconcile.rs | 48 +- rustfs/src/storage_api.rs | 4 +- 12 files changed, 2375 insertions(+), 536 deletions(-) diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index f5d308c7e..198fa5166 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -6743,6 +6743,99 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua Ok(()) } +#[tokio::test] +async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult { + init_logging(); + + // Keep compilation outside the scenario timeout. Recovery itself waits + // for the production 30-second lightweight retry tick. + let _rustfs_binary = rustfs_binary_path(); + + match timeout(Duration::from_secs(150), async { + let mut site_env = replication_fast_env(); + site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + + let mut site_a_env = RustFSTestEnvironment::new().await?; + site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?; + + let mut site_b_env = RustFSTestEnvironment::new().await?; + site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?; + + let site_a_client = site_a_env.create_s3_client(); + let site_b_client = site_b_env.create_s3_client(); + let bucket = "site-repl-peer-outage"; + let key = "after-recovery.txt"; + let payload = b"site replication recovered the missed bucket".to_vec(); + + let add_status = site_replication_add( + &site_a_env, + &[ + PeerSite { + name: "outage-site-a".to_string(), + endpoint: site_a_env.url.clone(), + access_key: site_a_env.access_key.clone(), + secret_key: site_a_env.secret_key.clone(), + ..Default::default() + }, + PeerSite { + name: "outage-site-b".to_string(), + endpoint: site_b_env.url.clone(), + access_key: site_b_env.access_key.clone(), + secret_key: site_b_env.secret_key.clone(), + ..Default::default() + }, + ], + ) + .await?; + assert!(add_status.success, "unexpected site add result: {add_status:?}"); + wait_for_site_replication_enabled(&site_a_env, 2).await?; + wait_for_site_replication_enabled(&site_b_env, 2).await?; + + site_b_env.stop_server(); + site_a_client.create_bucket().bucket(bucket).send().await?; + site_a_client.head_bucket().bucket(bucket).send().await?; + + let queued = site_replication_info(&site_a_env) + .await? + .retry_stats + .ok_or("peer outage did not persist a site replication retry event")?; + assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty"); + + site_b_env.restart_server_preserving_data(vec![], &site_env).await?; + let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75); + loop { + let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok(); + let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none(); + if bucket_recovered && queue_empty { + break; + } + if tokio::time::Instant::now() >= recovery_deadline { + return Err(format!( + "site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}" + ) + .into()); + } + sleep(Duration::from_millis(250)).await; + } + + site_a_client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from(payload.clone())) + .send() + .await?; + assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload); + + Ok(()) + }) + .await + { + Ok(result) => result, + Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()), + } +} + /// Re-applying a site's own replication config must not disable the peer's reverse direction. /// /// `PutBucketReplication` broadcasts the config to every peer — the console's replication diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index d75e3b850..51c7dda99 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -190,7 +190,8 @@ fn site_replicator_service_account_policy() -> S3Result { .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("parse site replicator policy failed: {e}"))) } -// Lock order: lifecycle -> bucket operation -> repair admission -> state -> per-bucket metadata. +// Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation +// -> bucket operation -> repair admission -> state -> per-bucket metadata. // "state" is the distributed state-object lock in // crate::site_replication::state_lock, entered through // update_site_replication_state (P1-15). There is no process-local state @@ -434,6 +435,7 @@ pub fn register_site_replication_route(r: &mut S3Router) -> std: // into this module: startup sits below this layer and must not depend upwards. The admin // router is built before startup reconciles, so the hook is always installed in time. crate::site_replication_reconcile::register_site_replication_reconciler(reconcile_site_replication_wiring); + crate::site_replication_reconcile::register_site_replication_retry_drainer(reconcile_site_replication_retry_drain); for (method, path, operation) in [ (Method::PUT, "/v3/site-replication/add", AdminOperation(&SiteReplicationAddHandler {})), @@ -1803,28 +1805,61 @@ async fn reconcile_site_replication_buckets() -> S3Result<()> { /// (`SiteReplicationEditHandler`), so a tick landing between them would rewrite the targets /// from the stale endpoint. The pending marker in the persisted state closes that window. /// Skipping costs nothing — the timer comes back. +async fn site_replication_reconcile_prerequisites_ready() -> bool { + if current_iam_handle().is_none() || current_object_store_handle().is_none() { + return false; + } + if let Err(err) = migrate_collapsed_retry_queue_paths().await { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_queue_migration_failed", + error = ?err, + "admin site replication state" + ); + return false; + } + true +} + +fn reconcile_site_replication_retry_drain() -> std::pin::Pin + Send>> { + Box::pin(async { + let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else { + return; + }; + if !site_replication_reconcile_prerequisites_ready().await { + return; + } + match load_site_replication_state().await { + Ok(state) => { + if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() || state.pending_remove.is_some() + { + return; + } + } + Err(_) => return, + } + // Admission above observes a lifecycle-stable state. The lightweight + // drain itself handles only idempotent bucket setup, reloads state + // under the distributed repair lock, and shares that lock with bucket + // deletion. Do not hold this process-local guard across peer I/O: an + // outage recovery must not make admin add/edit/remove time out. + drop(lifecycle); + drain_site_replication_retry_queue_lightweight().await; + }) +} + fn reconcile_site_replication_wiring() -> std::pin::Pin + Send>> { Box::pin(async { // The scheduler starts before IAM and the object store are guaranteed ready (IAM // bootstrap may still be recovering), so an early tick returns quietly instead of // logging a failure for every reconciler. - if current_iam_handle().is_none() || current_object_store_handle().is_none() { - return; - } - - let Some(_lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else { + let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else { return; }; - if let Err(err) = migrate_collapsed_retry_queue_paths().await { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - result = "retry_queue_migration_failed", - error = ?err, - "admin site replication state" - ); + if !site_replication_reconcile_prerequisites_ready().await { return; } @@ -1878,8 +1913,9 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin bool { let (origin, generation) = fence; if origin != local_deployment_id && state.peers.contains_key(origin) { @@ -4791,105 +4827,135 @@ async fn backfill_existing_buckets_after_add( let resync_id = Uuid::new_v4().to_string(); for bucket in &buckets { - let name = &bucket.name; + let operation_name = bucket.name.clone(); + let lock_bucket = operation_name.clone(); + let operation_state = state.clone(); + let operation_local_peer = local_peer.clone(); + let operation_resync_id = resync_id.clone(); + let operation_bootstrap_token = bootstrap_token.map(str::to_owned); + let bucket_errors = with_site_replication_bucket_mutation_lock(store.clone(), &lock_bucket, move || async move { + let mut errors = SiteReplicationErrorSummary::default(); + let name = &operation_name; - if let Err(err) = ensure_site_replication_bucket_versioning(name).await { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - bucket = %name, - result = "backfill_versioning_setup_failed", - error = ?err, - "admin site replication state" - ); - errors.push(format!("{name}: versioning setup failed: {err}")); - continue; - } - match ensure_site_replication_bucket_setup(name).await { - Ok(true) => {} - Ok(false) => { - // Runtime targets unavailable: the setup silently no-ops, which would make the - // downstream make-bucket broadcast and resync fail. Record it and skip so the - // operator sees this bucket was not propagated instead of an unqualified success. + if let Err(err) = ensure_site_replication_bucket_versioning(name).await { warn!( event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, bucket = %name, - result = "backfill_bucket_setup_skipped", - "admin site replication state" - ); - errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)")); - continue; - } - Err(err) => { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - bucket = %name, - result = "backfill_bucket_setup_failed", + result = "backfill_versioning_setup_failed", error = ?err, "admin site replication state" ); - errors.push(format!("{name}: bucket setup failed: {err}")); + errors.push(format!("{name}: versioning setup failed: {err}")); + return errors; } - } - // Broadcast the bucket to peers so they create it too (idempotent on the peer side). - // Read the real lock_enabled flag so peers recreate the bucket with the same object-lock - // setting — object lock cannot be added after bucket creation. - let lock_enabled = match metadata_sys::get(name).await { - Ok(bm) => bm.lock_enabled, - Err(err) => { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - bucket = %name, - result = "backfill_bucket_metadata_read_failed", - fallback = "lock_enabled=false", - error = ?err, - "admin site replication state" - ); - false + match ensure_site_replication_bucket_setup(name).await { + Ok(true) => {} + Ok(false) => { + // Runtime targets unavailable: the setup silently no-ops, which would make the + // downstream make-bucket broadcast and resync fail. Record it and skip so the + // operator sees this bucket was not propagated instead of an unqualified success. + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_bucket_setup_skipped", + "admin site replication state" + ); + errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)")); + return errors; + } + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_bucket_setup_failed", + error = ?err, + "admin site replication state" + ); + errors.push(format!("{name}: bucket setup failed: {err}")); + } } - }; - if let Err(err) = broadcast_site_replication_make_bucket(name, lock_enabled, None, bootstrap_token).await { - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - bucket = %name, - result = "backfill_make_bucket_broadcast_failed", - error = ?err, - "admin site replication state" - ); - errors.push(format!("{name}: make-bucket broadcast failed: {err}")); - } - // Kick a resync toward every remote peer so existing objects travel across. - for peer in state.peers.values() { - if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { - continue; - } - let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await; - let result = if manifest.target_arn.is_empty() { - manifest - } else { - start_site_bucket_resync(name, &manifest.target_arn, &resync_id).await + // Broadcast the bucket to peers so they create it too (idempotent on the peer side). + // Read the real lock_enabled flag so peers recreate the bucket with the same object-lock + // setting — object lock cannot be added after bucket creation. + let lock_enabled = match metadata_sys::get(name).await { + Ok(bm) => bm.lock_enabled, + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_bucket_metadata_read_failed", + fallback = "lock_enabled=false", + error = ?err, + "admin site replication state" + ); + false + } }; - if result.status == "failed" { + if let Err(err) = + broadcast_site_replication_make_bucket(name, lock_enabled, None, operation_bootstrap_token.as_deref()).await + { warn!( event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, bucket = %name, - peer = %peer.endpoint, - result = "backfill_resync_kick_failed", - detail = %result.err_detail, + result = "backfill_make_bucket_broadcast_failed", + error = ?err, "admin site replication state" ); - errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail)); + errors.push(format!("{name}: make-bucket broadcast failed: {err}")); + } + // Kick a resync toward every remote peer so existing objects travel across. + for peer in operation_state.peers.values() { + if peer.deployment_id == operation_local_peer.deployment_id + || same_identity_endpoint(&peer.endpoint, &operation_local_peer.endpoint) + { + continue; + } + let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await; + let result = if manifest.target_arn.is_empty() { + manifest + } else { + start_site_bucket_resync(name, &manifest.target_arn, &operation_resync_id).await + }; + if result.status == "failed" { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + peer = %peer.endpoint, + result = "backfill_resync_kick_failed", + detail = %result.err_detail, + "admin site replication state" + ); + errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail)); + } + } + errors + }) + .await; + match bucket_errors { + Ok(bucket_errors) => errors.extend(bucket_errors), + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %lock_bucket, + result = "backfill_bucket_mutation_lock_failed", + error = ?err, + "admin site replication state" + ); + errors.push(format!("{lock_bucket}: bucket mutation lock failed: {err}")); } } } @@ -6072,146 +6138,204 @@ fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result, present: &HashSet) -> S3Result<()> { + let mut missing = expected.difference(present).cloned().collect::>(); + if !missing.is_empty() { + missing.sort_unstable(); + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!( + "bucket `{}` disappeared while site replication was being added; peers may already be joined — re-run replicate add", + missing[0] + ), + )); + } + + let mut unexpected = present.difference(expected).cloned().collect::>(); + if !unexpected.is_empty() { + unexpected.sort_unstable(); + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!( + "bucket `{}` appeared while site replication was being added; peers may already be joined — re-run replicate add", + unexpected[0] + ), + )); + } + + Ok(()) +} + #[async_trait::async_trait] impl Operation for SiteReplicationAddHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; reject_site_replicator_on_public_admin(&cred)?; let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri); + let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers); let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?; - // Everything up to the commit below is preflight: peer probes, IAM - // work and the join fan-out all talk to the network, so none of it may - // run inside the state transaction. The snapshot read here is what the - // `updated_at` CAS in the commit validates. - let current_state = load_site_replication_state().await?; - if pending_endpoint_refresh(¤t_state).is_some() { - return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); - } - let local_peer = current_local_peer(&req, ¤t_state); let mut sites: Vec = read_site_replication_json(req, &cred.secret_key, true).await?; - // The web console's "Set Up Site Replication" omits the local deployment from the payload; - // inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`. - ensure_local_site_present(&mut sites, &local_peer); - validate_add_sites(&sites, &local_peer)?; - let preflight_infos = add_preflight_infos(&sites, ¤t_state, &local_peer).await?; - validate_add_preflight_topology(&preflight_infos, &local_peer)?; - let expected_updated_at = current_state.updated_at; - require_add_peer_tls_capability(&sites, &local_peer).await?; - // Early exit on a state that moved under the preflight probes, BEFORE - // the IAM write and the join fan-out change anything remote. Advisory - // only — the binding check is the CAS inside the commit — but it fences - // the common race off the side-effect path and refreshes the merge - // base so the CAS window is only the join round trips. - let latest_state = load_site_replication_state().await?; - ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?; - let current_state = latest_state; - let (service_account_access_key, service_account_secret_key) = - ensure_site_replicator_service_account(&cred.access_key, false).await?; - let bootstrap_buckets = preflight_infos - .iter() - .filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint)) - .flat_map(|info| info.buckets.keys().cloned()) - .collect(); - let add_in_progress_guard = SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets)?; - let mut state = merge_add_sites( - current_state, - local_peer.clone(), - sites.clone(), - service_account_access_key.clone(), - cred.access_key.clone(), - replicate_ilm_expiry, - ); - state.sync_state_initialized = true; - let join_req = SRPeerJoinEnvelope { - request: SRPeerJoinReq { - svc_acct_access_key: service_account_access_key, - svc_acct_secret_key: service_account_secret_key.clone(), - svc_acct_parent: String::new(), - peers: state.peers.clone(), - updated_at: state.updated_at, - }, - defer_sync_state_enable: true, - }; - let peer_join_path = - with_site_replication_bootstrap_token(SITE_REPLICATION_PEER_JOIN_PATH, &add_in_progress_guard.token.to_string()); + let admin_access_key = cred.access_key.clone(); + let admission_store = current_object_store_handle() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + let list_store = admission_store.clone(); + let (state, edit_generation, local_peer, service_account_secret_key, mut initial_sync_errors, _add_guard) = + with_site_replication_bucket_mutation_admission_lock(admission_store, move || async move { + // The writer starts before the local bucket snapshot and stays + // held through every peer join and the topology commit. A + // delete followed by a same-name create therefore cannot hide + // behind an unchanged final name set. Peer bootstrap callbacks + // use their internal path and do not acquire this public- + // mutation admission lock. + let current_state = load_site_replication_state().await?; + if pending_endpoint_refresh(¤t_state).is_some() { + return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); + } + let local_peer = local_peer_at_endpoint(local_endpoint, ¤t_state); + // The web console's "Set Up Site Replication" omits the local deployment from the payload; + // inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`. + ensure_local_site_present(&mut sites, &local_peer); + validate_add_sites(&sites, &local_peer)?; + let preflight_infos = add_preflight_infos(&sites, ¤t_state, &local_peer).await?; + validate_add_preflight_topology(&preflight_infos, &local_peer)?; + let expected_updated_at = current_state.updated_at; + require_add_peer_tls_capability(&sites, &local_peer).await?; + // Early exit on a state that moved under the preflight probes, BEFORE + // the IAM write and the join fan-out change anything remote. Advisory + // only — the binding check is the CAS inside the commit — but it fences + // the common race off the side-effect path and refreshes the merge + // base so the CAS window is only the join round trips. + let latest_state = load_site_replication_state().await?; + ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?; + let current_state = latest_state; + let (service_account_access_key, service_account_secret_key) = + ensure_site_replicator_service_account(&admin_access_key, false).await?; + let expected_buckets: HashSet = + preflight_infos.iter().flat_map(|info| info.buckets.keys().cloned()).collect(); + let bootstrap_buckets: HashSet = preflight_infos + .iter() + .filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint)) + .flat_map(|info| info.buckets.keys().cloned()) + .collect(); + let add_in_progress_guard = + SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets.clone())?; + let mut state = merge_add_sites( + current_state, + local_peer.clone(), + sites.clone(), + service_account_access_key.clone(), + admin_access_key, + replicate_ilm_expiry, + ); + state.sync_state_initialized = true; + let join_req = SRPeerJoinEnvelope { + request: SRPeerJoinReq { + svc_acct_access_key: service_account_access_key, + svc_acct_secret_key: service_account_secret_key.clone(), + svc_acct_parent: String::new(), + peers: state.peers.clone(), + updated_at: state.updated_at, + }, + defer_sync_state_enable: true, + }; + let peer_join_path = with_site_replication_bootstrap_token( + SITE_REPLICATION_PEER_JOIN_PATH, + &add_in_progress_guard.token.to_string(), + ); - let mut joined_endpoints = HashSet::new(); - let mut initial_sync_errors = SiteReplicationErrorSummary::default(); - for (site, preflight) in sites.iter().zip(preflight_infos.iter()) { - if same_identity_endpoint(&site.endpoint, &local_peer.endpoint) - || !joined_endpoints.insert(site_identity_key(&site.endpoint)) - { - continue; - } + let mut joined_endpoints = HashSet::new(); + let mut initial_sync_errors = SiteReplicationErrorSummary::default(); + for (site, preflight) in sites.iter().zip(preflight_infos.iter()) { + if same_identity_endpoint(&site.endpoint, &local_peer.endpoint) + || !joined_endpoints.insert(site_identity_key(&site.endpoint)) + { + continue; + } - let mut peer_join_req = join_req.clone(); - peer_join_req.request.svc_acct_parent = site.access_key.clone(); - let connection = PeerConnection::try_from(site)?; - let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key) - .send(&site.secret_key, &peer_join_req) + let mut peer_join_req = join_req.clone(); + peer_join_req.request.svc_acct_parent = site.access_key.clone(); + let connection = PeerConnection::try_from(site)?; + let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key) + .send(&site.secret_key, &peer_join_req) + .await?; + + let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint) + .unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry)); + fallback_peer.deployment_id = preflight.deployment_id.clone(); + let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("parse peer join response from {} failed: {e}", site.endpoint), + ) + })?; + if !join_response.initial_sync_error_message.is_empty() { + initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message)); + } + // An explicit no-op join. The peer answered 200 but wrote nothing — + // its persisted state is already newer than the snapshot it was + // sent — so the add is only PARTIALLY configured and saying + // "configured successfully" would be a lie (rustfs/rustfs#5963). + // `None` (a MinIO peer, or one older than the field) is not a + // no-op signal and is deliberately not reported. + if join_response.applied == Some(false) { + initial_sync_errors.push(format!( + "{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \ + the site is not configured against this peer", + site.endpoint + )); + } + state = reconcile_peer_with_actual_identity(state, join_response.peer); + let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("peer join response from {} did not identify the requested site", site.endpoint), + ) + })?; + validate_proposed_peer(&reconciled_peer).map_err(|err| { + S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("invalid peer join response from {}: {err}", site.endpoint), + ) + })?; + } + + mark_unknown_peer_sync_enabled(&mut state.peers); + + // Commit. The state transaction's CAS still fences topology + // writers that do not use bucket admission. By this point + // remote sites may already have accepted their joins, so a + // mismatch asks the operator to re-run add and reconverge. + let next_state = state; + let present = list_store + .list_bucket(&BucketOptions::default()) + .await + .map_err(ApiError::from)? + .into_iter() + .map(|bucket| bucket.name) + .collect::>(); + ensure_add_bucket_set_matches_preflight(&expected_buckets, &present)?; + let (state, edit_generation) = update_site_replication_state(move |state| { + if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() { + return Err(s3_error!( + InvalidRequest, + "site replication state changed during peer join; the peers may already be joined — re-run replicate add" + )); + } + adopt_add_commit_state(state, next_state); + let edit_generation = next_peer_edit_generation(state); + Ok((state.clone(), edit_generation)) + }) .await?; - - let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint) - .unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry)); - fallback_peer.deployment_id = preflight.deployment_id.clone(); - let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("parse peer join response from {} failed: {e}", site.endpoint), - ) - })?; - if !join_response.initial_sync_error_message.is_empty() { - initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message)); - } - // An explicit no-op join. The peer answered 200 but wrote nothing — - // its persisted state is already newer than the snapshot it was - // sent — so the add is only PARTIALLY configured and saying - // "configured successfully" would be a lie (rustfs/rustfs#5963). - // `None` (a MinIO peer, or one older than the field) is not a - // no-op signal and is deliberately not reported. - if join_response.applied == Some(false) { - initial_sync_errors.push(format!( - "{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \ - the site is not configured against this peer", - site.endpoint - )); - } - state = reconcile_peer_with_actual_identity(state, join_response.peer); - let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("peer join response from {} did not identify the requested site", site.endpoint), - ) - })?; - validate_proposed_peer(&reconciled_peer).map_err(|err| { - S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!("invalid peer join response from {}: {err}", site.endpoint), - ) - })?; - } - - mark_unknown_peer_sync_enabled(&mut state.peers); - - // Commit. The CAS runs inside the transaction, against the state the - // transaction itself loaded — the peer round trips above took however - // long they took, and only this check can tell whether the topology - // this add was planned against is still the current one. The error - // says so: by this point the remote sites already accepted their - // joins, and re-running the add is what reconverges the local side. - let next_state = state; - let (state, edit_generation) = update_site_replication_state(move |state| { - if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() { - return Err(s3_error!( - InvalidRequest, - "site replication state changed during peer join; the peers may already be joined — re-run replicate add" - )); - } - adopt_add_commit_state(state, next_state); - let edit_generation = next_peer_edit_generation(state); - Ok((state.clone(), edit_generation)) - }) - .await?; + Ok(( + state, + edit_generation, + local_peer, + service_account_secret_key, + initial_sync_errors, + add_in_progress_guard, + )) + }) + .await?; // The finalize fan-out delivers peer-edit payloads, so it carries the // generation allocated in the commit above: the receiving site orders @@ -7185,8 +7309,14 @@ impl Operation for SRPeerEditHandler { // The fence is self-reported — the shared service account means // the sender cannot be identified — so it is honoured only after // the admissibility check, against the same state it will gate. - let commit_fence = - commit_fence.filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence)); + let commit_fence = match commit_fence { + Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence), + // A fenced edit can only come from a current remote peer. If + // that origin left while the retry was in flight, applying + // its body here would resurrect the removed topology. + Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked)), + None => None, + }; // Ordering fence: the sending site allocates the generation under // its state-object lock, so a delivery that lost the race carries // a generation this site has already passed. Applying it would @@ -8886,6 +9016,41 @@ mod tests { ); } + #[test] + fn add_admission_starts_before_preflight_and_rejects_bucket_set_changes() { + let expected = HashSet::from(["remote-owned".to_string(), "shared".to_string()]); + let present = HashSet::from(["shared".to_string()]); + + let err = ensure_add_bucket_set_matches_preflight(&expected, &present) + .expect_err("a missing bootstrap bucket must reject the topology commit"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + let present = HashSet::from([ + "remote-owned".to_string(), + "shared".to_string(), + "created-during-add".to_string(), + ]); + let err = ensure_add_bucket_set_matches_preflight(&expected, &present) + .expect_err("a bucket created during add must reject the topology commit"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + let src = include_str!("site_replication.rs"); + let add = src + .split("impl Operation for SiteReplicationAddHandler") + .nth(1) + .and_then(|rest| rest.split("pub struct SiteReplicationRemoveHandler").next()) + .expect("add handler block"); + let admission = add + .find("with_site_replication_bucket_mutation_admission_lock") + .expect("distributed mutation admission"); + let preflight = add.find("add_preflight_infos").expect("bucket preflight"); + let validation = add + .find("ensure_add_bucket_set_matches_preflight") + .expect("bucket-set validation"); + let commit = add.find("adopt_add_commit_state").expect("topology commit"); + assert!(admission < preflight && preflight < validation && validation < commit); + } + #[test] fn test_tls_capability_gates_run_before_add_or_edit_state_side_effects() { let src = include_str!("site_replication.rs"); @@ -9182,13 +9347,19 @@ mod tests { ); // Fence hardening: origin and generation are self-reported by a // caller the shared service account cannot identify, so the handler - // must pass the fence through the admissibility check — against the - // same state the fence gates, i.e. inside the transaction — before - // reading or raising any high-water mark. + // must admit the fence against the same state it gates. An origin + // removed while a retry was in flight is acknowledged without + // applying the stale body; otherwise it could recreate topology. assert!( - handler_block.contains(".filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence))"), + handler_block.contains( + "Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence)" + ), "SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction" ); + assert!( + handler_block.contains("Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked))"), + "SRPeerEditHandler must not apply a fenced edit after its origin leaves the current topology" + ); // P1-15 PR2: both halves of the fence and the edit they fence share // ONE transaction. Checking the fence against a state read outside the // lock would let the check pass on one snapshot and the write land on @@ -10352,8 +10523,9 @@ mod tests { /// A fence is self-reported: every site authenticates peer traffic with /// the same site-replicator credential, so a compromised peer can stamp /// ANY origin with ANY generation. An origin the receiver does not - /// replicate with — or the receiver itself — is ignored and plants no - /// mark; a mark a compromised peer plants for a CURRENT origin cannot + /// replicate with — or the receiver itself — is inadmissible and plants + /// no mark; the handler acknowledges such a request without applying its + /// body. A mark a compromised peer plants for a CURRENT origin cannot /// silence that origin, because the staleness window refuses to fence on /// a mark implausibly far above the genuine deliveries. #[test] @@ -12791,6 +12963,7 @@ mod tests { last_error: "site replication is not enabled".to_string(), updated_at: Some(OffsetDateTime::now_utc()), edit_generation: None, + peer_unreachable: false, deletions_recorded: false, }], ..Default::default() @@ -12989,6 +13162,7 @@ mod tests { last_error: "peer offline".to_string(), updated_at: Some(OffsetDateTime::now_utc()), edit_generation: None, + peer_unreachable: false, deletions_recorded: false, }], ..Default::default() diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 7dada3492..e1951cd69 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -75,7 +75,8 @@ use crate::auth::get_condition_values_with_client_info; use crate::error::ApiError; use crate::shared_types::RemoteAddr; use crate::site_replication::{ - site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook, + cancel_site_replication_delete_bucket, commit_site_replication_delete_bucket, prepare_site_replication_delete_bucket, + site_replication_bucket_meta_hook, site_replication_make_bucket_hook, with_site_replication_bucket_mutation_lock, }; use crate::storage::storage_api::lock_bucket_targets_metadata; use http::StatusCode; @@ -1331,23 +1332,34 @@ impl DefaultBucketUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let make_result = store - .make_bucket( - &bucket, - &MakeBucketOptions { - force_create: false, - lock_enabled, - ..Default::default() - }, - ) - .await; + // Keep the local namespace mutation and its peer hook ordered across + // every node in this site. Otherwise a delete waiting for repair + // coordination can arrive after this create on remote sites. + let operation_bucket = bucket.clone(); + let operation_store = store.clone(); + let make_result = with_site_replication_bucket_mutation_lock(store, &bucket, move || async move { + let make_result = operation_store + .make_bucket( + &operation_bucket, + &MakeBucketOptions { + force_create: false, + lock_enabled, + ..Default::default() + }, + ) + .await; + if make_result.is_ok() { + crate::storage::invalidate_bucket_validation_cache(&operation_bucket); + if let Err(err) = site_replication_make_bucket_hook(&operation_bucket, lock_enabled).await { + warn!(bucket = %operation_bucket, error = ?err, "site replication make bucket hook failed"); + } + } + make_result + }) + .await?; match make_result { - Ok(()) => { - // Invalidate the bucket validation cache so subsequent GETs - // see the newly created bucket immediately. - crate::storage::invalidate_bucket_validation_cache(&bucket); - } + Ok(()) => {} Err(StorageError::BucketExists(_)) => { // Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK; // non-owner gets 409 BucketAlreadyExists. @@ -1358,10 +1370,6 @@ impl DefaultBucketUsecase { Err(e) => return Err(ApiError::from(e).into()), } - if let Err(err) = site_replication_make_bucket_hook(&bucket, lock_enabled).await { - warn!(bucket = %bucket, error = ?err, "site replication make bucket hook failed"); - } - let output = CreateBucketOutput::default(); counter!("rustfs_create_bucket_total").increment(1); let result = Ok(S3Response::new(output)); @@ -1397,16 +1405,41 @@ impl DefaultBucketUsecase { authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?; } - store - .delete_bucket( - &input.bucket, - &DeleteBucketOptions { - force, - ..Default::default() - }, - ) - .await - .map_err(ApiError::from)?; + // Keep the local namespace mutation and its peer hook ordered across + // every node in this site so an older delete cannot overtake a new + // same-name make while it waits for repair coordination. + let operation_bucket = input.bucket.clone(); + let operation_store = store.clone(); + with_site_replication_bucket_mutation_lock(store, &input.bucket, move || async move { + let intent = prepare_site_replication_delete_bucket(&operation_bucket, force).await?; + let delete_result = operation_store + .delete_bucket( + &operation_bucket, + &DeleteBucketOptions { + force, + ..Default::default() + }, + ) + .await; + match delete_result { + Ok(()) => { + crate::storage::invalidate_bucket_validation_cache(&operation_bucket); + if let Some(intent) = intent + && let Err(err) = commit_site_replication_delete_bucket(&intent).await + { + warn!(bucket = %operation_bucket, error = ?err, "site replication delete bucket hook failed"); + } + Ok::<(), S3Error>(()) + } + Err(err) => { + if let Some(intent) = intent { + cancel_site_replication_delete_bucket(intent).await; + } + Err(S3Error::from(ApiError::from(err))) + } + } + }) + .await??; // Drop every cached object body for the now-deleted bucket so dead // bytes do not sit resident until TTL. Covers both the normal and the @@ -1415,16 +1448,9 @@ impl DefaultBucketUsecase { let cache_adapter = current_object_data_cache_for_context(self.context.as_deref()); let _ = invalidate_object_data_cache_bucket_after_delete(&cache_adapter, &input.bucket).await; - // Invalidate bucket validation cache - crate::storage::invalidate_bucket_validation_cache(&input.bucket); - // Re-evaluate lifecycle and replication after bucket removal. rustfs_scanner::record_scanner_maintenance_change(&input.bucket); - if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await { - warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed"); - } - // Notify peers to drop their cached metadata for the now-deleted bucket. let request_context = req.extensions.get::().cloned(); notify_bucket_metadata_delete(input.bucket.clone(), request_context); diff --git a/rustfs/src/site_replication/hooks.rs b/rustfs/src/site_replication/hooks.rs index cd62704ee..95a830500 100644 --- a/rustfs/src/site_replication/hooks.rs +++ b/rustfs/src/site_replication/hooks.rs @@ -22,6 +22,57 @@ pub(crate) const SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION: &str = "confi pub(crate) static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock> = LazyLock::new(|| RwLock::new(())); +const SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX: &str = "config/site-replication/bucket-mutation"; +pub(crate) const SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH: &str = + "config/site-replication/bucket-mutation-admission.lock"; + +pub(crate) fn site_replication_bucket_mutation_lock_path(bucket: &str) -> String { + format!("{SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX}/{bucket}.lock") +} + +pub(crate) async fn with_site_replication_bucket_mutation_lock( + store: Arc, + bucket: &str, + operation: F, +) -> S3Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + let mutation_store = store.clone(); + let mutation_path = site_replication_bucket_mutation_lock_path(bucket); + with_config_object_read_lock( + store, + SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), + move || async move { + with_config_object_write_lock(mutation_store, mutation_path, operation) + .await + .map_err(|err| S3Error::from(ApiError::from(err))) + }, + ) + .await + .map_err(|err| S3Error::from(ApiError::from(err)))? +} + +/// Exclude every local bucket namespace mutation from an add's local preflight +/// snapshot until its topology commit. Peer bootstrap callbacks do not enter +/// this public-mutation admission path, so they can finish while the writer is +/// held; post-commit fan-out and backfill must run after it is released. +pub(crate) async fn with_site_replication_bucket_mutation_admission_lock( + store: Arc, + operation: F, +) -> S3Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, + T: Send + 'static, +{ + with_config_object_write_lock(store, SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), operation) + .await + .map_err(|err| S3Error::from(ApiError::from(err)))? +} + #[derive(Debug, Default)] pub(crate) struct SiteReplicationBootstrapPlan { pub(crate) iam_items: Vec, @@ -329,6 +380,91 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result S3Result { + let mut plan = SiteReplicationBootstrapPlan { + bucket_make_ops: vec![bootstrap_bucket_make_op_path(bucket)], + bucket_configure_ops: vec![bootstrap_bucket_op_path( + &bucket.bucket, + SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION, + )], + ..Default::default() + }; + append_bootstrap_bucket_items(&mut plan, bucket, replicate_ilm_expiry)?; + Ok(plan) +} + +pub(crate) fn site_replication_bucket_retry_plan_from_info( + bucket: &SRBucketInfo, + replicate_ilm_expiry: bool, +) -> S3Result { + let mut plan = site_replication_bucket_retry_plan_for(bucket, replicate_ilm_expiry)?; + // Omit only metadata the make/configure operations can reproduce exactly. + // Non-default versioning fields and operator-authored replication rules + // remain in the plan; their extra request cost intentionally defers the + // event to the complete drain when the lightweight budget is too small. + plan.bucket_items.retain(|item| !retry_bucket_metadata_is_redundant(item)); + Ok(plan) +} + +fn retry_bucket_metadata_is_redundant(item: &SRBucketMeta) -> bool { + match item.r#type.as_str() { + "version-config" => item.versioning.as_deref().is_some_and(|raw| { + deserialize::(&decode_bucket_meta_wire_value(raw)).is_ok_and(|config| { + config + == VersioningConfiguration { + status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)), + ..Default::default() + } + }) + }), + "replication-config" => item.replication_config.as_deref().is_some_and(|raw| { + deserialize::(&decode_bucket_meta_wire_value(raw)) + .is_ok_and(|config| config.role.trim().is_empty() && config.rules.iter().all(is_derived_site_replication_rule)) + }), + // `Some("")` is the in-memory sentinel used when the bucket is lock + // enabled but has no object-lock configuration body. The make query + // carries lockEnabled=true; sending an empty metadata body is neither + // useful nor parseable. + "object-lock-config" => item.object_lock_config.as_deref() == Some(""), + _ => false, + } +} + +pub(crate) async fn site_replication_bucket_retry_plan( + bucket: &str, + replicate_ilm_expiry: bool, +) -> S3Result { + let Some(store) = current_object_store_handle() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + let bucket_info = match store.get_bucket_info(bucket, &BucketOptions::default()).await { + Ok(bucket_info) => bucket_info, + Err(err) if is_err_bucket_not_found(&err) => return Ok(SiteReplicationBootstrapPlan::default()), + Err(err) => return Err(ApiError::from(err).into()), + }; + let lock_enabled = bucket_info.object_locking; + let metadata = metadata_sys::get(bucket).await.map_err(ApiError::from)?; + let mut bucket_info = SRBucketInfo { + bucket: bucket.to_string(), + created_at: bucket_info.created, + location: current_region().map(|region| region.to_string()).unwrap_or_default(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }; + populate_sr_bucket_info_from_metadata(&mut bucket_info, &metadata).await; + if lock_enabled && bucket_info.object_lock_config.is_none() { + bucket_info.object_lock_config = Some(String::new()); + } + site_replication_bucket_retry_plan_from_info(&bucket_info, replicate_ilm_expiry) +} + pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> { let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await; let runtime = { @@ -393,20 +529,273 @@ pub(crate) async fn broadcast_site_replication_make_bucket( broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await } -pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: bool) -> S3Result<()> { +const SITE_REPLICATION_DELETE_INTENT_PENDING: &str = + "bucket deletion reserved; local completion and peer delivery are not yet known"; + +#[derive(Clone)] +struct SiteReplicationDeleteBucketReservation { + peer: PeerInfo, + previous: Option, + observed: SiteReplicationRetryEvent, +} + +pub(crate) struct SiteReplicationDeleteBucketIntent { + path: String, + reservations: Vec, + displaced: Vec, +} + +fn site_replication_delete_bucket_path(bucket: &str, force_delete: bool) -> String { let operation = if force_delete { "force-delete-bucket" } else { "delete-bucket" }; - let path = format!( + format!( "/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", form_urlencoded::Serializer::new(String::new()) .append_pair("bucket", bucket) .append_pair("operation", operation) .finish() - ); - broadcast_site_replication_json(&path, &serde_json::json!({})).await + ) +} + +/// Reserve every destructive peer delivery before the local namespace is +/// changed. The state transaction either persists the complete set or writes +/// nothing, so a full/unreadable queue fails the S3 delete closed. +pub(crate) async fn prepare_site_replication_delete_bucket( + bucket: &str, + force_delete: bool, +) -> S3Result> { + let path = site_replication_delete_bucket_path(bucket, force_delete); + let reservation_path = path.clone(); + update_site_replication_state_when_changed(move |state| { + if !state.enabled() { + return Ok(StateCommit::Unchanged(None)); + } + let local_peer = current_local_runtime_peer(state); + let peers = state + .peers + .values() + .filter(|peer| { + peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) + }) + .cloned() + .collect::>(); + if peers.is_empty() { + return Ok(StateCommit::Unchanged(None)); + } + + let mut reservations = Vec::with_capacity(peers.len()); + let mut displaced = Vec::new(); + for peer in peers { + let previous = state + .retry_queue + .iter() + .find(|event| retry_event_matches(event, &peer, &reservation_path)) + .cloned(); + displaced.extend(upsert_site_replication_retry_event( + &mut state.retry_queue, + &peer, + &reservation_path, + SITE_REPLICATION_DELETE_INTENT_PENDING, + None, + )?); + let observed = state + .retry_queue + .iter() + .find(|event| retry_event_matches(event, &peer, &reservation_path)) + .cloned() + .ok_or_else(|| { + S3Error::with_message( + S3ErrorCode::InternalError, + "site replication delete reservation disappeared before commit".to_string(), + ) + })?; + reservations.push(SiteReplicationDeleteBucketReservation { + peer, + previous, + observed, + }); + } + Ok(StateCommit::Changed(Some(SiteReplicationDeleteBucketIntent { + path: reservation_path, + reservations, + displaced, + }))) + }) + .await +} + +/// Roll back a reservation when the local storage delete definitively failed. +/// A concurrently revised reservation is preserved; it belongs to a newer +/// observation and this operation has no authority to settle it. +pub(crate) async fn cancel_site_replication_delete_bucket(intent: SiteReplicationDeleteBucketIntent) { + let path = intent.path.clone(); + let result = update_site_replication_state_when_changed(move |state| { + let mut changed = false; + for reservation in intent.reservations { + let Some(index) = state.retry_queue.iter().position(|event| { + retry_event_matches(event, &reservation.peer, &reservation.observed.path) + && event.id == reservation.observed.id + && event.updated_at == reservation.observed.updated_at + }) else { + continue; + }; + if let Some(previous) = reservation.previous { + state.retry_queue[index] = previous; + } else { + state.retry_queue.remove(index); + } + changed = true; + } + + let mut restored_all = true; + for displaced in intent.displaced { + let duplicate = state.retry_queue.iter().any(|event| { + event.id == displaced.id + || (event.peer_deployment_id == displaced.peer_deployment_id && event.path == displaced.path) + }); + if duplicate { + continue; + } + if state.retry_queue.len() >= SITE_REPLICATION_RETRY_QUEUE_LIMIT { + restored_all = false; + continue; + } + state.retry_queue.push(displaced); + changed = true; + } + Ok(if changed { + StateCommit::Changed(restored_all) + } else { + StateCommit::Unchanged(restored_all) + }) + }) + .await; + + match result { + Ok(true) => {} + Ok(false) => warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + path, + result = "delete_intent_cancel_incomplete", + "admin site replication state" + ), + Err(err) => warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + path, + result = "delete_intent_cancel_failed", + error = ?err, + "admin site replication state" + ), + } +} + +async fn broadcast_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> { + let sends = intent.reservations.iter().cloned().map(|reservation| { + let request_path = intent.path.clone(); + async move { + let fallback_peer = reservation.peer.clone(); + let observed = reservation.observed.clone(); + let delivery_path = request_path.clone(); + let delivery = with_site_replication_state_read_lock(move |state| async move { + let Some(current_peer) = state.peers.get(&fallback_peer.deployment_id).cloned() else { + return Ok(None); + }; + let service_account_secret_key = + match site_replicator_service_account_secret(&state.service_account_access_key).await { + Ok(secret) => secret, + Err(err) => { + let Some(secret) = legacy_site_replicator_state_secret(&state) else { + return Err(err); + }; + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "legacy_state_service_account_secret_fallback", + error = ?err, + "admin site replication state" + ); + secret + } + }; + let result = async { + let transport = PeerTransport::for_runtime_peer(¤t_peer).await?; + PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key) + .with_client(&transport.client) + .send(&service_account_secret_key, &serde_json::json!({})) + .await + } + .await; + Ok(Some((current_peer, result))) + }) + .await; + match delivery { + Ok(Some((current_peer, Ok(_)))) => { + dequeue_observed_site_replication_retry_event(¤t_peer, &observed).await; + None + } + Ok(Some((current_peer, Err(err)))) => { + // Keep the failed deletion operator-visible, but never + // replay it automatically: without a bucket-incarnation + // fence, a delayed delete could erase a recreated bucket. + enqueue_site_replication_retry_event(¤t_peer, &request_path, &err).await; + Some(err) + } + Ok(None) => { + dequeue_observed_site_replication_retry_event(&reservation.peer, &observed).await; + None + } + Err(err) => { + enqueue_site_replication_retry_event(&reservation.peer, &request_path, &err).await; + Some(err) + } + } + } + }); + futures::future::join_all(sends) + .await + .into_iter() + .flatten() + .next() + .map_or(Ok(()), Err) +} + +pub(crate) async fn commit_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> { + let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await; + let store = + current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + let retry_peers = intent + .reservations + .iter() + .map(|reservation| reservation.peer.clone()) + .collect::>(); + let retry_path = intent.path.clone(); + let delivery_intent = SiteReplicationDeleteBucketIntent { + path: intent.path.clone(), + reservations: intent.reservations.clone(), + displaced: Vec::new(), + }; + match with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + broadcast_site_replication_delete_bucket(&delivery_intent).await + }) + .await + { + Ok(result) => result, + Err(err) => { + let err: S3Error = ApiError::from(err).into(); + for peer in &retry_peers { + enqueue_site_replication_retry_event(peer, &retry_path, &err).await; + } + Err(err) + } + } } pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> { @@ -515,6 +904,39 @@ pub(crate) fn maybe_time(value: OffsetDateTime) -> Option { (value != OffsetDateTime::UNIX_EPOCH).then_some(value) } +async fn populate_sr_bucket_info_from_metadata(entry: &mut SRBucketInfo, metadata: &BucketMetadata) { + entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok()); + entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml); + entry.tags = raw_config_to_base64(&metadata.tagging_config_xml); + entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml); + entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml); + entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml); + entry.quota_config = raw_config_to_base64(&metadata.quota_config_json); + // Expiry subset only: this entry feeds both the bootstrap/repair plan + // (peers must not receive transition rules) and cross-site consistency + // views (transition rules are site-local and would read as false + // mismatches). A deleted expiry state is a `None` value with the + // deletion's axis so repair can converge peers that missed the live + // delete. + let expiry_statement = lifecycle_expiry_statement(metadata); + entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone()); + entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml); + entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at); + entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at); + entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at); + entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at); + entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at); + entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at); + entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at); + // The expiry axis, not the whole-config write time: local transition-only + // edits inflate the latter, and a repair item stamped with it could + // out-rank a newer real expiry edit on a third site. + entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis); + entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at); + entry.replication_targets_online = + Some(site_replication_targets_online(&entry.bucket, &metadata.replication_config_xml).await); +} + pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result { let Some(store) = current_object_store_handle() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); @@ -546,37 +968,7 @@ pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &Pee }; if let Some(metadata) = metadata { - entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok()); - entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml); - entry.tags = raw_config_to_base64(&metadata.tagging_config_xml); - entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml); - entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml); - entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml); - entry.quota_config = raw_config_to_base64(&metadata.quota_config_json); - // Expiry subset only: this entry feeds both the bootstrap/repair - // plan (peers must not receive transition rules) and cross-site - // consistency views (transition rules are site-local and would - // read as false mismatches). A deleted expiry state is a `None` - // value with the deletion's axis so repair can converge peers - // that missed the live delete. - let expiry_statement = lifecycle_expiry_statement(&metadata); - entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone()); - entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml); - entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at); - entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at); - entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at); - entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at); - entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at); - entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at); - entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at); - // The expiry axis, not the whole-config write time: local - // transition-only edits inflate the latter, and a repair item - // stamped with it could out-rank a newer real expiry edit on a - // third site. - entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis); - entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at); - entry.replication_targets_online = - Some(site_replication_targets_online(&bucket.name, &metadata.replication_config_xml).await); + populate_sr_bucket_info_from_metadata(&mut entry, &metadata).await; } info.buckets.insert(bucket.name, entry); diff --git a/rustfs/src/site_replication/mod.rs b/rustfs/src/site_replication/mod.rs index 4c3ad07d9..630c50576 100644 --- a/rustfs/src/site_replication/mod.rs +++ b/rustfs/src/site_replication/mod.rs @@ -47,6 +47,7 @@ use self::identity::{ canonical_endpoint, deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with, same_identity_endpoint, }; +pub(crate) use self::state_lock::with_site_replication_state_read_lock; use self::state_lock::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock}; use crate::auth::constant_time_eq; use crate::config::get_config_snapshot; @@ -64,12 +65,12 @@ use crate::storage_api::site_replication::s3::{ #[cfg(test)] use crate::storage_api::site_replication::save_config as save_admin_config; use crate::storage_api::site_replication::{ - ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketTarget, - BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract, StorageError, - VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize, is_site_replication_role, - lock_bucket_targets_metadata, metadata_sys, read_config as read_admin_config, read_config_no_lock, - replication_target_arn_deployment_id, save_config_no_lock, serialize, site_replication_rule_deployment_id, - with_config_object_read_lock, with_config_object_write_lock, + ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketMetadata, BucketOperations, + BucketOptions, BucketTarget, BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract, + StorageError, VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize, + is_err_bucket_not_found, is_site_replication_role, lock_bucket_targets_metadata, metadata_sys, + read_config as read_admin_config, read_config_no_lock, replication_target_arn_deployment_id, save_config_no_lock, serialize, + site_replication_rule_deployment_id, with_config_object_read_lock, with_config_object_write_lock, }; use base64_simd::STANDARD as BASE64_STANDARD; use base64_simd::URL_SAFE_NO_PAD; diff --git a/rustfs/src/site_replication/repair.rs b/rustfs/src/site_replication/repair.rs index 5b3c260e3..b2785c0fd 100644 --- a/rustfs/src/site_replication/repair.rs +++ b/rustfs/src/site_replication/repair.rs @@ -649,7 +649,9 @@ pub(crate) async fn persist_site_replication_repair_task( let path = path.to_string(); update_site_replication_state(move |state| { match failure.as_deref() { - Some(error) => upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None), + Some(error) => { + upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None)?; + } None => { dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path); // A repair is the operator's accountability transfer for the diff --git a/rustfs/src/site_replication/retry.rs b/rustfs/src/site_replication/retry.rs index 818fbee40..4b818c752 100644 --- a/rustfs/src/site_replication/retry.rs +++ b/rustfs/src/site_replication/retry.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::*; +use futures::{StreamExt, stream}; pub(crate) const SITE_REPLICATION_RETRY_QUEUE_LIMIT: usize = 256; @@ -41,6 +42,12 @@ pub(crate) struct SiteReplicationRetryEvent { /// [`settle_site_replication_retry_events`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) edit_generation: Option, + /// The latest delivery failure happened before an authenticated peer + /// response was received (connect, DNS, or TLS). Such failures + /// may bypass the expensive replay backoff only after a cheap devnull + /// reachability probe proves the peer is back. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub(crate) peer_unreachable: bool, /// Whether every failure folded into this collapsed IAM entry had its /// deletion body (if it was a deletion) recorded in /// [`SiteReplicationState::iam_deletion_replays`]. Only then may a @@ -196,27 +203,69 @@ pub(crate) fn settle_site_replication_retry_events( before.saturating_sub(queue.len()) } +pub(crate) fn settle_observed_site_replication_retry_event( + queue: &mut Vec, + peer: &PeerInfo, + observed: &SiteReplicationRetryEvent, +) -> usize { + let before = queue.len(); + queue.retain(|current| { + !(retry_event_matches(current, peer, &observed.path) + && current.id == observed.id + && current.updated_at == observed.updated_at) + }); + before.saturating_sub(queue.len()) +} + pub(crate) fn upsert_site_replication_retry_event( queue: &mut Vec, peer: &PeerInfo, path: &str, error: &str, generation: Option, -) { +) -> S3Result> { let path = collapsed_retry_queue_path(path).unwrap_or(path); let now = OffsetDateTime::now_utc(); let detail = summarize_peer_error_detail(error); + let peer_unreachable = retry_error_indicates_peer_unreachable(error); if let Some(event) = queue.iter_mut().find(|event| retry_event_matches(event, peer, path)) { + // The id is the event revision used by probe promotion and replay + // settlement. Refresh it on every failure so an older in-flight + // success can never acknowledge the newer observation. + event.id = Uuid::new_v4().to_string(); event.retry_count = event.retry_count.saturating_add(1); event.failed = event.retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER; event.last_error = detail; event.updated_at = Some(now); + event.peer_unreachable = peer_unreachable; // Keep the newest generation: an older delivery that fails afterwards // must not lower the fence and let its own success settle the event. event.edit_generation = event.edit_generation.max(generation); - return; + return Ok(Vec::new()); } + let slots_needed = queue + .len() + .saturating_add(1) + .saturating_sub(SITE_REPLICATION_RETRY_QUEUE_LIMIT); + let mut evict_indices = queue + .iter() + .enumerate() + .filter_map(|(index, event)| retry_event_is_safely_replayable(event).then_some(index)) + .take(slots_needed) + .collect::>(); + if evict_indices.len() != slots_needed { + return Err(S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "site replication retry queue is full of non-evictable liabilities; repair them before recording more failures" + .to_string(), + )); + } + let mut evicted = Vec::with_capacity(evict_indices.len()); + while let Some(index) = evict_indices.pop() { + evicted.push(queue.remove(index)); + } + evicted.reverse(); queue.push(SiteReplicationRetryEvent { id: Uuid::new_v4().to_string(), peer_deployment_id: peer.deployment_id.clone(), @@ -227,12 +276,38 @@ pub(crate) fn upsert_site_replication_retry_event( last_error: detail, updated_at: Some(now), edit_generation: generation, + peer_unreachable, deletions_recorded: false, }); - if queue.len() > SITE_REPLICATION_RETRY_QUEUE_LIMIT { - let overflow = queue.len() - SITE_REPLICATION_RETRY_QUEUE_LIMIT; - queue.drain(0..overflow); + Ok(evicted) +} + +pub(crate) fn is_destructive_bucket_retry_path(path: &str) -> bool { + matches!( + retry_bucket_operation(path).as_deref(), + Some("delete-bucket" | "force-delete-bucket" | "purge-deleted-bucket") + ) +} + +fn retry_event_is_safely_replayable(event: &SiteReplicationRetryEvent) -> bool { + if is_destructive_bucket_retry_path(&event.path) { + return false; } + matches!( + classify_site_replication_retry_event(event), + Some(RetryDrainAction::PeerEdit | RetryDrainAction::BucketOpReplay { .. }) + ) +} + +pub(crate) fn retry_error_indicates_peer_unreachable(error: &str) -> bool { + let error = error.to_ascii_lowercase(); + let Some((_, request)) = error.split_once("peer request to ") else { + return false; + }; + let Some((_, failure)) = request.split_once(" failed ") else { + return false; + }; + failure.starts_with("(connect):") || failure.starts_with("(dns resolution):") || failure.starts_with("(tls handshake):") } pub(crate) fn retry_stats_for_state(state: &SiteReplicationState) -> Option { @@ -271,7 +346,7 @@ pub(crate) async fn enqueue_site_replication_retry_event_for_generation( // (remove_sites already pruned them); recording a late failure for it // would only pollute retry_stats until the queue cap evicts it. if state.peers.contains_key(&peer_owned.deployment_id) { - upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation); + upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation)?; } Ok(()) }) @@ -356,12 +431,17 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option { /// live in the same state so the caller commits them in one transaction — a /// retry entry can never exist whose deletion body was lost to a separate /// failed write. -pub(crate) fn record_failed_iam_delivery(state: &mut SiteReplicationState, peer: &PeerInfo, item: &SRIAMItem, error: &str) { +pub(crate) fn record_failed_iam_delivery( + state: &mut SiteReplicationState, + peer: &PeerInfo, + item: &SRIAMItem, + error: &str, +) -> S3Result<()> { let existed = state .retry_queue .iter() .any(|event| retry_event_matches(event, peer, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH)); - upsert_site_replication_retry_event(&mut state.retry_queue, peer, SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH, error, None); + upsert_site_replication_retry_event(&mut state.retry_queue, peer, SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH, error, None)?; if !existed && let Some(event) = state .retry_queue @@ -375,13 +455,13 @@ pub(crate) fn record_failed_iam_delivery(state: &mut SiteReplicationState, peer: } let Some(entity) = iam_item_deletion_entity(item) else { - return; + return Ok(()); }; let item_value = match serde_json::to_value(item) { Ok(value) => value, Err(_) => { degrade_iam_retry_event_to_escalation(state, peer); - return; + return Ok(()); } }; let now = OffsetDateTime::now_utc(); @@ -390,9 +470,13 @@ pub(crate) fn record_failed_iam_delivery(state: &mut SiteReplicationState, peer: .iter_mut() .find(|record| iam_deletion_replay_matches(record, peer) && record.entity == entity) { + // This id is the replay-record revision. A settlement that sent the + // previous body must not remove a same-entity deletion that failed + // while its snapshot was in flight. + existing.id = Uuid::new_v4().to_string(); existing.item = item_value; existing.recorded_at = Some(now); - return; + return Ok(()); } let per_peer = state @@ -424,6 +508,7 @@ pub(crate) fn record_failed_iam_delivery(state: &mut SiteReplicationState, peer: item: item_value, recorded_at: Some(now), }); + Ok(()) } pub(crate) fn degrade_iam_retry_event_to_escalation(state: &mut SiteReplicationState, peer: &PeerInfo) { @@ -467,7 +552,7 @@ pub(crate) async fn record_failed_site_replication_iam_delivery(peer: &PeerInfo, // A departed peer can never drain its entries again (remove_sites // already pruned them) — mirror enqueue_site_replication_retry_event. if state.peers.contains_key(&peer_owned.deployment_id) { - record_failed_iam_delivery(state, &peer_owned, &item_owned, &error_text); + record_failed_iam_delivery(state, &peer_owned, &item_owned, &error_text)?; } Ok(()) }) @@ -511,15 +596,14 @@ pub(crate) async fn record_failed_site_replication_iam_delivery(peer: &PeerInfo, pub(crate) fn settle_replayed_iam_retry_events( state: &mut SiteReplicationState, peer: &PeerInfo, - path: &str, - snapshot_updated_at: Option, + observed: &SiteReplicationRetryEvent, replayed_record_ids: &[String], ) -> bool { state .iam_deletion_replays .retain(|record| !(iam_deletion_replay_matches(record, peer) && replayed_record_ids.contains(&record.id))); - if collapsed_retry_queue_path(path) != Some(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH) { + if collapsed_retry_queue_path(&observed.path) != Some(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH) { return false; } let Some(index) = state @@ -530,11 +614,7 @@ pub(crate) fn settle_replayed_iam_retry_events( return false; }; let event = &state.retry_queue[index]; - let newer_failure_recorded = match (event.updated_at, snapshot_updated_at) { - (Some(current), Some(seen)) => current > seen, - (Some(_), None) => true, - (None, _) => false, - }; + let newer_failure_recorded = event.id != observed.id || event.updated_at != observed.updated_at; if newer_failure_recorded && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { // The newer failure's own deletion (if any) has its own record; the // next drain pass replays it. @@ -548,24 +628,22 @@ pub(crate) fn settle_replayed_iam_retry_events( state.retry_queue.remove(index); return true; } - escalate_site_replication_retry_events_up_to(&mut state.retry_queue, peer, path, snapshot_updated_at); + escalate_site_replication_retry_events_up_to(&mut state.retry_queue, peer, &observed.path, observed.updated_at); false } pub(crate) async fn settle_replayed_site_replication_iam_retry_event( peer: &PeerInfo, - path: &str, - snapshot_updated_at: Option, + observed: &SiteReplicationRetryEvent, replayed_record_ids: Vec, ) { let peer_owned = peer.clone(); - let path_owned = path.to_string(); + let observed_owned = observed.clone(); let result = update_site_replication_state(move |state| { Ok(settle_replayed_iam_retry_events( state, &peer_owned, - &path_owned, - snapshot_updated_at, + &observed_owned, &replayed_record_ids, )) }) @@ -591,7 +669,7 @@ pub(crate) async fn settle_replayed_site_replication_iam_retry_event( event = EVENT_ADMIN_SITE_REPLICATION_STATE, peer = %peer.endpoint, deployment_id = %peer.deployment_id, - path, + path = %observed.path, error = ?err, "failed to settle replayed site replication IAM retry event" ); @@ -652,8 +730,30 @@ pub(crate) const SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS: i64 = 600; /// converges at the next tick instead of waiting out this ceiling. pub(crate) const SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS: i64 = 86_400; -/// What the background drain may do for one retry event. Everything not -/// representable here is operator territory (manual repair). +/// Bound a background drain round to one complete bucket bootstrap chain: +/// make, at most nine metadata records, then replication configuration. +/// Larger snapshots and topology edits remain queued for operator repair. +pub(crate) const SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER: usize = 11; + +/// Bound sockets for every retry pass. The lightweight pass also admits at +/// most this many peer request chains per round, bounding its lock hold time. +pub(crate) const SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY: usize = 4; + +/// Rotate the bounded lightweight window instead of always admitting the +/// lexicographically first peers. Advancing by one full window per scheduler +/// round gives every queued peer a turn within `ceil(peer_count / limit)` +/// rounds, even when earlier peers each have a large bucket backlog. +pub(crate) fn lightweight_retry_peer_rotation(peer_count: usize, round: i64) -> usize { + if peer_count == 0 { + return 0; + } + let window = SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY.min(peer_count); + (round.rem_euclid(peer_count as i64) as usize * window) % peer_count +} + +/// A replay shape the retry machinery can derive from persisted state. The +/// lightweight 30-second scheduler admits only bounded bucket-op chains; +/// snapshot and topology-wide work remains operator territory. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum RetryDrainAction { /// Constant-path IAM item deliveries collapse into one queue entry per @@ -670,6 +770,10 @@ pub(crate) enum RetryDrainAction { PeerEdit, } +pub(crate) fn is_lightweight_retry_drain_action(action: &RetryDrainAction) -> bool { + matches!(action, RetryDrainAction::BucketOpReplay { .. }) +} + #[derive(Clone)] pub(crate) enum RetrySnapshot { Iam(Vec), @@ -724,27 +828,126 @@ impl RetrySnapshot { } } - pub(crate) async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result<()> { + pub(crate) async fn send( + &self, + peer: &PeerInfo, + transport: &PeerTransport, + access_key: &str, + secret_key: &str, + ) -> S3Result { match self { Self::Iam(items) => { for item in items { - SiteReplicationRepairTask::Iam(item) - .send(transport, access_key, secret_key) - .await?; + if !send_retry_task_if_peer_current( + peer, + &SiteReplicationRepairTask::Iam(item), + transport, + access_key, + secret_key, + ) + .await? + { + return Ok(false); + } } } Self::BucketMetadata(items) => { for item in items { - SiteReplicationRepairTask::BucketMetadata(item) - .send(transport, access_key, secret_key) - .await?; + if !send_retry_task_if_peer_current( + peer, + &SiteReplicationRepairTask::BucketMetadata(item), + transport, + access_key, + secret_key, + ) + .await? + { + return Ok(false); + } } } } - Ok(()) + Ok(true) } } +pub(crate) async fn send_retry_task_if_peer_current( + peer: &PeerInfo, + task: &SiteReplicationRepairTask<'_>, + transport: &PeerTransport, + access_key: &str, + secret_key: &str, +) -> S3Result { + let body = match task { + SiteReplicationRepairTask::Iam(item) => serde_json::to_value(item), + SiteReplicationRepairTask::BucketMetadata(item) => serde_json::to_value(item), + SiteReplicationRepairTask::BucketMake(_) | SiteReplicationRepairTask::Replication(_) => Ok(serde_json::json!({})), + } + .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry task failed: {err}")))?; + send_retry_request_if_peer_current(peer, transport, task.path(), access_key, secret_key, body).await +} + +pub(crate) async fn send_retry_request_if_peer_current( + peer: &PeerInfo, + transport: &PeerTransport, + path: &str, + access_key: &str, + secret_key: &str, + body: Value, +) -> S3Result { + let peer = peer.clone(); + let transport = transport.clone(); + let path = path.to_string(); + let access_key = access_key.to_string(); + let secret_key = secret_key.to_string(); + with_site_replication_state_read_lock(move |state| async move { + let current = state + .peers + .get(&peer.deployment_id) + .is_some_and(|current| same_identity_endpoint(¤t.endpoint, &peer.endpoint)); + if !current { + return Ok(false); + } + PeerAdminRequest::put(&transport.connection, &path, &access_key) + .with_client(&transport.client) + .send(&secret_key, &body) + .await?; + Ok(true) + }) + .await +} + +async fn send_peer_edit_retry_if_peer_current( + peer: &PeerInfo, + transport: &PeerTransport, + path: &str, + access_key: &str, + secret_key: &str, + body: Value, +) -> S3Result { + let peer_owned = peer.clone(); + let current = with_site_replication_state_read_lock(move |state| async move { + Ok(state + .peers + .get(&peer_owned.deployment_id) + .is_some_and(|current| same_identity_endpoint(¤t.endpoint, &peer_owned.endpoint))) + }) + .await?; + if !current { + return Ok(false); + } + // A peer-edit handler takes its own site's state write lock. Releasing + // this site's read lock before the request prevents simultaneous A -> B + // and B -> A retries from waiting on each other's write lock. The edit + // generation carried by `path` fences a delivery overtaken by a newer + // topology commit after this check. + PeerAdminRequest::put(&transport.connection, path, access_key) + .with_client(&transport.client) + .send(secret_key, &body) + .await?; + Ok(true) +} + #[derive(Hash, PartialEq, Eq)] pub(crate) enum IamSnapshotKey { Policy(String), @@ -872,6 +1075,66 @@ pub(crate) fn retry_bucket_name(path: &str) -> Option { .find_map(|(key, value)| (key == "bucket" && !value.is_empty()).then(|| value.into_owned())) } +pub(crate) fn bucket_op_retry_replay_tasks<'a>( + plan: &'a SiteReplicationBootstrapPlan, + operation: &str, + bucket: &str, +) -> S3Result>> { + let matches_bucket = |path: &&String| retry_bucket_name(path).as_deref() == Some(bucket); + match operation { + SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING => { + let mut tasks = plan + .bucket_make_ops + .iter() + .filter(matches_bucket) + .map(|path| SiteReplicationRepairTask::BucketMake(path.as_str())) + .collect::>(); + if tasks.is_empty() { + return Ok(tasks); + } + let configure_tasks = plan + .bucket_configure_ops + .iter() + .filter(matches_bucket) + .map(|path| SiteReplicationRepairTask::Replication(path.as_str())) + .collect::>(); + if configure_tasks.is_empty() { + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("site replication retry plan has no configure operation for bucket {bucket:?}"), + )); + } + tasks.extend( + plan.bucket_items + .iter() + .filter(|item| item.bucket == bucket) + .map(SiteReplicationRepairTask::BucketMetadata), + ); + tasks.extend(configure_tasks); + Ok(tasks) + } + SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION => Ok(plan + .bucket_configure_ops + .iter() + .filter(matches_bucket) + .map(|path| SiteReplicationRepairTask::Replication(path.as_str())) + .collect()), + _ => Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + format!("unsupported site replication retry bucket operation {operation:?}"), + )), + } +} + +pub(crate) fn retry_drain_request_count(action: &RetryDrainAction, plan: Option<&SiteReplicationBootstrapPlan>) -> usize { + match action { + RetryDrainAction::BucketOpReplay { operation, bucket } => plan + .and_then(|plan| bucket_op_retry_replay_tasks(plan, operation, bucket).ok()) + .map_or(0, |tasks| tasks.len()), + RetryDrainAction::IamSnapshot | RetryDrainAction::BucketMetadataSnapshot | RetryDrainAction::PeerEdit => usize::MAX, + } +} + /// A collapsed retry event after a stable snapshot resend is escalated with /// this marker instead of being cleared: the snapshot contains no task for a /// failed deletion, so remote absence remains operator-visible. Collapsed @@ -989,10 +1252,10 @@ pub(crate) fn actionable_site_replication_retry_events( /// backoff exists to spare a *dead* peer the expensive replay (plan build, /// snapshot resend) — it must not delay convergence to a peer that has /// already RECOVERED, or a failure window ends in up to a day of silent -/// divergence (backlog#2071). The drain probes each such peer with one cheap -/// request per tick and promotes its backlog when the probe succeeds. The -/// base backoff still floors individual re-attempts so a reachable peer that -/// keeps failing a delivery is not hammered faster than before. +/// divergence (backlog#2071). Peer connection failures may be probed before +/// the normal replay backoff elapses; request timeouts and application +/// failures still wait at least one base interval so a reachable peer that +/// keeps rejecting a replay is not hammered faster than before. pub(crate) fn deferred_site_replication_retry_events( state: &SiteReplicationState, now: OffsetDateTime, @@ -1005,7 +1268,14 @@ pub(crate) fn deferred_site_replication_retry_events( .filter(|event| !site_replication_retry_backoff_elapsed(event, now)) .filter(|event| { event.updated_at.is_none_or(|updated_at| { - now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) >= SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS + event.peer_unreachable + // Older binaries do not persist `peer_unreachable`. Parse + // only the locally-produced outer transport-error shape so + // rolling upgrades retain fast recovery without trusting + // an HTTP error body containing the same words. + || retry_error_indicates_peer_unreachable(&event.last_error) + || now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) + >= SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS }) }) .cloned() @@ -1036,11 +1306,31 @@ pub(crate) async fn probe_site_replication_peer_reachable(runtime: &SiteReplicat /// every peer that answers. A probe failure advances nothing: retry counts /// only move on real delivery attempts, so the per-event backoff is intact /// when the peer is genuinely down. +pub(crate) fn mark_reachable_deferred_retry_events( + state: &mut SiteReplicationState, + recovered: &[SiteReplicationRetryEvent], +) -> usize { + let mut promoted = 0; + for recovered in recovered { + if let Some(current) = state.retry_queue.iter_mut().find(|current| { + current.id == recovered.id + && current.peer_deployment_id == recovered.peer_deployment_id + && current.path == recovered.path + && current.updated_at == recovered.updated_at + }) { + current.updated_at = None; + current.peer_unreachable = false; + promoted += 1; + } + } + promoted +} + pub(crate) async fn promote_reachable_deferred_retry_events( runtime: &SiteReplicationRuntime, - actionable: &mut Vec, + actionable: &[SiteReplicationRetryEvent], deferred: Vec, -) { +) -> S3Result<()> { let due_peers: HashSet = actionable.iter().map(|event| event.peer_deployment_id.clone()).collect(); let mut deferred_by_peer: BTreeMap> = BTreeMap::new(); for event in deferred { @@ -1054,29 +1344,50 @@ pub(crate) async fn promote_reachable_deferred_retry_events( .or_default() .push(event); } - for (deployment_id, events) in deferred_by_peer { - let Some(peer) = runtime.state.peers.get(&deployment_id) else { - continue; - }; + let probes = deferred_by_peer.into_iter().filter_map(|(deployment_id, events)| { + let peer = runtime.state.peers.get(&deployment_id)?; if deployment_id == runtime.local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) { - continue; + return None; } - if probe_site_replication_peer_reachable(runtime, peer).await { + Some(async move { + let reachable = probe_site_replication_peer_reachable(runtime, peer).await; + (deployment_id, peer.endpoint.clone(), events, reachable) + }) + }); + let mut recovered = Vec::new(); + let probe_results = stream::iter(probes) + .buffer_unordered(SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY) + .collect::>() + .await; + for (deployment_id, peer_endpoint, events, reachable) in probe_results { + if reachable { info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - peer = %peer.endpoint, + result = "retry_backoff_probe_promoted", + peer = %peer_endpoint, deployment_id = %deployment_id, promoted = events.len(), - result = "retry_backoff_probe_promoted", - "peer reachable again; replaying its backed-off retry events this tick" + "peer reachable again; promoting backed-off retry events for replay" ); - actionable.extend(events); + recovered.extend(events); } } + if recovered.is_empty() { + return Ok(()); + } + update_site_replication_state_when_changed(move |state| { + let promoted = mark_reachable_deferred_retry_events(state, &recovered); + Ok(if promoted == 0 { + StateCommit::Unchanged(()) + } else { + StateCommit::Changed(()) + }) + }) + .await } /// Operator-visible per-tick alert for retry entries that no longer converge @@ -1142,6 +1453,78 @@ pub(crate) async fn drain_site_replication_retry_queue() { } } +/// Fast outage-recovery pass used by the 30-second scheduler. It limits work +/// to one bounded bucket-op chain per peer, builds no site-wide snapshot, and +/// runs reachability probes concurrently with replay for other peers. +pub(crate) async fn drain_site_replication_retry_queue_lightweight() { + if let Err(err) = drain_site_replication_retry_queue_lightweight_inner().await { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_failed", + error = ?err, + "admin site replication state" + ); + } +} + +async fn drain_site_replication_retry_queue_lightweight_inner() -> S3Result<()> { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + log_site_replication_retry_liabilities(&runtime.state); + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + return Ok(()); + } + let now = OffsetDateTime::now_utc(); + let mut actionable = actionable_site_replication_retry_events(&runtime.state, now); + let mut deferred = deferred_site_replication_retry_events(&runtime.state, now); + actionable.retain(|event| { + classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action)) + }); + deferred.retain(|event| { + classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action)) + }); + if actionable.is_empty() && deferred.is_empty() { + return Ok(()); + } + let Some(store) = current_object_store_handle() else { + return Ok(()); + }; + + // Probes are read-only and can consume the full request timeout. Keep + // them outside repair coordination; promotion is fenced by event id and + // timestamp, and the locked reload below decides what may actually send. + promote_reachable_deferred_retry_events(&runtime, &actionable, deferred).await?; + + with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + return Ok(()); + } + let now = OffsetDateTime::now_utc(); + let mut actionable = actionable_site_replication_retry_events(&runtime.state, now); + actionable.retain(|event| { + classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action)) + }); + if actionable.is_empty() { + return Ok(()); + } + drain_site_replication_retry_queue_lightweight_locked(Arc::new(runtime), actionable, now).await + }) + .await + .map_err(ApiError::from)? +} + pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { let Some(runtime) = runtime_site_replication_targets().await? else { return Ok(()); @@ -1150,7 +1533,7 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { // escalated markers are exactly the entries the drain skips. log_site_replication_retry_liabilities(&runtime.state); let now = OffsetDateTime::now_utc(); - let mut actionable = actionable_site_replication_retry_events(&runtime.state, now); + let actionable = actionable_site_replication_retry_events(&runtime.state, now); let deferred = deferred_site_replication_retry_events(&runtime.state, now); if actionable.is_empty() && deferred.is_empty() { return Ok(()); @@ -1167,27 +1550,146 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { // guard) may have started since. Re-check on the fresh state. return Ok(()); } - // Probe before taking the repair lock: probes are read-only peer traffic - // and a dead peer's connect timeout must not hold the lock. - promote_reachable_deferred_retry_events(&runtime, &mut actionable, deferred).await; - if actionable.is_empty() { - return Ok(()); - } - // Serialize against operator repair execution. This does NOT close the + + // Persist successful recovery probes without monopolizing repair + // coordination. The locked reload below observes those promotions and + // replays them in this same round. + promote_reachable_deferred_retry_events(&runtime, &actionable, deferred).await?; + + // Serialize against operator repair execution. Peer membership is + // re-checked from a distributed state snapshot immediately before each + // network request, so the caller need not hold the lifecycle guard while + // a large snapshot is replayed. This does NOT close the // dry-run -> execute window (dry-run takes no lock): a drain settling a // replayable bucket-op entry in that window changes the preflight token // and execute fails safe with "preflight is stale" — the operator - // re-runs the dry-run. Lock order matches repair: lifecycle guard (held - // by the reconcile tick) -> repair execution lock -> state object lock - // inside the send bookkeeping. An operator repair holding the lock makes - // this tick skip after the lock-acquire timeout. + // re-runs the dry-run. The lock elects one server to replay the queue; + // after acquiring it, reload state so a settled event or deleted bucket + // cannot be replayed from this admission snapshot. with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + // Runtime and queue snapshots captured before this distributed lock + // are only admission hints. Another node may have settled the event, + // or a local bucket may have been deleted, while this node waited. + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + return Ok(()); + } + let now = OffsetDateTime::now_utc(); + let actionable = actionable_site_replication_retry_events(&runtime.state, now); + if actionable.is_empty() { + return Ok(()); + } drain_site_replication_retry_queue_locked(runtime, actionable).await }) .await .map_err(ApiError::from)? } +async fn drain_site_replication_retry_queue_lightweight_locked( + runtime: Arc, + events: Vec, + now: OffsetDateTime, +) -> S3Result<()> { + let mut events_by_peer: BTreeMap> = BTreeMap::new(); + for event in events { + events_by_peer + .entry(event.peer_deployment_id.clone()) + .or_default() + .push(event); + } + + let mut peer_groups = events_by_peer + .into_iter() + .filter_map(|(deployment_id, peer_events)| { + let peer = runtime.state.peers.get(&deployment_id)?.clone(); + if deployment_id == runtime.local_peer.deployment_id + || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) + { + return None; + } + Some((peer, peer_events)) + }) + .collect::>(); + let interval_secs = crate::site_replication_reconcile::RETRY_DRAIN_INTERVAL.as_secs() as i64; + let round = now.unix_timestamp().div_euclid(interval_secs); + let rotation = lightweight_retry_peer_rotation(peer_groups.len(), round); + peer_groups.rotate_left(rotation); + + let peer_replays = peer_groups + .into_iter() + .map(|(peer, peer_events)| { + let runtime = Arc::clone(&runtime); + async move { + let Some((event, action, bucket)) = peer_events.into_iter().find_map(|event| { + let action = classify_site_replication_retry_event(&event)?; + let bucket = match &action { + RetryDrainAction::BucketOpReplay { bucket, .. } => bucket.clone(), + _ => return None, + }; + Some((event, action, bucket)) + }) else { + return (0, 0); + }; + let plan = match site_replication_bucket_retry_plan( + &bucket, + site_replication_state_replicates_ilm_expiry(&runtime.state), + ) + .await + { + Ok(plan) => plan, + Err(err) => { + enqueue_site_replication_retry_event(&peer, &event.path, &err).await; + return (0, 1); + } + }; + if retry_drain_request_count(&action, Some(&plan)) > SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER { + return (0, 0); + } + let transport = match PeerTransport::for_runtime_peer(&peer).await { + Ok(transport) => transport, + Err(err) => { + enqueue_site_replication_retry_event(&peer, &event.path, &err).await; + return (0, 1); + } + }; + match drain_one_site_replication_retry_event(&runtime, &peer, &transport, &event, action, Some(&plan)).await { + Ok(true) => (1, 0), + Ok(false) => (0, 0), + Err(_) => (0, 1), + } + } + }) + .take(SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY); + + let mut settled = 0usize; + let mut failures = 0usize; + let replay_results = stream::iter(peer_replays) + .buffer_unordered(SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY) + .collect::>() + .await; + for (peer_settled, peer_failures) in replay_results { + settled += peer_settled; + failures += peer_failures; + } + if settled > 0 || failures > 0 { + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_settled", + settled, + failures, + "admin site replication state" + ); + } + Ok(()) +} + pub(crate) async fn drain_site_replication_retry_queue_locked( runtime: SiteReplicationRuntime, events: Vec, @@ -1212,39 +1714,54 @@ pub(crate) async fn drain_site_replication_retry_queue_locked( .push(event); } - let mut settled = 0usize; - let mut failures = 0usize; - for (deployment_id, peer_events) in events_by_peer { - let Some(peer) = runtime.state.peers.get(&deployment_id) else { - continue; - }; + let runtime = Arc::new(runtime); + let plan = plan.map(Arc::new); + let peer_replays = events_by_peer.into_iter().filter_map(|(deployment_id, peer_events)| { + let peer = runtime.state.peers.get(&deployment_id)?.clone(); if deployment_id == runtime.local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) { - continue; + return None; } - let transport = match PeerTransport::for_runtime_peer(peer).await { - Ok(transport) => transport, - Err(err) => { - // Record the attempt so backoff advances for an unreachable - // peer instead of re-dialing it every tick. - for event in &peer_events { - enqueue_site_replication_retry_event(peer, &event.path, &err).await; + let runtime = Arc::clone(&runtime); + let plan = plan.as_ref().map(Arc::clone); + Some(async move { + let mut settled = 0usize; + let mut failures = 0usize; + let transport = match PeerTransport::for_runtime_peer(&peer).await { + Ok(transport) => transport, + Err(err) => { + // Record the attempt so backoff advances for an unreachable + // peer instead of re-dialing it every tick. + for event in &peer_events { + enqueue_site_replication_retry_event(&peer, &event.path, &err).await; + } + return (0, peer_events.len()); } - failures += peer_events.len(); - continue; - } - }; - for event in peer_events { - let Some(action) = classify_site_replication_retry_event(&event) else { - continue; }; - match drain_one_site_replication_retry_event(&runtime, peer, &transport, &event, action, plan.as_ref()).await { - Ok(true) => settled += 1, - Ok(false) => {} - Err(_) => failures += 1, + for event in peer_events { + let Some(action) = classify_site_replication_retry_event(&event) else { + continue; + }; + match drain_one_site_replication_retry_event(&runtime, &peer, &transport, &event, action, plan.as_deref()).await { + Ok(true) => settled += 1, + Ok(false) => {} + Err(_) => failures += 1, + } } - } + (settled, failures) + }) + }); + + let mut settled = 0usize; + let mut failures = 0usize; + let replay_results = stream::iter(peer_replays) + .buffer_unordered(SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY) + .collect::>() + .await; + for (peer_settled, peer_failures) in replay_results { + settled += peer_settled; + failures += peer_failures; } if settled > 0 || failures > 0 { @@ -1292,12 +1809,21 @@ pub(crate) async fn drain_one_site_replication_retry_event( drop_corrupt_iam_deletion_replay(peer, &record.id).await; continue; }; - if let Err(err) = SiteReplicationRepairTask::Iam(&item) - .send(transport, access_key, secret_key) - .await + match send_retry_task_if_peer_current( + peer, + &SiteReplicationRepairTask::Iam(&item), + transport, + access_key, + secret_key, + ) + .await { - enqueue_site_replication_retry_event(peer, &event.path, &err).await; - return Err(err); + Ok(true) => {} + Ok(false) => return Ok(false), + Err(err) => { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } } replayed_record_ids.push(record.id.clone()); } @@ -1306,22 +1832,20 @@ pub(crate) async fn drain_one_site_replication_retry_event( let mut replay = current_snapshot.clone(); for _ in 0..SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS { let current_fingerprint = current_snapshot.fingerprint()?; - if let Err(err) = replay.send(transport, access_key, secret_key).await { - enqueue_site_replication_retry_event(peer, &event.path, &err).await; - return Err(err); + match replay.send(peer, transport, access_key, secret_key).await { + Ok(true) => {} + Ok(false) => return Ok(false), + Err(err) => { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } } let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?; let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?; let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot"); if fresh_snapshot.fingerprint()? == current_fingerprint { if is_iam { - settle_replayed_site_replication_iam_retry_event( - peer, - &event.path, - event.updated_at, - replayed_record_ids, - ) - .await; + settle_replayed_site_replication_iam_retry_event(peer, event, replayed_record_ids).await; } else { escalate_site_replication_retry_event_up_to(peer, &event.path, event.updated_at).await; } @@ -1339,37 +1863,29 @@ pub(crate) async fn drain_one_site_replication_retry_event( // Replay from the CURRENT plan, never the recorded path: the // recorded query can carry an expired one-shot bootstrap token or // a stale createdAt. - let make_op = operation == SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING; - let paths = if make_op { - &plan.bucket_make_ops - } else { - &plan.bucket_configure_ops - }; - let tasks: Vec> = paths - .iter() - .filter(|path| retry_bucket_name(path).as_deref() == Some(bucket.as_str())) - .map(|path| { - if make_op { - SiteReplicationRepairTask::BucketMake(path) - } else { - SiteReplicationRepairTask::Replication(path) - } - }) - .collect(); - if tasks.is_empty() { - // The bucket left the plan (deleted, or replication no longer - // configured): the recorded intent is stale, settle it. - dequeue_site_replication_retry_event(peer, &event.path).await; - return Ok(true); - } - for task in &tasks { - if let Err(err) = task.send(transport, access_key, secret_key).await { + let tasks = match bucket_op_retry_replay_tasks(plan, &operation, &bucket) { + Ok(tasks) => tasks, + Err(err) => { enqueue_site_replication_retry_event(peer, &event.path, &err).await; return Err(err); } + }; + if tasks.is_empty() { + // The bucket left the plan (deleted, or replication no longer + // configured): the recorded intent is stale, settle it. + return Ok(dequeue_observed_site_replication_retry_event(peer, event).await); } - dequeue_site_replication_retry_event(peer, &event.path).await; - Ok(true) + for task in &tasks { + match send_retry_task_if_peer_current(peer, task, transport, access_key, secret_key).await { + Ok(true) => {} + Ok(false) => return Ok(false), + Err(err) => { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } + } + } + Ok(dequeue_observed_site_replication_retry_event(peer, event).await) } RetryDrainAction::PeerEdit => { // The recorded generation is stale by definition — the receiver @@ -1394,19 +1910,22 @@ pub(crate) async fn drain_one_site_replication_retry_event( let edit_path = peer_edit_path_with_fence(local_deployment_id, generation); let delivery_fence = local_deployment_id.is_some().then_some(generation); for body in &bodies { - if let Err(err) = PeerAdminRequest::put(&transport.connection, &edit_path, access_key) - .with_client(&transport.client) - .send(secret_key, body) - .await - { - enqueue_site_replication_retry_event_for_generation( - peer, - SITE_REPLICATION_PEER_EDIT_PATH, - &err, - delivery_fence, - ) - .await; - return Err(err); + let body = serde_json::to_value(body).map_err(|err| { + S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry peer edit failed: {err}")) + })?; + match send_peer_edit_retry_if_peer_current(peer, transport, &edit_path, access_key, secret_key, body).await { + Ok(true) => {} + Ok(false) => return Ok(false), + Err(err) => { + enqueue_site_replication_retry_event_for_generation( + peer, + SITE_REPLICATION_PEER_EDIT_PATH, + &err, + delivery_fence, + ) + .await; + return Err(err); + } } } dequeue_site_replication_retry_event_for_generation(peer, SITE_REPLICATION_PEER_EDIT_PATH, delivery_fence).await; @@ -1422,6 +1941,40 @@ pub(crate) async fn dequeue_site_replication_retry_event(peer: &PeerInfo, path: dequeue_site_replication_retry_event_for_generation(peer, path, None).await } +pub(crate) async fn dequeue_observed_site_replication_retry_event(peer: &PeerInfo, observed: &SiteReplicationRetryEvent) -> bool { + let result = async { + let mut probe = load_site_replication_state().await?; + if settle_observed_site_replication_retry_event(&mut probe.retry_queue, peer, observed) == 0 { + return Ok(false); + } + let peer_owned = peer.clone(); + let observed_owned = observed.clone(); + update_site_replication_state(move |state| { + Ok(settle_observed_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &observed_owned) > 0) + }) + .await + } + .await; + + match result { + Ok(settled) => settled, + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_event_dequeue_failed", + peer = %peer.endpoint, + deployment_id = %peer.deployment_id, + path = %observed.path, + error = ?err, + "failed to dequeue observed site replication retry event" + ); + false + } + } +} + pub(crate) async fn dequeue_site_replication_retry_event_for_generation(peer: &PeerInfo, path: &str, generation: Option) { let result = async { // Fast path: this sits on every successful hook broadcast, so probe diff --git a/rustfs/src/site_replication/state_lock.rs b/rustfs/src/site_replication/state_lock.rs index 6f9cc7bfd..9eb228882 100644 --- a/rustfs/src/site_replication/state_lock.rs +++ b/rustfs/src/site_replication/state_lock.rs @@ -28,14 +28,18 @@ //! process-local lock must never be reintroduced in front of it as if it //! added protection. All IO inside the closure must use the `*_no_lock` //! config helpers — the locked variants would self-deadlock on the same -//! object lock. Do not perform peer network calls or take other config locks -//! inside the closure. +//! object lock. Write-lock closures must not perform peer network calls or +//! take other config locks. A read-lock closure may carry bounded peer +//! delivery only when the receiver cannot write this state. Peer-edit +//! delivery must run after the read lock is released. //! -//! Lock order: lifecycle -> bucket operation -> repair admission -//! -> state object lock -> per-bucket metadata. +//! Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation +//! -> bucket operation -> repair admission -> state object lock -> +//! per-bucket metadata. A path may skip levels, but must not acquire an +//! earlier level while holding a later one. -use super::{S3Error, S3ErrorCode, S3Result}; -use crate::storage_api::site_replication::{ECStore, with_config_object_write_lock}; +use super::{S3Error, S3ErrorCode, S3Result, SiteReplicationState, load_site_replication_state_no_lock}; +use crate::storage_api::site_replication::{ECStore, with_config_object_read_lock, with_config_object_write_lock}; use std::sync::Arc; use crate::runtime_sources::current_object_store_handle; @@ -57,6 +61,27 @@ where with_site_replication_state_lock_on(store, operation).await } +/// Hold the distributed state-object read lock while `operation` validates a +/// topology snapshot. The closure may carry a bounded peer delivery only when +/// its receiver cannot write site replication state; peer-edit delivery must +/// run after this lock is released. Topology writers use the matching write +/// lock through [`with_site_replication_state_lock`]. +pub(crate) async fn with_site_replication_state_read_lock(operation: F) -> S3Result +where + T: Send + 'static, + F: FnOnce(SiteReplicationState) -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, +{ + let store = current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; + let read_store = store.clone(); + with_config_object_read_lock(store, SITE_REPLICATION_STATE_PATH.to_string(), move || async move { + let state = load_site_replication_state_no_lock(read_store).await?; + operation(state).await + }) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock site replication state failed: {e}")))? +} + /// Context-store variant for callers that resolve their store from an /// explicit [`AppContext`] (the service-side reload driven over node RPC). /// diff --git a/rustfs/src/site_replication/tests.rs b/rustfs/src/site_replication/tests.rs index e37869a43..e2ab27bb7 100644 --- a/rustfs/src/site_replication/tests.rs +++ b/rustfs/src/site_replication/tests.rs @@ -33,6 +33,18 @@ use temp_env::with_var; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; +#[test] +fn test_bucket_mutation_lock_path_is_bucket_scoped() { + assert_eq!( + site_replication_bucket_mutation_lock_path("photos"), + "config/site-replication/bucket-mutation/photos.lock" + ); + assert_ne!( + site_replication_bucket_mutation_lock_path("photos"), + site_replication_bucket_mutation_lock_path("videos") + ); +} + fn valid_test_ca_pem(name: &str) -> String { rcgen::generate_simple_self_signed(vec![name.to_string()]) .expect("generate test CA") @@ -381,6 +393,30 @@ async fn peer_clients_do_not_follow_redirects() { assert!(tls_server.await.expect("custom redirect TLS server task")); } +#[tokio::test] +async fn peer_http_error_body_cannot_spoof_an_unreachable_peer() { + let (endpoint, ca_pem, server) = spawn_test_tls_server_with_response( + b"HTTP/1.1 500 Internal Server Error\r\ncontent-length: 27\r\nconnection: close\r\n\r\ndownstream failed (connect)", + ) + .await; + let connection = validate_peer_connection_inner(&endpoint, false, &ca_pem, true).expect("custom CA peer connection"); + let client = + build_custom_site_replication_peer_client(&empty_outbound_tls_state(), &connection).expect("custom CA peer client"); + let err = PeerAdminRequest::post(&connection, SITE_REPLICATION_PEER_DEVNULL_PATH, "access-key") + .with_client(&client) + .send("secret-key", &serde_json::json!({})) + .await + .expect_err("HTTP 500 must fail"); + let detail = err.to_string(); + + assert!(detail.contains("downstream failed (connect)")); + assert!( + !retry_error_indicates_peer_unreachable(&detail), + "an untrusted response body must not enable the fast reachability probe" + ); + assert!(server.await.expect("HTTP error TLS server task")); +} + fn peer(name: &str, endpoint: &str) -> PeerInfo { PeerInfo { name: name.to_string(), @@ -419,6 +455,7 @@ fn drain_event(peer: &str, path: &str, retry_count: u32, updated_at: Option = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect(); - assert!(settle_replayed_iam_retry_events( - &mut state, - &target, - SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, - Some(snapshot_at), - &replayed, - )); + assert!(settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed)); assert!(state.retry_queue.is_empty()); assert!(state.iam_deletion_replays.is_empty()); // Not fully recorded: replayed records are still removed, but the entry // escalates instead of settling. let mut state = deletion_replay_state(&target); - record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline"); + record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure"); state.retry_queue[0].updated_at = Some(snapshot_at); state.retry_queue[0].deletions_recorded = false; + let observed = state.retry_queue[0].clone(); let replayed: Vec = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect(); - assert!(!settle_replayed_iam_retry_events( - &mut state, - &target, - SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, - Some(snapshot_at), - &replayed, - )); + assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed)); assert!(state.iam_deletion_replays.is_empty()); assert_eq!(state.retry_queue.len(), 1); assert_eq!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER); @@ -654,17 +683,13 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() { // Newer failure since the snapshot: entry untouched and drain-eligible, // residual (unreplayed) record kept for the next pass. let mut state = deletion_replay_state(&target); - record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline"); + record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure"); + state.retry_queue[0].updated_at = Some(snapshot_at); + let observed = state.retry_queue[0].clone(); let replayed: Vec = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect(); - record_failed_iam_delivery(&mut state, &target, &user_delete_item("bob"), "peer offline"); + record_failed_iam_delivery(&mut state, &target, &user_delete_item("bob"), "peer offline").expect("record failure"); state.retry_queue[0].updated_at = Some(snapshot_at + time::Duration::seconds(5)); - assert!(!settle_replayed_iam_retry_events( - &mut state, - &target, - SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, - Some(snapshot_at), - &replayed, - )); + assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed)); assert_eq!(state.retry_queue.len(), 1); assert_ne!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER); assert!( @@ -673,6 +698,24 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() { ); assert_eq!(state.iam_deletion_replays.len(), 1); assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:bob"); + + // A newer deletion of the same entity gets a fresh replay-record id. An + // older settlement therefore removes neither its body nor its queue + // revision, even if the persisted timestamps happen to be equal. + let mut state = deletion_replay_state(&target); + record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "first failure").expect("record failure"); + state.retry_queue[0].updated_at = Some(snapshot_at); + let observed = state.retry_queue[0].clone(); + let replayed = vec![state.iam_deletion_replays[0].id.clone()]; + record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "newer failure").expect("record newer failure"); + state.retry_queue[0].updated_at = Some(snapshot_at); + assert_ne!(state.iam_deletion_replays[0].id, replayed[0]); + + assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed)); + assert_eq!(state.retry_queue.len(), 1); + assert_ne!(state.retry_queue[0].id, observed.id); + assert_eq!(state.iam_deletion_replays.len(), 1); + assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:alice"); } /// Merging legacy wire-path rows into the collapsed entry must not launder an @@ -748,6 +791,281 @@ fn test_classify_site_replication_retry_event_actions() { assert_eq!(classify("/rustfs/admin/v3/site-replication/peer/unknown"), None); } +#[test] +fn test_bucket_make_retry_replays_matching_configure_before_settlement() { + let make_photos = + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(); + let configure_photos = + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication".to_string(); + let plan = SiteReplicationBootstrapPlan { + bucket_make_ops: vec![ + make_photos.clone(), + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=videos&operation=make-with-versioning".to_string(), + ], + bucket_configure_ops: vec![ + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=videos&operation=configure-replication".to_string(), + configure_photos.clone(), + ], + bucket_items: vec![ + SRBucketMeta { + bucket: "videos".to_string(), + r#type: "tags".to_string(), + ..Default::default() + }, + SRBucketMeta { + bucket: "photos".to_string(), + r#type: "policy".to_string(), + ..Default::default() + }, + ], + ..Default::default() + }; + + let tasks = bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos") + .expect("make retry plan should include its configure follow-up"); + assert_eq!( + tasks.iter().map(SiteReplicationRepairTask::path).collect::>(), + vec![ + make_photos.as_str(), + "/rustfs/admin/v3/site-replication/peer/bucket-meta", + configure_photos.as_str() + ] + ); + assert!(matches!(tasks[0], SiteReplicationRepairTask::BucketMake(_))); + assert!(matches!(&tasks[1], SiteReplicationRepairTask::BucketMetadata(item) if item.bucket == "photos")); + assert!(matches!(tasks[2], SiteReplicationRepairTask::Replication(_))); +} + +#[test] +fn test_bucket_make_retry_without_matching_configure_fails_closed() { + let plan = SiteReplicationBootstrapPlan { + bucket_make_ops: vec![ + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(), + ], + ..Default::default() + }; + + let err = match bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos") { + Ok(_) => panic!("make retry must not settle without a matching configure operation"), + Err(err) => err, + }; + assert_eq!(err.code(), &S3ErrorCode::InternalError); +} + +#[test] +fn test_retry_drain_bounds_each_peer_round_to_one_small_request_chain() { + let plan = SiteReplicationBootstrapPlan { + iam_items: vec![SRIAMItem::default(); 3], + bucket_make_ops: vec![ + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(), + ], + bucket_configure_ops: vec![ + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication".to_string(), + ], + bucket_items: vec![SRBucketMeta { + bucket: "photos".to_string(), + r#type: "tags".to_string(), + ..Default::default() + }], + ..Default::default() + }; + let make = RetryDrainAction::BucketOpReplay { + operation: SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING.to_string(), + bucket: "photos".to_string(), + }; + + assert!(is_lightweight_retry_drain_action(&make)); + assert!(!is_lightweight_retry_drain_action(&RetryDrainAction::IamSnapshot)); + assert!(!is_lightweight_retry_drain_action(&RetryDrainAction::PeerEdit)); + assert_eq!(retry_drain_request_count(&make, Some(&plan)), 3); + assert!(retry_drain_request_count(&make, Some(&plan)) <= SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER); + assert!( + retry_drain_request_count(&RetryDrainAction::IamSnapshot, Some(&plan)) + > SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER + ); + assert!( + retry_drain_request_count(&RetryDrainAction::PeerEdit, Some(&plan)) > SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER + ); +} + +#[test] +fn test_lightweight_retry_peer_rotation_covers_all_queued_peers() { + let limit = SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY; + for peer_count in 1..=(limit * 3 + 1) { + let rounds = peer_count.div_ceil(limit); + let mut seen = HashSet::new(); + for round in 7..(7 + rounds as i64) { + let start = lightweight_retry_peer_rotation(peer_count, round); + for offset in 0..limit.min(peer_count) { + seen.insert((start + offset) % peer_count); + } + } + assert_eq!( + seen.len(), + peer_count, + "every peer must enter the bounded lightweight window within {rounds} rounds" + ); + } +} + +#[test] +fn test_lightweight_bucket_retry_plan_is_targeted_and_preserves_make_options() { + let created_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let bucket = SRBucketInfo { + bucket: "photos".to_string(), + created_at: Some(created_at), + object_lock_config: Some(String::new()), + tags: Some("dGFncy14bWw=".to_string()), + tag_config_updated_at: Some(created_at), + ..Default::default() + }; + let plan = site_replication_bucket_retry_plan_for(&bucket, false).expect("targeted retry plan"); + + assert!(plan.iam_items.is_empty()); + assert_eq!(plan.bucket_make_ops.len(), 1); + assert!(plan.bucket_make_ops[0].contains("bucket=photos")); + assert!(plan.bucket_make_ops[0].contains("lockEnabled=true")); + assert!(plan.bucket_make_ops[0].contains("createdAt=")); + assert_eq!(plan.bucket_items.len(), 2); + assert_eq!(plan.bucket_items[0].r#type, "tags"); + assert_eq!(plan.bucket_items[1].r#type, "object-lock-config"); + assert_eq!(plan.bucket_configure_ops.len(), 1); + assert!(plan.bucket_configure_ops[0].contains("operation=configure-replication")); + + let tasks = + bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos").expect("retry task chain"); + assert!(matches!(tasks[0], SiteReplicationRepairTask::BucketMake(_))); + assert!(matches!(tasks[1], SiteReplicationRepairTask::BucketMetadata(_))); + assert!(matches!(tasks[2], SiteReplicationRepairTask::BucketMetadata(_))); + assert!(matches!(tasks[3], SiteReplicationRepairTask::Replication(_))); +} + +#[test] +fn test_lightweight_bucket_retry_plan_orders_real_metadata_and_counts_it() { + let versioning = bucket_versioning_xml().expect("canonical versioning config"); + let replication = serialize(&site_repl_config("remote-dep")).expect("derived replication config"); + let bucket = SRBucketInfo { + bucket: "photos".to_string(), + policy: Some(serde_json::json!({"Version":"2012-10-17","Statement":[]})), + tags: Some(BASE64_STANDARD.encode_to_string("")), + versioning: Some(BASE64_STANDARD.encode_to_string(&versioning)), + replication_config: Some(BASE64_STANDARD.encode_to_string(&replication)), + ..Default::default() + }; + let plan = site_replication_bucket_retry_plan_from_info(&bucket, false).expect("targeted retry plan"); + let tasks = bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos") + .expect("bucket replay tasks"); + + assert!(matches!(tasks.first(), Some(SiteReplicationRepairTask::BucketMake(_)))); + assert!(matches!(tasks.last(), Some(SiteReplicationRepairTask::Replication(_)))); + assert!( + tasks[1..tasks.len() - 1] + .iter() + .all(|task| matches!(task, SiteReplicationRepairTask::BucketMetadata(_))) + ); + assert_eq!(tasks.len(), 4, "make + policy + tags + configure must all count against the budget"); + assert!( + tasks.len() <= SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER, + "the complete metadata chain must fit the bounded lightweight replay" + ); + + let mut operator_replication = site_repl_config("remote-dep"); + operator_replication.rules.push(operator_rule("operator-backup")); + let mut bucket_with_operator_rule = bucket; + bucket_with_operator_rule.replication_config = + Some(BASE64_STANDARD.encode_to_string(&serialize(&operator_replication).expect("operator replication config"))); + let plan = site_replication_bucket_retry_plan_from_info(&bucket_with_operator_rule, false).expect("targeted retry plan"); + assert!( + plan.bucket_items.iter().any(|item| item.r#type == "replication-config"), + "operator-authored replication rules cannot be replaced by configure-replication" + ); +} + +#[test] +fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() { + let hooks = include_str!("hooks.rs"); + let delete_broadcast = hooks + .split("async fn broadcast_site_replication_delete_bucket") + .nth(1) + .and_then(|rest| rest.split("pub(crate) async fn commit_site_replication_delete_bucket").next()) + .expect("delete-bucket broadcast should exist"); + assert!( + delete_broadcast.contains("with_site_replication_state_read_lock(move |state| async move {") + && delete_broadcast.contains("state.peers.get(&fallback_peer.deployment_id)") + && delete_broadcast.contains("site_replicator_service_account_secret(&state.service_account_access_key)") + && delete_broadcast + .contains("PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key)"), + "a destructive bucket delivery must resolve current topology and credentials under the distributed state read lock" + ); + assert!( + delete_broadcast.contains("enqueue_site_replication_retry_event(¤t_peer, &request_path, &err).await"), + "a failed destructive delivery must remain visible for operator repair" + ); + + let usecase = include_str!("../app/bucket_usecase.rs"); + let delete = usecase + .split("async fn execute_delete_bucket_inner") + .nth(1) + .and_then(|rest| rest.split("pub async fn execute_head_bucket").next()) + .expect("delete bucket usecase"); + assert!( + delete + .find("prepare_site_replication_delete_bucket") + .expect("durable reservation") + < delete.find(".delete_bucket(").expect("local delete"), + "destructive peer liabilities must be persisted before the local bucket is deleted" + ); +} + +#[test] +fn test_bucket_retry_settlement_preserves_a_newer_same_path_failure() { + let peer = peer("remote", "https://remote.example.com"); + let path = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication"; + let observed_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let observed = drain_event("remote", path, 1, Some(observed_at)); + let mut queue = vec![observed.clone()]; + + queue[0].id = "evt-remote-new-revision".to_string(); + queue[0].retry_count += 1; + assert_eq!(settle_observed_site_replication_retry_event(&mut queue, &peer, &observed), 0); + assert_eq!(queue.len(), 1, "a newer same-timestamp failure must survive stale settlement"); + + let current = queue[0].clone(); + assert_eq!(settle_observed_site_replication_retry_event(&mut queue, &peer, ¤t), 1); + assert!(queue.is_empty()); +} + +#[test] +fn test_reachable_probe_promotion_is_fenced_by_the_observed_event() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let path = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning"; + let mut event = drain_event("remote", path, 3, Some(now)); + event.peer_unreachable = true; + let recovered = event.clone(); + let mut state = SiteReplicationState { + retry_queue: vec![event], + ..Default::default() + }; + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + + assert_eq!(mark_reachable_deferred_retry_events(&mut state, &[recovered.clone()]), 1); + assert_eq!(state.retry_queue[0].updated_at, None); + assert!(!state.retry_queue[0].peer_unreachable); + assert_eq!( + actionable_site_replication_retry_events(&state, now).len(), + 1, + "a successful probe must make the event replayable in the same drain tick" + ); + + state.retry_queue[0].updated_at = Some(now + time::Duration::seconds(1)); + state.retry_queue[0].peer_unreachable = true; + assert_eq!(mark_reachable_deferred_retry_events(&mut state, &[recovered]), 0); + assert_eq!(state.retry_queue[0].updated_at, Some(now + time::Duration::seconds(1))); + assert!(state.retry_queue[0].peer_unreachable); +} + #[test] fn test_retry_snapshot_fingerprint_detects_concurrent_iam_change() { let old = SRIAMItem { @@ -828,6 +1146,100 @@ fn test_site_replication_retry_backoff_schedule() { assert!(elapsed(30, 86_401)); } +#[test] +fn test_retry_error_marks_peer_unreachable_only_for_connection_failures() { + let mut queue = Vec::new(); + let peer = peer("remote", "https://remote.example.com"); + let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning"; + + upsert_site_replication_retry_event( + &mut queue, + &peer, + bucket_make, + "peer request to https://remote.example.com failed (connect): connection refused", + None, + ) + .expect("upsert retry event"); + assert!(queue[0].peer_unreachable); + + upsert_site_replication_retry_event( + &mut queue, + &peer, + bucket_make, + "peer request to https://remote.example.com failed (timeout): request exceeded 10 seconds", + None, + ) + .expect("upsert retry event"); + assert!( + !queue[0].peer_unreachable, + "a whole-request timeout does not prove the peer is unreachable" + ); + + upsert_site_replication_retry_event( + &mut queue, + &peer, + bucket_make, + "peer request to https://remote.example.com failed with 500 Internal Server Error: downstream failed (connect)", + None, + ) + .expect("upsert retry event"); + assert!( + !queue[0].peer_unreachable, + "application failures and their untrusted bodies must keep the normal replay backoff" + ); + + upsert_site_replication_retry_event( + &mut queue, + &peer, + bucket_make, + "peer request to https://remote.example.com failed with 500 Internal Server Error: backend failed (connect): spoofed", + None, + ) + .expect("upsert retry event"); + assert!(!queue[0].peer_unreachable, "peer response bodies must not spoof transport failures"); +} + +#[test] +fn test_connect_timeout_is_classified_as_a_connection_failure() { + assert_eq!(classify_peer_transport_error(true, true, "tcp connect timed out"), "connect"); + assert_eq!(classify_peer_transport_error(false, true, "request timed out"), "timeout"); + assert_eq!( + classify_peer_transport_error(false, true, "request timed out for https://tls-gateway.example"), + "timeout" + ); + assert_eq!(classify_peer_transport_error(true, false, "tls handshake failed"), "tls handshake"); +} + +#[test] +fn test_retry_event_peer_unreachable_is_legacy_serde_default() { + let json = r#"{ + "id":"evt-legacy", + "peer_deployment_id":"remote", + "peer_endpoint":"https://remote.example.com", + "path":"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning", + "retry_count":1, + "failed":false, + "last_error":"peer request to https://remote.example.com failed (connect): connection refused" + }"#; + + let mut event: SiteReplicationRetryEvent = serde_json::from_str(json).expect("legacy retry event decodes"); + assert!(!event.peer_unreachable); + + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + event.updated_at = Some(now - time::Duration::seconds(30)); + let mut state = SiteReplicationState::default(); + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + state.retry_queue.push(event); + + assert_eq!( + deferred_site_replication_retry_events(&state, now).len(), + 1, + "rolling-upgrade records must retain fast recovery from their trusted outer error shape" + ); +} + /// The actionable subset respects classification, peer membership and /// backoff; everything else stays untouched in the queue. #[test] @@ -915,6 +1327,51 @@ fn test_deferred_site_replication_retry_events_partition() { assert_eq!(actionable[0].path, "/rustfs/admin/v3/site-replication/peer/bucket-meta"); } +#[test] +fn test_deferred_retry_events_probe_fresh_peer_transport_failures() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let mut state = SiteReplicationState::default(); + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + + let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning"; + let mut fresh_transport_failure = drain_event("remote", bucket_make, 1, Some(now - time::Duration::seconds(30))); + fresh_transport_failure.peer_unreachable = true; + state.retry_queue.push(fresh_transport_failure); + + let deferred = deferred_site_replication_retry_events(&state, now); + assert_eq!( + deferred.len(), + 1, + "fresh transport failures must be eligible for a cheap reachability probe" + ); + assert_eq!(deferred[0].path, bucket_make); + + let actionable = actionable_site_replication_retry_events(&state, now); + assert!(actionable.is_empty(), "the event is still protected from direct replay by normal backoff"); +} + +#[test] +fn test_deferred_retry_events_do_not_probe_fresh_application_failures() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let mut state = SiteReplicationState::default(); + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + + let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning"; + state + .retry_queue + .push(drain_event("remote", bucket_make, 1, Some(now - time::Duration::seconds(30)))); + + assert!( + deferred_site_replication_retry_events(&state, now).is_empty(), + "reachable peers that reject an operation must keep the base replay backoff" + ); + assert!(actionable_site_replication_retry_events(&state, now).is_empty()); +} + /// The drain settles a peer-edit success under a freshly allocated /// generation; legacy queue entries carry `edit_generation: None` and /// must be cleared by that generation-scoped settlement (`(Some, None)` @@ -996,7 +1453,7 @@ fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() { // successful Bob update on the shared wire path cannot erase it even // before the drain runs. let mut queue = Vec::new(); - upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None); + upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None).expect("upsert retry event"); assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(dequeue_site_replication_retry_events(&mut queue, &target, path), 0); assert_eq!(queue.len(), 1); @@ -1005,7 +1462,7 @@ fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() { // A later hook failure overwrites the marker and re-arms the drain. let mut queue = vec![drain_event("remote", path, 2, Some(snapshot_at))]; escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)); - upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None); + upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None).expect("upsert retry event"); assert!(classify_site_replication_retry_event(&queue[0]).is_some()); // Legacy entry without a timestamp: escalated. @@ -1781,17 +2238,85 @@ fn test_retry_event_upsert_marks_repeated_failures() { }; let mut queue = Vec::new(); - upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "first", None); - upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "second", None); - upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None); + upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "first", None) + .expect("upsert retry event"); + let first_revision = queue[0].id.clone(); + upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "second", None) + .expect("upsert retry event"); + let second_revision = queue[0].id.clone(); + upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None) + .expect("upsert retry event"); assert_eq!(queue.len(), 1); + assert_ne!(first_revision, second_revision); + assert_ne!(second_revision, queue[0].id, "each failure must advance the settlement revision"); assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(queue[0].retry_count, SITE_REPLICATION_RETRY_FAILED_AFTER); assert!(queue[0].failed); assert_eq!(queue[0].last_error, "third"); } +#[test] +fn retry_queue_capacity_never_evicts_destructive_bucket_liabilities() { + let target = PeerInfo { + deployment_id: "remote-dep".to_string(), + ..peer("remote", "https://remote.example.com") + }; + let destructive = |index: usize| SiteReplicationRetryEvent { + id: format!("delete-{index}"), + peer_deployment_id: target.deployment_id.clone(), + peer_endpoint: target.endpoint.clone(), + path: format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?bucket=bucket-{index}&operation=delete-bucket"), + ..Default::default() + }; + let mut queue = (0..SITE_REPLICATION_RETRY_QUEUE_LIMIT).map(destructive).collect::>(); + let original_ids = queue.iter().map(|event| event.id.clone()).collect::>(); + let new_path = format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?bucket=overflow&operation=force-delete-bucket"); + + let err = upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None) + .expect_err("an all-destructive full queue must fail closed"); + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); + assert_eq!(queue.len(), SITE_REPLICATION_RETRY_QUEUE_LIMIT); + assert_eq!(queue.iter().map(|event| event.id.clone()).collect::>(), original_ids); + + queue[0] = SiteReplicationRetryEvent { + id: "iam-snapshot".to_string(), + peer_deployment_id: target.deployment_id.clone(), + peer_endpoint: target.endpoint.clone(), + path: SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH.to_string(), + deletions_recorded: true, + ..Default::default() + }; + upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None) + .expect_err("a collapsed IAM liability may contain a deletion and must not be evicted"); + assert!(queue.iter().any(|event| event.id == "iam-snapshot")); + + queue[0] = SiteReplicationRetryEvent { + id: "rebuildable".to_string(), + peer_deployment_id: target.deployment_id.clone(), + peer_endpoint: target.endpoint.clone(), + path: SITE_REPLICATION_PEER_EDIT_PATH.to_string(), + ..Default::default() + }; + let preserved_delete_ids = queue + .iter() + .filter(|event| is_destructive_bucket_retry_path(&event.path)) + .map(|event| event.id.clone()) + .collect::>(); + let evicted = upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None) + .expect("a rebuildable row may make room for a destructive liability"); + + assert_eq!(evicted.len(), 1); + assert_eq!(evicted[0].id, "rebuildable"); + assert_eq!(queue.len(), SITE_REPLICATION_RETRY_QUEUE_LIMIT); + assert!( + preserved_delete_ids + .iter() + .all(|id| queue.iter().any(|event| &event.id == id)) + ); + assert!(queue.iter().any(|event| event.path == new_path)); +} + /// P1-15 review follow-up: a successful peer-edit delivery only proves the /// peer reached the state THAT delivery carried. Settling it must not /// erase a retry event a newer edit left behind, or the local site sits on @@ -1807,7 +2332,8 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() { // Edit A (generation 5) delivered successfully and is stalled before // settling. Edit B (generation 6) commits meanwhile, fails delivery to // the same peer, and enqueues. - upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "peer offline", Some(6)); + upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "peer offline", Some(6)) + .expect("upsert retry event"); // A resumes: its own settlement must leave B's retry alone. assert_eq!( @@ -1818,7 +2344,8 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() { assert_eq!(queue[0].edit_generation, Some(6)); // An even older delivery failing afterwards must not lower the fence. - upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "still offline", Some(4)); + upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "still offline", Some(4)) + .expect("upsert retry event"); assert_eq!(queue[0].edit_generation, Some(6)); // B's own delivery succeeding is what clears it. @@ -1831,7 +2358,7 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() { // Collapsed broadcast failures live under an internal snapshot path; // an unrelated success on their shared wire path cannot settle them. let iam_path = "/rustfs/admin/v3/site-replication/peer/iam-item"; - upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None); + upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None).expect("upsert retry event"); assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 0); assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); } diff --git a/rustfs/src/site_replication/transport.rs b/rustfs/src/site_replication/transport.rs index 30ad28f95..bc941c9f0 100644 --- a/rustfs/src/site_replication/transport.rs +++ b/rustfs/src/site_replication/transport.rs @@ -331,6 +331,7 @@ pub(crate) fn runtime_peer_connection(peer: &PeerInfo) -> S3Result PeerAdminRequest<'a> { } let response = req.send().await.map_err(|e| { - let classify = if e.is_timeout() { - "timeout" - } else if e.is_connect() && e.to_string().to_ascii_lowercase().contains("dns") { - "dns resolution" - } else if e.to_string().to_ascii_lowercase().contains("certificate") - || e.to_string().to_ascii_lowercase().contains("tls") - { - "tls handshake" - } else if e.is_connect() { - "connect" - } else { - "request" - }; + let classify = classify_peer_transport_error(e.is_connect(), e.is_timeout(), &e.to_string()); S3Error::with_message(S3ErrorCode::InternalError, format!("peer request to {url} failed ({classify}): {e}")) })?; @@ -825,6 +814,21 @@ impl<'a> PeerAdminRequest<'a> { } } +pub(crate) fn classify_peer_transport_error(is_connect: bool, is_timeout: bool, detail: &str) -> &'static str { + let detail = detail.to_ascii_lowercase(); + if is_connect && detail.contains("dns") { + "dns resolution" + } else if is_connect && (detail.contains("certificate") || detail.contains("tls")) { + "tls handshake" + } else if is_connect { + "connect" + } else if is_timeout { + "timeout" + } else { + "request" + } +} + pub(crate) fn peer_error_may_be_secret_mismatch(detail: &str) -> bool { let detail = detail.to_ascii_lowercase(); detail.contains("signaturedoesnotmatch") diff --git a/rustfs/src/site_replication_reconcile.rs b/rustfs/src/site_replication_reconcile.rs index e327e3669..b4ea52c93 100644 --- a/rustfs/src/site_replication_reconcile.rs +++ b/rustfs/src/site_replication_reconcile.rs @@ -28,16 +28,19 @@ use std::pin::Pin; use std::sync::OnceLock; use std::time::Duration; +use tokio::time::Instant; use tokio_util::sync::CancellationToken; use tracing::warn; const RECONCILE_INTERVAL: Duration = Duration::from_secs(600); +pub(crate) const RETRY_DRAIN_INTERVAL: Duration = Duration::from_secs(30); /// A reconciler reports its own failures; the outcome carries no value because neither /// caller can act on one — a site that cannot repair its replication wiring still serves S3. type ReconcileHook = fn() -> Pin + Send>>; static RECONCILER: OnceLock = OnceLock::new(); +static RETRY_DRAINER: OnceLock = OnceLock::new(); /// Install the admin layer's reconciler. Idempotent: a second call is ignored, which keeps /// repeated router construction (tests, the embedded server) from panicking. @@ -45,6 +48,12 @@ pub(crate) fn register_site_replication_reconciler(reconcile: ReconcileHook) { let _ = RECONCILER.set(reconcile); } +/// Install the admin layer's lightweight retry drain. Idempotent for the same +/// reason as [`register_site_replication_reconciler`]. +pub(crate) fn register_site_replication_retry_drainer(drain: ReconcileHook) { + let _ = RETRY_DRAINER.set(drain); +} + /// Repair drifted site-replication wiring, immediately and then on a timer. /// /// The first pass runs inside the spawned task rather than on the caller's path: it walks @@ -62,16 +71,38 @@ pub(crate) fn spawn_site_replication_reconcile_task(ctx: CancellationToken) { return; } + spawn_reconcile_loop(ctx.clone(), RECONCILE_INTERVAL, &RECONCILER, true); + + if RETRY_DRAINER.get().is_none() { + warn!("site replication retry drainer is not registered; periodic retry drain disabled"); + return; + } + spawn_reconcile_loop(ctx, RETRY_DRAIN_INTERVAL, &RETRY_DRAINER, false); +} + +fn spawn_reconcile_loop( + ctx: CancellationToken, + interval: Duration, + hook: &'static OnceLock, + run_immediately: bool, +) { tokio::spawn(async move { - let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); + let first_tick = if run_immediately { + Instant::now() + } else { + Instant::now() + interval + }; + let mut ticker = tokio::time::interval_at(first_tick, interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tokio::select! { _ = ctx.cancelled() => break, - // The first tick fires immediately, which is the startup repair pass. + // The heavy reconciler owns the startup repair pass. The lightweight + // retry drain starts on its normal cadence so it cannot steal that + // first lifecycle lock and defer bucket/IAM repair for a full interval. _ = ticker.tick() => { - if let Some(reconcile) = RECONCILER.get() { + if let Some(reconcile) = hook.get() { reconcile().await; } } @@ -79,3 +110,14 @@ pub(crate) fn spawn_site_replication_reconcile_task(ctx: CancellationToken) { } }); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retry_drain_runs_faster_than_heavy_reconcile() { + assert!(RETRY_DRAIN_INTERVAL < RECONCILE_INTERVAL); + assert!(RETRY_DRAIN_INTERVAL <= Duration::from_secs(60)); + } +} diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index e74e88d02..db459b72d 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -244,8 +244,8 @@ pub(crate) mod site_replication { pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints}; pub(crate) use crate::storage::storage_api::{ - ECStore, EndpointServerPools, StorageError, delete_config_no_lock, lock_bucket_targets_metadata, read_config, - read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock, + ECStore, EndpointServerPools, StorageError, delete_config_no_lock, is_err_bucket_not_found, lock_bucket_targets_metadata, + read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock, }; pub(crate) mod metadata_sys { From 8ae8fb7eea00d92acc771f9efb25129925217055 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 16:16:53 +0800 Subject: [PATCH 16/40] fix(ecstore): drain control writes and preserve uncertain rollback (#7163) * fix(ecstore): drain durable control-plane write tails * fix(ecstore): retain PUT staging after incomplete rollback * fix(ecstore): drain backfill checkpoint before confirmation * fix(ecstore): retain per-disk rename rollback outcomes * fix(ecstore): retain indeterminate rename recovery evidence * test(ecstore): mark rollback fixtures as inline data * test(ecstore): match sealed context fixture map type * fix(ecstore): preserve known preflight rename rejections * test(ecstore): cover observed rename outer failures * test(ecstore): count decommission faults across retry restarts --- crates/ecstore/src/disk/local.rs | 997 +++++++++++++++++++++++++++++++ 1 file changed, 997 insertions(+) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 9cd683578..bf2239a9f 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -9862,7 +9862,1004 @@ fn should_read_legacy_inline_part(fi: &FileInfo, storage_class_config: &crate::c storage_class_config.should_inline(shard_size, fi.erasure.data_blocks, versioned) } +/// Proof produced only when the local rename returns at an existing access +/// preflight, before metadata, backups, or object data can be published. +#[derive(Debug)] +pub(in crate::disk) struct LocalRenamePreflightRejection(()); + impl LocalDisk { + #[tracing::instrument(name = "rename_data", level = "trace", skip_all)] + async fn rename_data_inner( + &self, + src_volume: &str, + src_path: &str, + fi: FileInfo, + dst_volume: &str, + dst_path: &str, + preflight_rejection: &mut Option, + ) -> Result { + crate::hp_guard!("LocalDisk::rename_data"); + let mut fi = fi; + // A non-force DeleteBucket must not remove a directory while a local + // object commit is publishing into it. The peer's empty scan remains + // optimistic; this lease establishes the local commit/delete order and + // remains owned by any blocking syscall that outlives async cancellation. + let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; + let quota_fence_token = + match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { + Some(value) => { + let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; + Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) + } + None if rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) => + { + return Err(DiskError::FileCorrupt); + } + None => None, + }; + rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); + let quota_fence_claim = match quota_fence_token { + Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), + None => None, + }; + let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + if let Some(claim) = quota_fence_claim { + mutation_lease.attach_external_guard(claim); + } + if fi.is_legacy_indexed_delete_marker() { + fi.erasure.index = 0; + } + fi.validate_for_metadata_read()?; + // Snapshot the destination part paths before `fi` is consumed below. These + // are the descriptors a reader may hold for the version this call is about + // to replace (backlog#1145); readers build the identical string in + // `io_primitives`. An inline-data version has no parts and yields none. + let invalidate_part_paths: Vec = { + let data_dir = fi.data_dir.unwrap_or_default(); + fi.parts + .iter() + .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) + .collect() + }; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; + if !skip_access_checks(src_volume) + && let Err(e) = super::fs::access_std(&src_volume_dir) + { + info!( + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?src_volume_dir, + operation = "rename_data_src_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; + if !skip_access_checks(dst_volume) + && let Err(e) = super::fs::access_std(&dst_volume_dir) + { + info!( + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?dst_volume_dir, + operation = "rename_data_dst_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + // xl.meta path + let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; + let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; + + // data_dir path + let has_data_dir_path = { + let has_data_dir = { + if !fi.is_remote() { + fi.data_dir + .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) + } else { + None + } + }; + + if let Some(data_dir) = has_data_dir { + let src_data_path = self.io_get_object_path( + src_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), + )?; + let dst_data_path = self.io_get_object_path( + dst_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), + )?; + + Some((src_data_path, dst_data_path)) + } else { + None + } + }; + + check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; + check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; + + let no_inline = fi.data.is_none() && fi.size > 0; + // Captured before `fi` is consumed by add_version; gates the stale + // destination purge below. + let fi_healing = fi.is_healing(); + + // Resolved once for the whole commit so a concurrent configuration + // change can never leave a single rename_data half-synced. The tier is + // keyed on the destination volume: user data staged in scratch + // namespaces follows the configured tier, while commits into + // system-critical namespaces (IAM, config, bucket metadata) stay + // pinned to strict. + let durability = effective_durability(dst_volume); + + let src_file_parent = src_file_path + .parent() + .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; + let dst_file_parent = dst_file_path + .parent() + .ok_or_else(|| DiskError::other("missing object metadata parent"))?; + if !no_inline { + fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; + } + // Acquire the common trees before reading destination metadata. On + // Windows this pins the object directory identity across metadata + // preparation, data publication, rollback backup, and final commit. + let rename_commit_guard = lock_rename_commit_directories( + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; + + if no_inline { + // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta + let mut xlmeta = FileMeta::new(); + // An existing dst xl.meta that fails to parse leaves `xlmeta` empty + // and gets overwritten by the commit below (pre-existing behavior); + // track that so the old-size observation reports unknown instead of + // a false `Absent` (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(dst_buf) = has_dst_buf.as_ref() { + if FileMeta::is_xl2_v1_format(dst_buf) + && let Ok(nmeta) = FileMeta::load(dst_buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let mut skip_parent = dst_volume_dir.clone(); + if has_dst_buf.as_ref().is_some() + && let Some(parent) = dst_file_path.parent() + { + skip_parent = parent.to_path_buf(); + } + + let version_id = fi.version_id.unwrap_or_default(); + let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = has_old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + if let Some(old_data_dir) = has_old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *old_data_dir); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_dst_buf = xlmeta.marshal_msg()?; + + // This tmp xl.meta is renamed onto dst_file_path at the commit + // point below, so only its contents must be durable before the + // rename (SyncMode::FileOnly); the dst parent directory is fsynced + // after the commit rename, and a crash before the rename means the + // PUT was never acknowledged. A metadata commit: relaxed tiers + // leave it to the page cache. + let tmp_meta_sync = if durability.syncs_commit_metadata() { + SyncMode::FileOnly + } else { + SyncMode::None + }; + // The tmp xl.meta write and the shard-file fdatasync are independent + // (disjoint paths) and both only need to be durable before the commit + // renames below, so run them concurrently to drop a blocking + // round-trip from the PUT commit critical path (rustfs/backlog#922 + // step 2). The "contents durable -> rename -> dst dir fsync" ordering + // is unchanged — both futures complete before any rename — which the + // rename_data crash-consistency harness (backlog#935) exercises. + // + // Shard durability: once rename_data succeeds the write is + // acknowledged, so data must not live only in the page cache. + // Multipart parts were already synced during rename_part, so their + // fdatasync here is a cheap no-op. A missing source dir is left for the + // rename below to report through the existing rollback path. Payload + // durability is kept by both strict and relaxed. + let tmp_meta_write = { + let src_file_path = src_file_path.clone(); + let dst_file_path = dst_file_path.clone(); + let rename_commit_guard = rename_commit_guard.clone(); + let mutation_lease = mutation_lease.clone(); + async move { + os::run_blocking_namespace_operation(mutation_lease, move || { + #[cfg(test)] + run_owned_file_write_before_open(&src_file_path); + let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( + &src_file_path, + &dst_file_path, + &rename_commit_guard, + )?; + prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; + Ok(prepared_metadata_source) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from) + } + }; + let shard_sync = async { + if durability.syncs_data_shards() + && let Some((src_data_path, _)) = has_data_dir_path.as_ref() + && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await + && err.kind() != ErrorKind::NotFound + { + return Err::<(), DiskError>(to_file_error(err).into()); + } + Ok(()) + }; + let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); + // Surface a tmp-meta failure first (its prior serial position), then a + // shard-sync failure; either aborts before any rename, exactly as the + // sequential version did. + let prepared_metadata_source = tmp_meta_res?; + shard_sync_res?; + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + std::fs::remove_file(&src_file_path).map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + // Heal reuses the version's data_dir, so for in-place corruption + // the destination dir still exists — and rename(2) cannot replace + // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge + // it first, healing commits only; fresh PUTs mint a new data_dir + // and never collide. Best effort: a real failure surfaces in the + // rename below. + if fi_healing + && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = self.move_to_trash(dst_data_path, true, false).await + { + warn!( + event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + dst_path = ?dst_data_path, + error = ?err, + "Healing commit could not purge the stale destination data dir" + ); + } + if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = os::rename_all_with_commit_guard( + src_data_path, + dst_data_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_data_path_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + #[cfg(test)] + if has_data_dir_path.is_some() { + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + } + + // Crash-consistency injection: hard power loss after the data dir + // is in place but before xl.meta commits. No cleanup — the harness + // reopens the disk and asserts the object still reads as the old + // version (the staged data dir is a harmless orphan for GC). + if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { + return Err(DiskError::Unexpected); + } + + if should_fail_before_old_metadata_backup(dst_path) { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "test_fail_before_old_metadata_backup", + "Disk local rename flow failed before metadata commit" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::Unexpected); + } + + // The rollback backup stays where it is written (no rename) and is + // the sole restore source for a later undo_write, so under strict + // it keeps SyncMode::FileAndDir: contents and directory entry both + // durable. It is part of the metadata commit machinery, so relaxed + // tiers leave it to the page cache like the xl.meta it mirrors. + let backup_sync = if durability.syncs_commit_metadata() { + SyncMode::FileAndDir + } else { + SyncMode::None + }; + if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { + let backup_parent = dst_file_parent.join(old_data_dir.to_string()); + #[cfg(not(windows))] + if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { + Ok(guard) => guard, + Err(err) => { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::from(to_file_error(err))); + } + }; + let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); + if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { + #[cfg(windows)] + drop(backup_path_guard); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_bytes = dst_buf.clone(); + // Keep the volume, commit-tree, and exact destination-path + // guards in this task until the backup write and durability + // sync finish. A detached spawn_blocking writer could survive + // cancellation and later truncate a newer transaction's + // deterministic rollback backup. + let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { + #[cfg(test)] + run_owned_file_write_before_open(&backup_path); + backup_path_guard.write_file_for_path_access( + &backup_path, + backup_bytes.as_ref(), + backup_sync != SyncMode::None, + backup_sync == SyncMode::FileAndDir, + ) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from); + if let Err(err) = write_result { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "write_old_metadata_backup_failed", + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + } + + // Crash-consistency injection: hard power loss after the rollback + // backup is durable but before the xl.meta commit rename. No + // cleanup — the harness asserts the object still reads as the old + // version, since the destination xl.meta is untouched here. + if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + if let Err(err) = os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_metadata_failed", + src_path = ?src_file_path, + dst_path = ?dst_file_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + + let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); + if should_fail_after_metadata_commit(dst_path) { + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + return Err(DiskError::Unexpected); + } + + // Crash-consistency injection: hard power loss immediately after the + // xl.meta commit rename but before the durability fsync. Unlike the + // graceful failpoint above, no rollback runs — the commit rename is + // already on disk, so the harness asserts the object reads back as + // the new version. + if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + // Persist the directory entries for both the data dir and xl.meta renames; + // without this the commit itself can vanish on power loss. Relaxed tiers + // accept that window (documented in docs/operations/durability-modes.md). + if durability.syncs_commit_metadata() + && let Some(parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // The commit rename changed the dst part inodes before this fsync + // failed and rolled them back; drop any fd cached during that + // window so readers re-open the restored inode (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // First PUT of an object creates its directory (and any missing prefix + // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The + // commit fsync above persists the object dir's *contents*, not its own + // entry in the bucket/prefix dir, so on power loss after ack the whole + // object dir could vanish (rustfs/backlog#922 step 4). For a new object + // (no prior xl.meta) fsync the ancestor chain from the object dir's + // parent up to and including the bucket so those new directory entries + // are durable. Overwrites already have a durable object dir. The + // starts_with guard bounds the walk to the bucket subtree. Relaxed/none + // accept the wider window, like the commit fsync above. + if has_dst_buf.is_none() && durability.syncs_commit_metadata() { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(dir) = ancestor { + if !dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dir(dir).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // Same post-commit rollback window as above — drop cached + // dst part fds so readers re-open the restored inode + // (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if dir == dst_volume_dir.as_path() { + break; + } + ancestor = dir.parent(); + } + } + + // Publication and every rollback-capable durability step are now + // complete. Do not retain the Windows object identity guard while + // cleaning staging paths or invalidating cached descriptors. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(src_file_path_parent) = src_file_path.parent() { + if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { + let _ = std::fs::remove_dir(src_file_path_parent); + } else { + let _ = self + .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) + .await; + } + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: has_old_data_dir, + rollback_data_dir, + cleanup_data_dir: has_old_data_dir, + sign: version_signature, + old_current_size, + }) + } else { + // Inline metadata preparation is blocking. The transaction lease is + // moved into that work so a timeout can release the async waiter without + // allowing a retry to reuse the deterministic staging path too early. + let src = src_file_path.clone(); + let dst = dst_file_path.clone(); + let cleanup_path = if src_volume == super::RUSTFS_META_MULTIPART_BUCKET { + src_file_path.parent().map(|p| p.to_path_buf()) + } else { + None + }; + let dst_path_for_failpoint = dst_path.to_string(); + #[cfg(windows)] + let source_parent = src_file_parent.to_path_buf(); + let rename_commit_guard_for_preparation = rename_commit_guard.clone(); + let sync = durability.syncs_commit_metadata(); + #[cfg(test)] + run_inline_before_file_sync_admission(dst_path); + let mut file_sync_admission = if sync { + Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ) + } else { + None + }; + let prepare_inline_metadata = move || { + let mut prepared_metadata_source = + os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; + #[cfg(windows)] + let source_metadata_guard = + rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; + let mut xlmeta = FileMeta::new(); + // Same as the non-inline branch: an unparsable existing dst + // xl.meta must surface as unknown, not `Absent` + // (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(ref buf) = has_dst_buf { + if FileMeta::is_xl2_v1_format(buf) + && let Ok(nmeta) = FileMeta::load(buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let version_id = fi.version_id.unwrap_or_default(); + let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + let mut staged_rollback_path = None; + if let Some(d) = old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *d); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_buf = xlmeta.marshal_msg()?; + // Write the staged xl.meta. Inline objects carry their data inside + // xl.meta, so this is the durable preparation for the metadata commit: + // relaxed tiers do no per-object fsync here at all (aligned + // with MinIO's default), trading a documented power-loss + // window for latency. + prepared_metadata_source.write_all(&new_buf, sync)?; + run_inline_preparation_before_backup(&dst_path_for_failpoint); + if let Some(ref old_metadata) = has_dst_buf + && (rollback_data_dir.is_some() || sync || cfg!(test)) + { + #[cfg(windows)] + let backup_path = { + let backup_path = src + .parent() + .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? + .join(STORAGE_FORMAT_FILE_BACKUP); + source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; + backup_path + }; + #[cfg(not(windows))] + let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; + #[cfg(not(windows))] + if sync { + std::fs::File::open(&backup_path)?.sync_data()?; + } + staged_rollback_path = Some(backup_path); + } + + Ok::<_, std::io::Error>(( + rollback_data_dir, + old_data_dir, + version_signature, + old_current_size, + staged_rollback_path, + has_dst_buf.is_none(), + prepared_metadata_source, + )) + }; + let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { + os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await + } else { + os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await + } + .map_err(to_file_error) + .map_err(DiskError::from); + + let ( + rollback_data_dir, + cleanup_data_dir, + version_signature, + old_current_size, + mut local_rollback_path, + destination_was_absent, + prepared_metadata_source, + ) = match inline_preparation { + Ok(prepared) => prepared, + Err(err) => { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + }; + + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + let remove_result = std::fs::remove_file(&src_file_path); + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + remove_result.map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { + let Some(dst_parent) = dst_file_path.parent() else { + return Err(DiskError::other("missing object metadata parent")); + }; + let backup_path = dst_parent + .join(rollback_data_dir.to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + // rename_all acquires the backup path's namespace lease. Do not + // hold a disk admission while acquiring another namespace lock. + drop(file_sync_admission.take()); + if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { + let _ = remove_file_if_exists(staged_backup); + return Err(err); + } + #[cfg(test)] + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + if sync { + file_sync_admission = Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ); + } + if let Some(admission) = file_sync_admission.as_ref() + && let Some(backup_parent) = backup_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + return Err(DiskError::from(to_file_error(err))); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + } + local_rollback_path = None; + } + + let commit_result = if should_fail_commit_rename(dst_path) { + Err(DiskError::other("test fail during metadata commit rename")) + } else { + os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &dst_volume_dir, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + }; + if let Err(err) = commit_result { + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + + let post_commit = async { + if should_fail_after_metadata_commit(dst_path) { + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(std::io::Error::other("test fail after metadata commit")); + } + + // Persist the commit rename's directory entry across power loss. + if let Some(admission) = file_sync_admission.as_ref() + && let Some(dst_parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) + .await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // Same power-loss gap as the non-inline path (rustfs/backlog#922 + // step 4): a first PUT creates the object dir (and any missing + // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all + // never fsynced. The fsync above persists the object dir's contents, + // not its own entry, so for a new inline object fsync the ancestor + // chain up to and including the bucket. Overwrites already have a + // durable object dir; the starts_with guard bounds the walk. + if let Some(admission) = file_sync_admission.as_ref() + && destination_was_absent + { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(ancestor_dir) = ancestor { + if !ancestor_dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std( + &dst_file_path, + rollback_data_dir, + local_rollback_path.as_deref(), + )?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if ancestor_dir == dst_volume_dir.as_path() { + break; + } + ancestor = ancestor_dir.parent(); + } + } + + Ok::<(), std::io::Error>(()) + } + .await; + + // The disk admission protects the durability chain, not staging + // cleanup or cache invalidation after that chain has completed. + drop(file_sync_admission.take()); + + // A post-commit rollback (for example, a commit-metadata fsync + // failure under strict durability) restores the old metadata; drop any + // descriptors cached during the committed window before propagating the + // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so + // this is mostly defensive and keeps both commit branches consistent. + if let Err(err) = post_commit { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(DiskError::from(err)); + } + + // The commit no longer has a rollback path. Release the Windows + // object identity guard before best-effort staging cleanup. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + + // Cleanup + if let Some(ref cleanup) = cleanup_path { + let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; + } else if let Some(parent) = src_file_path.parent() { + let _ = std::fs::remove_dir(parent); + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: cleanup_data_dir, + rollback_data_dir, + cleanup_data_dir, + sign: version_signature, + old_current_size, + }) + } + } + + pub(in crate::disk) async fn rename_data_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> super::RenameDataObservation { + let mut preflight_rejection = None; + let result = self + .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection) + .await; + super::RenameDataObservation { + result, + preflight_rejection, + } + } + pub(crate) async fn rename_data_borrowed( &self, src_volume: &str, From 2f02d1d2d8bd1026cc43dfd155586c894e7b266d Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 16:26:13 +0800 Subject: [PATCH 17/40] fix(ci): preserve security suite failures and isolate reports (#7188) --- .config/make/tests.mak | 1 + .github/workflows/ci-docs-only.yml | 1 + .github/workflows/ci.yml | 1 + .github/workflows/rustfs-security-test.yml | 83 ++++++--- scripts/test_security_workflow.py | 197 +++++++++++++++++++++ 5 files changed, 255 insertions(+), 28 deletions(-) create mode 100644 scripts/test_security_workflow.py diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 3ee3337fa..d1297e9ad 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -40,6 +40,7 @@ script-tests: ## Run shell script tests $(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test + $(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh $(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index de78e7f7f..7c6ad22ec 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -129,6 +129,7 @@ jobs: run: | python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/test_security_workflow.py python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98fc6cc30..171f52380 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,7 @@ jobs: run: | python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/test_security_workflow.py python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml index 2b2a5d68b..16e10a80c 100644 --- a/.github/workflows/rustfs-security-test.yml +++ b/.github/workflows/rustfs-security-test.yml @@ -74,10 +74,23 @@ env: jobs: security-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for the OIDC live gate script) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize security evidence + id: evidence + run: | + set -euo pipefail + umask 077 + SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${SECURITY_ARTIFACTS_DIR}" + printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -98,11 +111,6 @@ jobs: echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2 exit 1 - - name: Checkout repository (for the OIDC live gate script) - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - name: Show environment run: | uname -a @@ -135,7 +143,8 @@ jobs: id: test continue-on-error: true env: - REPORT_FILE: /tmp/rustfs-security-report.md + REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md + TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }} RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh run: | set -euo pipefail @@ -159,29 +168,48 @@ jobs: else ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") fi - ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" + GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" - name: Generate report - if: always() + id: report + if: ${{ always() && steps.evidence.outcome == 'success' }} + env: + TEST_OUTCOME: ${{ steps.test.outcome }} run: | set -euo pipefail - if [ ! -f /tmp/rustfs-security-report.md ]; then - { - echo "# RustFS security test report" - echo "" - echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - echo "- Trigger: ${{ github.event_name }}" - echo "- Test Step Outcome: failure (suite did not produce a report)" - } > /tmp/rustfs-security-report.md + RESULT=failure + if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then + RESULT=success fi - cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}" + { + echo "# RustFS security test report" + echo "" + echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" + echo "- Trigger: ${GITHUB_EVENT_NAME}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${TEST_OUTCOME}" + echo "" + # The dashboard prioritizes case rows over the step outcome. + # Keep partial case results in the artifact when the suite fails. + if [ "${RESULT}" = "success" ]; then + cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md" + elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then + echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics." + else + echo "The suite did not produce a non-empty report." + fi + } > "${SECURITY_ARTIFACTS_DIR}/report.md" + cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-security-report.md + REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md SUITE: security run: | set -euo pipefail @@ -210,8 +238,9 @@ jobs: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} SUITE: 'security' SUITE_LABEL: 'Security' + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-security-report.md' + REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md LOG_FILE: '' run: | set -euo pipefail @@ -245,7 +274,7 @@ jobs: echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" elif [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" @@ -263,14 +292,12 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-security-test-${{ github.run_id }} - path: | - /tmp/rustfs-security-report.md - /tmp/rustfs-security.*/* - if-no-files-found: ignore + name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.SECURITY_ARTIFACTS_DIR }}/ + if-no-files-found: error retention-days: 3 - name: Cleanup environment (after) diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py new file mode 100644 index 000000000..ae82d3fb1 --- /dev/null +++ b/scripts/test_security_workflow.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Run the security workflow's evidence and result steps without remote VMs.""" + +from __future__ import annotations + +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + +from check_test_wiring import yaml_block + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml" +CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |" + + +class SecurityWorkflowTests(unittest.TestCase): + def setUp(self) -> None: + self.source = WORKFLOW.read_text() + self.job = yaml_block(self.source.splitlines(), "security-test", 2) + self.assertIsNotNone(self.job) + starts = [i for i, line in enumerate(self.job) if line.startswith(" - name: ")] + self.steps = { + self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end] + for start, end in zip(starts, starts[1:] + [len(self.job)]) + } + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.context = { + "runner.temp": self.temp.name, + "github.server_url": "https://github.com", + "github.repository": "rustfs/rustfs", + "github.run_id": "314159", + "github.run_attempt": "2", + "github.sha": "0123456789abcdef0123456789abcdef01234567", + "github.event_name": "workflow_dispatch", + "github.workspace": self.temp.name, + "inputs.package_url": "", + "inputs.rustfs_version": "test-version", + "inputs.topology": "all", + "inputs.oidc_live": "false", + "steps.evidence.outcome": "skipped", + "steps.test.outcome": "skipped", + "steps.report.outcome": "skipped", + } + self.env = { + **os.environ, "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), + "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name, "TMPDIR": self.temp.name, + } + for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"): + self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"] + self.context["env.SECURITY_ARTIFACTS_DIR"] = "" + self.artifacts = self.directory / "rustfs-security-314159-2" + suite = self.directory / "auto-testing/rustfs-security-test.sh" + suite.parent.mkdir() + suite.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n' + 'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n' + 'case "$FAKE_REPORT" in\n' + f' present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n' + ' empty) : > "$REPORT_FILE" ;;\n' + 'esac\n' + 'echo "UNWRAPPED SUITE SUMMARY" >> "$GITHUB_STEP_SUMMARY"\n' + 'exit "$FAKE_EXIT"\n' + ) + + def render(self, value: str) -> str: + return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value) + + def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]: + result = {} + for line in yaml_block(lines, "env", indent) or []: + if line.strip() and not line.lstrip().startswith("#"): + key, value = line.strip().split(": ", 1) + result[key] = self.render(value.strip("'\"")) + return result + + def run_step(self, name: str) -> subprocess.CompletedProcess[str]: + lines = self.steps[name] + start = lines.index(" run: |") + 1 + shell_lines = [] + for line in lines[start:]: + if line.strip() and not line.startswith(" "): + break + shell_lines.append(line[10:]) + self.assertTrue(shell_lines, f"missing literal shell body: {name}") + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render("\n".join(shell_lines))], + cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True, + ) + for line in lines: + if line.startswith(" id: "): + self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success" + if Path(self.env["GITHUB_ENV"]).exists(): + for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines(): + key, value = line.split("=", 1) + self.env[key] = value + self.context[f"env.{key}"] = value + return result + + def test_workflow_wiring(self) -> None: + names = list(self.steps) + self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)")) + self.assertNotIn(" continue-on-error: true", self.job) + self.assertIn(" continue-on-error: true", self.steps["Run security suite"]) + for name in ("Initialize security evidence", "Generate report"): + self.assertNotIn(" continue-on-error: true", self.steps[name]) + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps["Generate report"]) + self.assertNotIn("/tmp/rustfs-security", self.source) + for name in ("Upload functional report to dashboard", "File failure issue in rustfs/backlog"): + report = next(line for line in self.steps[name] if line.strip().startswith("REPORT_FILE:")) + self.assertIn("${{ env.SECURITY_ARTIFACTS_DIR }}/report.md", report) + for name in ("Upload functional report to dashboard", "Upload report and logs"): + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name]) + artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8) + self.assertIn(" path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings) + self.assertIn(" if-no-files-found: error", artifact_settings) + + def test_suite_report_and_result_matrix(self) -> None: + for outcome, mode, exit_code in ( + ("success", "present", 0), ("failure", "present", 7), ("failure", "missing", 7), + ("success", "missing", 0), ("success", "empty", 0), + ("skipped", "missing", 0), ("skipped", "present", 0), + ("cancelled", "missing", 0), ("cancelled", "present", 0), + ): + with self.subTest(outcome=outcome, report=mode): + self.setUp() + initialized = self.run_step("Initialize security evidence") + self.assertEqual(initialized.returncode, 0, initialized.stderr) + self.assertEqual(self.env["SECURITY_ARTIFACTS_DIR"], str(self.artifacts)) + self.env.update(FAKE_REPORT=mode, FAKE_EXIT=str(exit_code)) + if outcome != "skipped" or mode == "present": + suite = self.run_step("Run security suite") + self.assertEqual(suite.returncode, exit_code, suite.stderr) + logs = list(self.artifacts.glob("rustfs-security.*/suite.log")) + self.assertEqual(len(logs), 1) + self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n") + self.context["steps.test.outcome"] = outcome + report = self.run_step("Generate report") + success = outcome == "success" and mode == "present" + self.assertEqual(report.returncode == 0, success, report.stderr) + contents = (self.artifacts / "report.md").read_text() + for expected in ( + "https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2", + f"Workflow Commit: {self.context['github.sha']}", "Trigger: workflow_dispatch", + f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}", + ): + self.assertIn(expected, contents) + self.assertEqual(CASE_ROW in contents, success) + self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, success) + if mode == "present": + raw = (self.artifacts / "suite-report.md").read_text() + self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n") + summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text() + self.assertEqual(summary, contents) + self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary) + + def test_existing_evidence_directory_is_rejected(self) -> None: + self.artifacts.mkdir() + stale = self.artifacts / "suite-report.md" + stale.write_text("OLD RUN REPORT") + self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0) + self.assertEqual(stale.read_text(), "OLD RUN REPORT") + self.assertFalse(Path(self.env["GITHUB_ENV"]).exists()) + (self.artifacts / "report.md").write_text("OLD RUN REPORT") + self.context.update({ + "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token", + }) + fake_bin = self.directory / "bin" + fake_bin.mkdir() + gh = fake_bin / "gh" + gh.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'if [ "$1 $2" = "issue create" ]; then\n' + ' while [ "$#" -gt 0 ]; do\n' + ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n' + ' shift\n' + ' done\n' + 'fi\n' + ) + gh.chmod(0o755) + body = self.directory / "issue-body.md" + self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body)) + result = self.run_step("File failure issue in rustfs/backlog") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("OLD RUN REPORT", body.read_text()) + self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text()) + + +if __name__ == "__main__": + unittest.main() From 2d159635eddb9f17ad504410be91e20557eee4fe Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sat, 5 Sep 2026 16:32:02 +0800 Subject: [PATCH 18/40] feat(scanner): plan dirty bucket cache refreshes (#7146) * feat(scanner): plan dirty bucket cache refreshes * fix(scanner): route peer snapshot through storage boundary --------- Co-authored-by: Henry Guo Co-authored-by: cxymds Co-authored-by: Zhengchao An --- crates/scanner/src/scanner.rs | 23 ++- crates/scanner/src/scanner/activity.rs | 41 ++++ crates/scanner/src/scanner/tests.rs | 38 ++++ crates/scanner/src/scanner_io.rs | 119 ++++++++++- crates/scanner/src/scanner_io/cache.rs | 18 ++ crates/scanner/src/scanner_io/io_cycle.rs | 95 ++++++++- .../src/scanner_io/publish_gate_tests.rs | 22 ++ crates/scanner/src/scanner_io/tests.rs | 190 ++++++++++++++++++ crates/scanner/src/storage_api.rs | 4 +- 9 files changed, 539 insertions(+), 11 deletions(-) diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index fcf67eed0..0f97924a4 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1703,14 +1703,18 @@ where let (sender, receiver) = mpsc::channel::(1); let done_cycle = Metrics::time(Metric::ScanCycle); - let scan_result = crate::scanner_io::nsscanner_with_storage_status( + let scan_result = crate::scanner_io::nsscanner_with_storage_status_scoped( storeapi.as_ref(), - cycle_budget.token(), - cycle_budget.clone(), - sender, - cycle_info.current, - leader_epoch, - scan_mode, + crate::scanner_io::ScannerCycleRequest { + ctx: cycle_budget.token(), + budget: cycle_budget.clone(), + updates: sender, + want_cycle: cycle_info.current, + leader_epoch, + scan_mode, + scan_scope: crate::scanner_io::ScannerBucketScanScope::default(), + persisted_usage_baseline: usage_persist_baseline.data.clone(), + }, ) .await; let publication_defer_reason = match &scan_result { @@ -3424,10 +3428,13 @@ use cycle_state::*; use leadership::*; use usage_store::*; +#[cfg(test)] +pub(crate) use activity::scanner_activity_snapshot_digest; pub use activity::scanner_topology_digest; pub(crate) use activity::{ ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication, - scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements, + scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest, + scanner_dirty_usage_acknowledgements, }; pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance}; pub use backlog::{ diff --git a/crates/scanner/src/scanner/activity.rs b/crates/scanner/src/scanner/activity.rs index 2f8aed1b5..ffcbc5313 100644 --- a/crates/scanner/src/scanner/activity.rs +++ b/crates/scanner/src/scanner/activity.rs @@ -902,6 +902,7 @@ where observation } +#[cfg(test)] pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] { let mut hasher = Sha256::new(); hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes()); @@ -925,6 +926,30 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho hasher.finalize().into() } +/// Hash the activity inputs that make an existing scanner cache unsafe to +/// reuse. Regular namespace writes and dirty-usage generations are omitted: +/// their affected buckets are tracked separately and may be refreshed from a +/// complete authoritative cache baseline. +pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes()); + for (host, activity) in snapshot { + let host = host.as_bytes(); + let instance_id = activity.instance_id.as_bytes(); + hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(host); + hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(instance_id); + hasher.update(activity.maintenance_generation.to_be_bytes()); + hasher.update(activity.protocol_version.to_be_bytes()); + hasher.update(activity.topology_digest); + hasher.update([u8::from(activity.data_movement_active)]); + hasher.update(activity.movement_generation.to_be_bytes()); + hasher.update([u8::from(activity.publication_blocked)]); + } + hasher.finalize().into() +} + pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool { !snapshot.is_empty() && snapshot.values().all(|activity| { @@ -955,6 +980,22 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna .collect() } +pub(crate) fn scanner_activity_dirty_usage_state_for_host<'a>( + snapshot: &'a ScannerActivitySnapshot, + host: &str, +) -> Option<(&'a str, u64, bool)> { + snapshot + .get(host) + .filter(|_| host != LOCAL_SCANNER_ACTIVITY_NODE) + .map(|activity| { + ( + activity.instance_id.as_str(), + activity.dirty_usage_generation, + activity.dirty_usage_pending, + ) + }) +} + pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] { let endpoint_pools = storeapi.endpoints(); let mut hasher = Sha256::new(); diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index efe29f612..244e9088f 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -8169,6 +8169,44 @@ fn scanner_activity_snapshot_digest_fences_dirty_usage_state() { assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending)); } +#[test] +fn scanner_activity_structural_digest_ignores_regular_bucket_writes() { + let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut written = baseline.clone(); + let activity = written.get_mut("node-2").expect("node should exist"); + activity.namespace_generation = 8; + activity.dirty_usage_generation = 6; + activity.dirty_usage_pending = true; + + assert_ne!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&written)); + assert_eq!( + scanner_activity_structural_digest(&baseline), + scanner_activity_structural_digest(&written), + "bucket writes are refreshed through the dirty-bucket scope rather than invalidating every cache" + ); +} + +#[test] +fn scanner_activity_structural_digest_fences_restart_and_maintenance() { + let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut restarted = baseline.clone(); + restarted.get_mut("node-2").expect("node should exist").instance_id = "epoch-b".to_string(); + let mut maintained = baseline.clone(); + maintained + .get_mut("node-2") + .expect("node should exist") + .maintenance_generation = 4; + + assert_ne!( + scanner_activity_structural_digest(&baseline), + scanner_activity_structural_digest(&restarted) + ); + assert_ne!( + scanner_activity_structural_digest(&baseline), + scanner_activity_structural_digest(&maintained) + ); +} + #[test] fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() { let snapshot = BTreeMap::from([ diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 4807e11b8..c8353bae9 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -21,6 +21,7 @@ use crate::{ DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState, ScannerError, SizeSummary, TierStats, }; +use bytes::Bytes; use futures::future::join_all; use metrics::counter; use rand::seq::SliceRandom as _; @@ -54,6 +55,7 @@ use tokio_util::task::AbortOnDropHandle; use tracing::{debug, error, warn}; use crate::ScannerObjectInfo as ObjectInfo; +use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot; use crate::storage_api::ScannerStorage; use crate::storage_api::scan::NamespaceLocking as _; use crate::storage_api::scanner_io::{BucketInfo, BucketOptions}; @@ -111,6 +113,121 @@ pub(crate) struct ScannerBucketScanScope { baseline_scan_plan_digest: Option, } +impl ScannerBucketScanScope { + fn is_default(&self) -> bool { + self.selected_buckets.is_none() && self.baseline_scan_plan_digest.is_none() + } + + fn from_dirty_buckets(selected_buckets: HashSet, baseline_scan_plan_digest: DataUsageScanPlanDigest) -> Self { + Self { + selected_buckets: Some(Arc::new(selected_buckets)), + baseline_scan_plan_digest: Some(baseline_scan_plan_digest), + } + } +} + +#[derive(Clone, Copy)] +pub(super) struct ScannerCacheBaselineProof<'a> { + pub(super) data: Option<&'a Bytes>, + pub(super) expected_sources: &'a HashSet, + pub(super) leader_epoch: u64, + pub(super) want_cycle: u64, + pub(super) scan_plan_digest: DataUsageScanPlanDigest, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct ScannerPeerDirtyUsageExpectation { + instance_id: String, + generation: u64, + pending: bool, +} + +fn verified_remote_dirty_usage_buckets( + expected_peers: &HashMap, + peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>, +) -> Option> { + if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() { + return None; + } + + let mut received_peers = HashSet::with_capacity(peer_snapshots.len()); + let mut dirty_buckets = HashSet::new(); + for (host, snapshot) in peer_snapshots { + let expected = expected_peers.get(&host)?; + if !received_peers.insert(host) + || snapshot.instance_id != expected.instance_id + || snapshot.generation != expected.generation + || snapshot.generation == u64::MAX + || snapshot.protocol_version != crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + || !snapshot.complete + || snapshot.pending_bucket_count != u64::try_from(snapshot.buckets.len()).unwrap_or(u64::MAX) + || (expected.pending && snapshot.pending_bucket_count == 0) + { + return None; + } + dirty_buckets.extend(snapshot.buckets.into_keys()); + } + + (received_peers.len() == expected_peers.len()).then_some(dirty_buckets) +} + +fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option { + let data = proof.data?; + let baseline = serde_json::from_slice::(data).ok()?; + if !baseline.is_complete_bucket_usage_snapshot() + || baseline.usage_snapshot_partial + || baseline.usage_snapshot_converged != Some(true) + || baseline.scanner_epoch != Some(proof.leader_epoch) + || baseline.usage_snapshot_set_states.len() != proof.expected_sources.len() + { + return None; + } + + let mut states = HashSet::with_capacity(baseline.usage_snapshot_set_states.len()); + for state in &baseline.usage_snapshot_set_states { + let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?); + if !proof.expected_sources.contains(&source) + || !states.insert(source) + || !state.complete + || state.tombstone + || state.scanner_epoch != Some(proof.leader_epoch) + || state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle) + || state.scan_plan_digest != Some(proof.scan_plan_digest.0) + { + return None; + } + } + + (states == *proof.expected_sources).then_some(proof.scan_plan_digest) +} + +fn scoped_scan_scope_from_dirty_buckets( + requested_scope: ScannerBucketScanScope, + dirty_buckets: HashSet, + dirty_snapshot_complete: bool, + all_buckets: &[BucketInfo], + baseline_proof: ScannerCacheBaselineProof<'_>, +) -> ScannerBucketScanScope { + if !requested_scope.is_default() || !dirty_snapshot_complete { + return requested_scope; + } + + let current_buckets = all_buckets.iter().map(|bucket| bucket.name.as_str()).collect::>(); + let selected_buckets = dirty_buckets + .into_iter() + .filter(|bucket| current_buckets.contains(bucket.as_str())) + .collect::>(); + if selected_buckets.is_empty() { + return requested_scope; + } + + let Some(baseline_scan_plan_digest) = complete_scanner_cache_baseline_plan_digest(baseline_proof) else { + return requested_scope; + }; + + ScannerBucketScanScope::from_dirty_buckets(selected_buckets, baseline_scan_plan_digest) +} + pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool { matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR)) } @@ -749,7 +866,7 @@ mod io_cache; mod io_cycle; #[cfg(test)] use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan}; -pub(crate) use io_cycle::nsscanner_with_storage_status; +pub(crate) use io_cycle::{ScannerCycleRequest, nsscanner_with_storage_status_scoped}; mod io_disk; #[cfg(test)] mod publish_gate_tests; diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 9522ffed2..7ff684c99 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -282,9 +282,26 @@ pub(super) fn completed_data_usage_info( .iter() .map(|(bucket, usage)| (bucket.clone(), usage.size)) .collect(); + let mut usage_snapshot_set_states = results + .iter() + .map(|result| { + let source = result.info.source?; + Some(DataUsageSnapshotSetState { + pool_index: u64::try_from(source.pool_index).ok()?, + set_index: u64::try_from(source.set_index).ok()?, + scanner_cycle: Some(result.info.next_cycle), + scanner_epoch: Some(result.info.leader_epoch), + scan_plan_digest: Some(result.info.scan_plan_digest?.0), + complete: true, + tombstone: false, + }) + }) + .collect::>>()?; + usage_snapshot_set_states.sort_by_key(|state| (state.pool_index, state.set_index)); let data_usage_info = DataUsageInfo { last_update: Some(merged_last_update), scanner_cycle: Some(results.first()?.info.next_cycle), + scanner_epoch: Some(results.first()?.info.leader_epoch), objects_total_count: u64::try_from(total.objects).ok()?, versions_total_count: u64::try_from(total.versions).ok()?, delete_markers_total_count: u64::try_from(total.delete_markers).ok()?, @@ -295,6 +312,7 @@ pub(super) fn completed_data_usage_info( bucket_sizes, buckets_usage, usage_snapshot_complete: true, + usage_snapshot_set_states, ..Default::default() }; Some((data_usage_info, merged_last_update)) diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index 3b14e9ff2..2947d638d 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -71,6 +71,7 @@ where leader_epoch, scan_mode, scan_scope: ScannerBucketScanScope::default(), + persisted_usage_baseline: None, }; nsscanner_with_storage_status_scoped(store, request).await } @@ -83,6 +84,79 @@ pub(crate) struct ScannerCycleRequest { pub(crate) leader_epoch: u64, pub(crate) scan_mode: HealScanMode, pub(crate) scan_scope: ScannerBucketScanScope, + pub(crate) persisted_usage_baseline: Option, +} + +struct ScannerBucketScopeResolution<'a> { + requested_scope: ScannerBucketScanScope, + baseline_proof: ScannerCacheBaselineProof<'a>, + activity_before: &'a crate::scanner::ScannerActivitySnapshot, + dirty_usage_snapshot: &'a DirtyUsageSnapshot, + all_buckets: &'a [BucketInfo], +} + +async fn resolve_scanner_bucket_scan_scope( + store: &S, + distributed: bool, + resolution: ScannerBucketScopeResolution<'_>, +) -> ScannerBucketScanScope +where + S: ScannerStorage, +{ + if !resolution.requested_scope.is_default() + || !resolution.dirty_usage_snapshot.covers_all_pending + || resolution.dirty_usage_snapshot.generation == u64::MAX + || resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + { + return resolution.requested_scope; + } + + let mut dirty_buckets = resolution + .dirty_usage_snapshot + .buckets + .keys() + .cloned() + .collect::>(); + if distributed { + let Some(notification_system) = store.scanner_notification_system() else { + return resolution.requested_scope; + }; + let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else { + return resolution.requested_scope; + }; + let mut expected_peers = HashMap::new(); + for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before) + { + let Some((activity_instance_id, generation, pending)) = + crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host) + else { + return resolution.requested_scope; + }; + if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) { + return resolution.requested_scope; + } + expected_peers.insert( + host, + ScannerPeerDirtyUsageExpectation { + instance_id: activity_instance_id.to_string(), + generation, + pending, + }, + ); + } + let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else { + return resolution.requested_scope; + }; + dirty_buckets.extend(remote_dirty_buckets); + } + + scoped_scan_scope_from_dirty_buckets( + resolution.requested_scope, + dirty_buckets, + true, + resolution.all_buckets, + resolution.baseline_proof, + ) } pub(crate) async fn nsscanner_with_storage_status_scoped(store: &S, request: ScannerCycleRequest) -> Result @@ -97,6 +171,7 @@ where leader_epoch, scan_mode, scan_scope, + persisted_usage_baseline, } = request; let child_token = ctx.child_token(); let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch); @@ -186,8 +261,26 @@ where } bucket_plan_complete &= buckets_by_source.keys().copied().collect::>() == *expected_sources; let scan_plan_digest = - scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before)); + scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before)); let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list)); + let scan_scope = resolve_scanner_bucket_scan_scope( + store, + distributed, + ScannerBucketScopeResolution { + requested_scope: scan_scope, + baseline_proof: ScannerCacheBaselineProof { + data: persisted_usage_baseline.as_ref(), + expected_sources: &expected_sources, + leader_epoch, + want_cycle, + scan_plan_digest, + }, + activity_before: &activity_before, + dirty_usage_snapshot: &dirty_usage_snapshot, + all_buckets: &all_buckets, + }, + ) + .await; let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle)); let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await; let tier_registry_generation = tier_registry.generation; diff --git a/crates/scanner/src/scanner_io/publish_gate_tests.rs b/crates/scanner/src/scanner_io/publish_gate_tests.rs index d740cb2eb..f131cf437 100644 --- a/crates/scanner/src/scanner_io/publish_gate_tests.rs +++ b/crates/scanner/src/scanner_io/publish_gate_tests.rs @@ -655,9 +655,31 @@ fn completed_data_usage_info_requires_every_set_before_publish() { .expect("all completed sets should produce a publishable data usage snapshot"); assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20)); assert_eq!(data_usage_info.scanner_cycle, Some(0)); + assert_eq!(data_usage_info.scanner_epoch, Some(0)); assert_eq!(data_usage_info.objects_total_count, 3); assert_eq!(data_usage_info.buckets_usage.len(), 3); assert!(data_usage_info.usage_snapshot_complete); + assert_eq!( + data_usage_info + .usage_snapshot_set_states + .iter() + .map(|state| { + ( + state.pool_index, + state.set_index, + state.scanner_cycle, + state.scanner_epoch, + state.scan_plan_digest, + state.complete, + state.tombstone, + ) + }) + .collect::>(), + vec![ + (0, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false), + (1, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false), + ] + ); assert_eq!( data_usage_info .buckets_usage diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 6fddafdfe..ec2c1ad65 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -17,6 +17,7 @@ use super::io_disk::tier_stats_template; use super::*; use crate::scanner_budget::ScannerCycleBudgetConfig; use crate::scanner_folder::ScannerItem; +use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot; use crate::storage_api::owner::{ EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats, }; @@ -796,6 +797,195 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa cache } +fn complete_usage_baseline( + source: DataUsageCacheSource, + scan_plan_digest: DataUsageScanPlanDigest, + scanner_cycle: u64, + scanner_epoch: u64, +) -> bytes::Bytes { + let baseline = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)), + scanner_cycle: Some(scanner_cycle), + scanner_epoch: Some(scanner_epoch), + buckets_count: 1, + buckets_usage: HashMap::from([("photos".to_string(), Default::default())]), + usage_snapshot_complete: true, + usage_snapshot_converged: Some(true), + usage_snapshot_set_states: vec![DataUsageSnapshotSetState { + pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"), + set_index: u64::try_from(source.set_index).expect("test set index should fit"), + scanner_cycle: Some(scanner_cycle), + scanner_epoch: Some(scanner_epoch), + scan_plan_digest: Some(scan_plan_digest.0), + complete: true, + tombstone: false, + }], + ..Default::default() + }; + bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode")) +} + +#[test] +fn scoped_scan_requires_a_converged_complete_baseline_with_exact_set_provenance() { + let source = DataUsageCacheSource::new(1, 2); + let expected_sources = HashSet::from([source]); + let scan_plan_digest = DataUsageScanPlanDigest([9; 32]); + let baseline = complete_usage_baseline(source, scan_plan_digest, 7, 11); + + assert_eq!( + complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof { + data: Some(&baseline), + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }), + Some(scan_plan_digest) + ); + + let mut incomplete = serde_json::from_slice::(&baseline).expect("test baseline should decode"); + incomplete.usage_snapshot_converged = Some(false); + let incomplete = bytes::Bytes::from(serde_json::to_vec(&incomplete).expect("test baseline should encode")); + assert_eq!( + complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof { + data: Some(&incomplete), + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }), + None + ); + + let mut wrong_provenance = serde_json::from_slice::(&baseline).expect("test baseline should decode"); + wrong_provenance.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32]); + let wrong_provenance = bytes::Bytes::from(serde_json::to_vec(&wrong_provenance).expect("test baseline should encode")); + assert_eq!( + complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof { + data: Some(&wrong_provenance), + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }), + None + ); +} + +#[test] +fn scoped_scan_selects_only_current_dirty_buckets_after_baseline_validation() { + let source = DataUsageCacheSource::new(1, 2); + let expected_sources = HashSet::from([source]); + let baseline_scan_plan_digest = DataUsageScanPlanDigest([4; 32]); + let current_scan_plan_digest = DataUsageScanPlanDigest([5; 32]); + let baseline = complete_usage_baseline(source, current_scan_plan_digest, 7, 11); + let scope = scoped_scan_scope_from_dirty_buckets( + ScannerBucketScanScope::default(), + HashSet::from(["photos".to_string(), "deleted".to_string()]), + true, + &[bucket_info("photos")], + ScannerCacheBaselineProof { + data: Some(&baseline), + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest: current_scan_plan_digest, + }, + ); + + assert_eq!(scope.baseline_scan_plan_digest, Some(current_scan_plan_digest)); + assert_eq!( + scope + .selected_buckets + .as_deref() + .expect("validated scope should select a bucket"), + &HashSet::from(["photos".to_string()]) + ); + assert_ne!(scope.baseline_scan_plan_digest, Some(baseline_scan_plan_digest)); +} + +fn peer_dirty_usage_snapshot( + instance_id: &str, + generation: u64, + complete: bool, + buckets: &[(&str, u64)], +) -> EcstoreScannerPeerDirtyUsageSnapshot { + EcstoreScannerPeerDirtyUsageSnapshot { + instance_id: instance_id.to_string(), + generation, + pending_bucket_count: u64::try_from(buckets.len()).expect("test bucket count should fit"), + protocol_version: crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + complete, + buckets: buckets + .iter() + .map(|(bucket, generation)| ((*bucket).to_string(), *generation)) + .collect(), + } +} + +#[test] +fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots() { + let expected_peers = HashMap::from([ + ( + "node-a:9000".to_string(), + ScannerPeerDirtyUsageExpectation { + instance_id: "instance-a".to_string(), + generation: 7, + pending: true, + }, + ), + ( + "node-b:9000".to_string(), + ScannerPeerDirtyUsageExpectation { + instance_id: "instance-b".to_string(), + generation: 3, + pending: false, + }, + ), + ]); + + assert_eq!( + verified_remote_dirty_usage_buckets( + &expected_peers, + vec![ + ( + "node-a:9000".to_string(), + peer_dirty_usage_snapshot("instance-a", 7, true, &[("photos", 7)]), + ), + ( + "node-b:9000".to_string(), + peer_dirty_usage_snapshot("instance-b", 3, true, &[("archive", 3)]), + ), + ], + ), + Some(HashSet::from(["photos".to_string(), "archive".to_string()])) + ); +} + +#[test] +fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state() { + let expected_peers = HashMap::from([( + "node-a:9000".to_string(), + ScannerPeerDirtyUsageExpectation { + instance_id: "instance-a".to_string(), + generation: 7, + pending: true, + }, + )]); + + for snapshot in [ + peer_dirty_usage_snapshot("instance-a", 7, false, &[("photos", 7)]), + peer_dirty_usage_snapshot("instance-a", 6, true, &[("photos", 6)]), + peer_dirty_usage_snapshot("instance-b", 7, true, &[("photos", 7)]), + peer_dirty_usage_snapshot("instance-a", 7, true, &[]), + ] { + assert!( + verified_remote_dirty_usage_buckets(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(), + "incomplete, stale, mismatched, or empty pending peer state must fall back to a full scan" + ); + } +} + #[test] fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() { let baseline_digest = DataUsageScanPlanDigest([1; 32]); diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index a0aa874b0..f81065d59 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -103,7 +103,9 @@ pub(crate) use rustfs_ecstore::api::rebalance::{ RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta, RebalanceStats as EcstoreRebalanceStats, }; -pub(crate) use rustfs_ecstore::api::rpc::ScannerBucketListing as EcstoreScannerBucketListing; +pub(crate) use rustfs_ecstore::api::rpc::{ + ScannerBucketListing as EcstoreScannerBucketListing, ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot, +}; #[cfg(test)] pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext; pub(crate) use rustfs_ecstore::api::runtime::{ From cf9688898d9fc9ae11d6b16793f355435618d122 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 16:39:26 +0800 Subject: [PATCH 19/40] fix(heal): retain completed task progress (#7177) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(heal): retain completed task progress Refs rustfs/backlog#2262 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/heal/src/heal/manager.rs | 72 ++++- crates/heal/src/heal/manager/queue.rs | 129 ++++++++ crates/heal/src/heal/manager/scheduler.rs | 114 ++++--- crates/heal/src/heal/manager/tests.rs | 351 +++++++++++++++++++++- crates/heal/src/heal/task.rs | 2 +- 5 files changed, 613 insertions(+), 55 deletions(-) diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index d36c9f9a7..971590fdb 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -45,6 +45,11 @@ use tracing::{debug, error, info, warn}; use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read}; const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60); +// Each cache includes alias tokens in its count and byte budget. Eviction +// removes every token sharing a snapshot; neither cache retains repair state. +const MAX_COMPLETED_HEAL_TOKENS: usize = 1024; +const MAX_COMPLETED_HEAL_BYTES: usize = 64 * 1024 * 1024; +const MAX_COMPLETED_HEAL_RESULT_BYTES: usize = 1024 * 1024; const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again"; const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner"; @@ -180,6 +185,8 @@ fn record_displaced_terminal( request: &HealRequest, ) -> Arc { let terminal = Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type.clone(), status: HealTaskStatus::Failed { error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"), @@ -193,6 +200,7 @@ fn record_displaced_terminal( let mut terminals = lock_displaced_terminals(registry); prune_completed_heal_statuses(&mut terminals); terminals.insert(request.id.clone(), Arc::clone(&terminal)); + prune_completed_heal_statuses(&mut terminals); terminal } @@ -209,9 +217,15 @@ async fn remove_displaced_task_aliases( .collect::>(); let mut displaced_terminals = lock_displaced_terminals(terminals); prune_completed_heal_statuses(&mut displaced_terminals); - for alias_id in alias_ids { - displaced_terminals.insert(alias_id, Arc::clone(terminal)); + if displaced_terminals + .get(task_id) + .is_some_and(|current| Arc::ptr_eq(current, terminal)) + { + for alias_id in alias_ids { + displaced_terminals.insert(alias_id, Arc::clone(terminal)); + } } + prune_completed_heal_statuses(&mut displaced_terminals); aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); } @@ -222,6 +236,36 @@ async fn remove_task_aliases_for_task(registry: &Arc +// retrying (when needed) -> aliases -> completed; queries release aliases +// before looking up active state. Publishing aliases before removing their +// mapping keeps both an already-resolved token and a new lookup valid. +async fn publish_completed_heal( + completed_heals: &Mutex>>, + task_aliases: &Mutex>, + task_id: &str, + completed: CompletedHealStatus, + terminal: bool, +) { + let completed = Arc::new(completed); + completed.retained_bytes(); + let mut aliases = task_aliases.lock().await; + let mut retained = completed_heals.lock().await; + if let Some(previous) = retained.get(task_id).cloned() { + for entry in retained.values_mut().filter(|entry| Arc::ptr_eq(entry, &previous)) { + *entry = Arc::clone(&completed); + } + } + retained.insert(task_id.to_owned(), Arc::clone(&completed)); + if terminal { + for (alias_id, _) in aliases.iter().filter(|(_, alias)| alias.task_id == task_id) { + retained.insert(alias_id.clone(), Arc::clone(&completed)); + } + aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); + } + prune_completed_heal_statuses(&mut retained); +} + #[derive(Debug, Clone)] pub struct HealTaskReport { pub status: HealTaskStatus, @@ -268,7 +312,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option) -> let result_items = match since { None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(), Some(cursor) => { - if cursor + 1 < completed.min_seq { + if cursor.saturating_add(1) < completed.min_seq { lagged = true; } completed @@ -283,7 +327,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option) -> status: completed.status.clone(), result_items, result_items_truncated: completed.result_items_truncated || lagged, - progress: None, + progress: completed.progress.clone(), next_seq: completed.next_seq, min_seq: completed.min_seq, } @@ -1847,14 +1891,14 @@ impl HealManager { pub async fn get_task_progress(&self, task_id: &str) -> Result { let canonical_task_id = self.canonical_task_id(task_id).await; - let active_heals = self.active_heals.lock().await; - if let Some(task) = active_heals.get(&canonical_task_id) { - Ok(task.get_progress().await) - } else { - Err(Error::TaskNotFound { - task_id: task_id.to_string(), - }) - } + let progress = match self.lookup_task_state(&canonical_task_id, None).await { + TaskStateLookup::Active(task) => Some(task.get_progress().await), + TaskStateLookup::Completed(completed) => completed.progress.clone(), + _ => None, + }; + progress.ok_or_else(|| Error::TaskNotFound { + task_id: task_id.to_string(), + }) } /// Cancel task @@ -1864,6 +1908,8 @@ impl HealManager { let mut active_heals = self.active_heals.lock().await; if let Some(task) = active_heals.get(&canonical_task_id) { task.cancel().await?; + let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await; + publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await; active_heals.remove(&canonical_task_id); publish_active_heal_count(&active_heals); info!( @@ -1940,6 +1986,8 @@ impl HealManager { for task_id in &task_ids { if let Some(task) = active_heals.get(task_id) { task.cancel().await?; + let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await; + publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await; } active_heals.remove(task_id); cancelled += 1; diff --git a/crates/heal/src/heal/manager/queue.rs b/crates/heal/src/heal/manager/queue.rs index b4608f27b..aceabe42a 100644 --- a/crates/heal/src/heal/manager/queue.rs +++ b/crates/heal/src/heal/manager/queue.rs @@ -82,6 +82,8 @@ pub(super) enum QueuePushOutcome { pub(super) struct CompletedHealStatus { pub(super) heal_type: HealType, pub(super) status: HealTaskStatus, + pub(super) progress: Option, + pub(super) retained_bytes: std::sync::OnceLock, pub(super) result_items_truncated: bool, pub(super) completed_at: SystemTime, /// Sequence-stamped retained window, archived with the completion so @@ -92,6 +94,133 @@ pub(super) struct CompletedHealStatus { pub(super) min_seq: u64, } +impl CompletedHealStatus { + // Account for owned capacities, including nested drive arrays. Aliases + // conservatively charge the shared allocation again, keeping both token + // count and retained payload bounded without a second ownership index. + pub(super) fn retained_bytes(&self) -> usize { + *self.retained_bytes.get_or_init(|| self.measure_retained_bytes()) + } + + fn measure_retained_bytes(&self) -> usize { + let mut bytes = size_of::(); + let mut add = |amount: usize| bytes = bytes.saturating_add(amount); + match &self.heal_type { + HealType::Cluster => {} + HealType::Bucket { bucket } => add(bucket.capacity()), + HealType::Object { + bucket, + object, + version_id, + } + | HealType::ECDecode { + bucket, + object, + version_id, + } => { + add(bucket.capacity()); + add(object.capacity()); + add(version_id.as_ref().map_or(0, String::capacity)); + } + HealType::Prefix { bucket, prefix } => { + add(bucket.capacity()); + add(prefix.capacity()); + } + HealType::Metadata { bucket, object } => { + add(bucket.capacity()); + add(object.capacity()); + } + HealType::ErasureSet { buckets, set_disk_id } => { + add(buckets.capacity().saturating_mul(size_of::())); + for bucket in buckets { + add(bucket.capacity()); + } + add(set_disk_id.capacity()); + } + } + if let HealTaskStatus::Failed { error } | HealTaskStatus::Retrying { error, .. } = &self.status { + add(error.capacity()); + } + add(self + .progress + .as_ref() + .and_then(|progress| progress.current_object.as_ref()) + .map_or(0, String::capacity)); + add(self.seqed_items.capacity().saturating_mul(size_of::<(u64, HealResultItem)>())); + for (_, item) in &self.seqed_items { + add(Self::result_item_heap_bytes(item)); + } + bytes + } + + fn result_item_heap_bytes(item: &HealResultItem) -> usize { + let mut bytes = 0usize; + let mut add = |amount: usize| bytes = bytes.saturating_add(amount); + for value in [ + &item.heal_item_type, + &item.bucket, + &item.object, + &item.version_id, + &item.detail, + ] { + add(value.capacity()); + } + for infos in [&item.before, &item.after] { + add(infos + .drives + .capacity() + .saturating_mul(size_of::())); + for drive in &infos.drives { + add(drive.uuid.capacity()); + add(drive.endpoint.capacity()); + add(drive.state.capacity()); + } + } + bytes + } + + pub(super) fn bound_result_window(&mut self) { + let mut bytes = 0usize; + let retained = self + .seqed_items + .iter() + .rev() + .take_while(|(_, item)| { + bytes = bytes + .saturating_add(size_of::<(u64, HealResultItem)>()) + .saturating_add(Self::result_item_heap_bytes(item)); + bytes <= MAX_COMPLETED_HEAL_RESULT_BYTES + }) + .count(); + let truncated = retained < self.seqed_items.len(); + if truncated { + self.seqed_items.drain(..self.seqed_items.len() - retained); + self.seqed_items.shrink_to_fit(); + self.min_seq = self.seqed_items.first().map_or(self.next_seq, |(seq, _)| *seq); + self.result_items_truncated = true; + self.retained_bytes.take(); + } + } + + pub(super) async fn snapshot(task: &HealTask, status: HealTaskStatus) -> Self { + let seqed_items = task.get_seqed_result_items().await; + let (next_seq, min_seq) = task.result_seq_cursors(); + let mut snapshot = Self { + heal_type: task.heal_type.clone(), + status, + progress: Some(task.get_progress().await), + retained_bytes: std::sync::OnceLock::new(), + result_items_truncated: task.result_items_truncated(), + completed_at: SystemTime::now(), + seqed_items, + next_seq, + min_seq, + }; + snapshot.bound_result_window(); + snapshot + } +} + #[derive(Debug, Clone)] pub(super) struct HealTaskAlias { pub(super) task_id: String, diff --git a/crates/heal/src/heal/manager/scheduler.rs b/crates/heal/src/heal/manager/scheduler.rs index c0deae524..feb0c1231 100644 --- a/crates/heal/src/heal/manager/scheduler.rs +++ b/crates/heal/src/heal/manager/scheduler.rs @@ -264,7 +264,7 @@ impl HealManager { error: error.clone(), retry_attempt: request.retry_attempts, }); - let retry_request_for_queue = retry_request; + let mut retry_request_for_queue = retry_request; let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new()); if retry_request_for_queue.is_none() { replacement_recovery_anchors_clone @@ -272,7 +272,35 @@ impl HealManager { .unwrap_or_else(|poisoned| poisoned.into_inner()) .remove(&task_id); } + let mut completed_status = match retry_request_for_status { + Some(status) => status, + None => task.get_status().await, + }; + let mut completed_status_entry = CompletedHealStatus::snapshot(&task, completed_status.clone()).await; + let completed_progress = task.get_progress().await; + #[cfg(test)] + tests::pause_completed_retention_before_publish(&task_id, &completed_status).await; let mut active_heals_guard = active_heals_clone.lock().await; + let owns_completion = active_heals_guard.contains_key(&task_id); + let cancelled_completion = if owns_completion { + false + } else { + // Cancellation can win while a finished worker waits + // for active ownership. It must not resurrect a retry + // or replace an acknowledged cancellation with success. + retry_request_for_queue = None; + completed_heals_clone + .lock() + .await + .get(&task_id) + .is_some_and(|completed| completed.status == HealTaskStatus::Cancelled) + }; + if cancelled_completion { + completed_status = HealTaskStatus::Cancelled; + completed_status_entry.status = HealTaskStatus::Cancelled; + } + let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. }); + let successful_completion = matches!(completed_status, HealTaskStatus::Completed); // Keep retry ownership continuous: status snapshots acquire // these locks in the same active -> retrying order. let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) = @@ -295,6 +323,16 @@ impl HealManager { } else { None }; + if owns_completion || cancelled_completion { + publish_completed_heal( + &completed_heals_clone, + &task_aliases_clone, + &task_id, + completed_status_entry, + terminal_completion, + ) + .await; + } let completed_task = active_heals_guard.remove(&task_id); if let Some(completed_task) = completed_task.as_ref() { publish_active_heal_count(&active_heals_guard); @@ -304,33 +342,10 @@ impl HealManager { drop(retrying_heals_guard.take()); drop(active_heals_guard); - if let Some(completed_task) = completed_task { - let completed_status = if let Some(status) = retry_request_for_status { - status - } else { - completed_task.get_status().await - }; - let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. }); - let successful_completion = matches!(completed_status, HealTaskStatus::Completed); - let completed_progress = completed_task.get_progress().await; - // Single snapshot of the retained window: the task is - // finished and already off the active map, so there is - // no concurrent writer to race with. - let seqed_items = completed_task.get_seqed_result_items().await; - let (next_seq, min_seq) = completed_task.result_seq_cursors(); - let completed_status_entry = CompletedHealStatus { - heal_type: completed_task.heal_type.clone(), - status: completed_status.clone(), - result_items_truncated: completed_task.result_items_truncated(), - completed_at: SystemTime::now(), - seqed_items, - next_seq, - min_seq, - }; - let mut completed_heals_guard = completed_heals_clone.lock().await; - prune_completed_heal_statuses(&mut completed_heals_guard); - completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry)); - drop(completed_heals_guard); + #[cfg(test)] + tests::pause_completed_retention_handoff(&task_id).await; + + if completed_task.is_some() { // update statistics let mut stats = statistics_clone.write().await; match completed_status { @@ -352,10 +367,6 @@ impl HealManager { } else { release_mrf_repair_notice_targets(notice_targets); } - task_aliases_clone - .lock() - .await - .retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id); } } @@ -718,17 +729,42 @@ pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option { } pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap>) { - let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else { - return; - }; + prune_completed_heal_statuses_at(completed_heals, SystemTime::now()); +} +pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap>, now: SystemTime) { completed_heals.retain(|_, completed| { - completed - .completed_at - .duration_since(SystemTime::UNIX_EPOCH) - .map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION) + now.duration_since(completed.completed_at) + .map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION) .unwrap_or(false) }); + let entry_bytes = |key: &String, value: &Arc| { + key.capacity() + .saturating_add(size_of::<(String, Arc)>()) + .saturating_add(value.retained_bytes()) + }; + let mut bytes = completed_heals + .iter() + .fold(0usize, |total, (key, value)| total.saturating_add(entry_bytes(key, value))); + while completed_heals.len() > MAX_COMPLETED_HEAL_TOKENS || bytes > MAX_COMPLETED_HEAL_BYTES { + let Some(oldest) = completed_heals + .iter() + .min_by(|(left_id, left), (right_id, right)| { + left.completed_at.cmp(&right.completed_at).then_with(|| left_id.cmp(right_id)) + }) + .map(|(_, value)| Arc::clone(value)) + else { + break; + }; + completed_heals.retain(|key, value| { + if Arc::ptr_eq(value, &oldest) { + bytes = bytes.saturating_sub(entry_bytes(key, value)); + false + } else { + true + } + }); + } } pub(super) fn can_schedule_request( diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index 37a100f4b..aa1c1293f 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -101,6 +101,326 @@ async fn process_manager_queue_once(manager: &HealManager) { struct MockStorage; +fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus { + CompletedHealStatus { + heal_type: HealType::Cluster, + status: HealTaskStatus::Completed, + progress: Some(HealProgress { + objects_scanned: 9, + objects_healed: 8, + objects_failed: 1, + ..Default::default() + }), + retained_bytes: std::sync::OnceLock::new(), + result_items_truncated: false, + completed_at, + seqed_items: vec![(3, HealResultItem::default()), (4, HealResultItem::default())], + next_seq: 5, + min_seq: 3, + } +} + +#[test] +fn completed_retention_cursor_boundaries_preserve_progress() { + let completed = completed_retention_fixture(SystemTime::now()); + for (cursor, count, lagged) in [ + (0, 2, true), + (1, 2, true), + (2, 2, false), + (3, 1, false), + (4, 0, false), + (5, 0, false), + (u64::MAX, 0, false), + ] { + let report = completed_task_report(&completed, Some(cursor)); + assert_eq!(report.result_items.len(), count, "cursor={cursor}"); + assert_eq!(report.result_items_truncated, lagged, "cursor={cursor}"); + assert_eq!(report.progress, completed.progress); + assert_eq!((report.next_seq, report.min_seq), (5, 3)); + } + assert_eq!(completed_task_report(&completed, None).result_items.len(), 2); +} + +#[tokio::test] +async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() { + let manager = HealManager::new(Arc::new(MockStorage), None); + let request = HealRequest::bucket("bucket".to_string()); + manager.insert_task_alias("alias", &request.id).await; + let terminal = record_displaced_terminal(&manager.displaced_terminals, &request); + lock_displaced_terminals(&manager.displaced_terminals).remove(&request.id); + remove_displaced_task_aliases(&manager.task_aliases, &manager.displaced_terminals, &request.id, &terminal).await; + for token in [&request.id, &"alias".to_string()] { + assert!(matches!(manager.get_task_report(token).await, Err(Error::TaskNotFound { .. }))); + } + assert!(manager.task_aliases.lock().await.is_empty()); + assert!(lock_displaced_terminals(&manager.displaced_terminals).is_empty()); +} + +#[test] +fn completed_retention_count_ttl_and_alias_eviction_are_bounded() { + let now = SystemTime::now(); + let mut entries = HashMap::new(); + let oldest = Arc::new(completed_retention_fixture(now - KEEP_HEAL_TASK_STATUS_DURATION)); + entries.insert("oldest".to_string(), Arc::clone(&oldest)); + entries.insert("oldest-alias".to_string(), Arc::clone(&oldest)); + for index in 2..MAX_COMPLETED_HEAL_TOKENS { + entries.insert(format!("task-{index}"), Arc::new(completed_retention_fixture(now))); + } + prune_completed_heal_statuses_at(&mut entries, now); + assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS); + entries.insert("cap-plus-one".to_string(), Arc::new(completed_retention_fixture(now))); + prune_completed_heal_statuses_at(&mut entries, now); + assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1); + assert!(!entries.contains_key("oldest")); + assert!(!entries.contains_key("oldest-alias")); + entries.clear(); + entries.insert("ttl-boundary".to_string(), oldest); + entries.insert( + "expired".to_string(), + Arc::new(completed_retention_fixture( + now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1), + )), + ); + entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1)))); + prune_completed_heal_statuses_at(&mut entries, now); + assert_eq!(entries.len(), 1); + assert!(entries.contains_key("ttl-boundary")); + prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1)); + assert!(entries.is_empty()); +} + +#[test] +fn completed_retention_total_byte_cap_and_cap_plus_one() { + let now = SystemTime::now(); + let key = "large".to_string(); + let mut entry = completed_retention_fixture(now); + let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc)>(); + entry.retained_bytes.take(); + entry.status = HealTaskStatus::Failed { + error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes), + }; + assert_eq!( + entry.retained_bytes() + key.capacity() + size_of::<(String, Arc)>(), + MAX_COMPLETED_HEAL_BYTES + ); + let mut entries = HashMap::from([(key, Arc::new(entry))]); + prune_completed_heal_statuses_at(&mut entries, now); + assert_eq!(entries.len(), 1, "exact byte cap remains retained"); + let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared"); + over.retained_bytes.take(); + if let HealTaskStatus::Failed { error } = &mut over.status { + *error = "x".repeat(error.len() + 1); + } + entries.insert("large".to_string(), Arc::new(over)); + prune_completed_heal_statuses_at(&mut entries, now); + assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound"); +} + +#[tokio::test] +async fn completed_retention_large_window_keeps_cursors_and_progress() { + let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), Arc::new(MockStorage)); + let mut snapshot = completed_retention_fixture(SystemTime::now()); + snapshot.seqed_items[0].1.detail = "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES); + snapshot.bound_result_window(); + assert_eq!(snapshot.seqed_items.len(), 1); + assert_eq!((snapshot.min_seq, snapshot.next_seq), (4, 5)); + assert!(snapshot.result_items_truncated); + assert!(snapshot.retained_bytes() < MAX_COMPLETED_HEAL_RESULT_BYTES); + let report = completed_task_report(&snapshot, Some(0)); + assert_eq!(report.progress.expect("progress retained").objects_scanned, 9); + assert!(report.result_items_truncated); + let active_max = task.get_result_items_since(Some(u64::MAX)).await; + assert!(active_max.items.is_empty()); + assert!(!active_max.lagged); +} + +#[test] +fn completed_retention_result_byte_cap_and_cap_plus_one() { + for extra in [0, 1] { + let mut snapshot = completed_retention_fixture(SystemTime::now()); + snapshot.seqed_items = vec![( + 4, + HealResultItem { + detail: "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES - size_of::<(u64, HealResultItem)>() + extra), + ..Default::default() + }, + )]; + snapshot.min_seq = 4; + snapshot.bound_result_window(); + assert_eq!(snapshot.seqed_items.len(), 1 - extra); + assert_eq!(snapshot.result_items_truncated, extra == 1); + assert_eq!(snapshot.min_seq, if extra == 0 { 4 } else { 5 }); + assert_eq!(snapshot.next_seq, 5); + assert_eq!(snapshot.progress.as_ref().expect("progress retained").objects_scanned, 9); + } +} + +#[derive(Default)] +struct CompletedRetentionHook { + started: Notify, + execute: Notify, + handoff: Notify, + finish: Notify, + pause_before_publish: bool, + before_publish: Notify, + publish: Notify, + prepared_status: Mutex>, +} + +static COMPLETED_RETENTION_HOOKS: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +pub(super) async fn pause_completed_retention_handoff(task_id: &str) { + let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned(); + if let Some(hook) = hook { + hook.handoff.notify_one(); + hook.finish.notified().await; + } +} + +pub(super) async fn pause_completed_retention_before_publish(task_id: &str, status: &HealTaskStatus) { + let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned(); + if let Some(hook) = hook.filter(|hook| hook.pause_before_publish) { + *hook.prepared_status.lock().await = Some(status.clone()); + hook.before_publish.notify_one(); + hook.publish.notified().await; + } +} + +#[tokio::test] +async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() { + let bucket = "completed-retention-retry-cancel"; + let manager = HealManager::new(Arc::new(MockStorage), None); + let request = HealRequest::object(bucket.to_string(), "object".to_string(), None); + let task_id = request.id.clone(); + let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None); + let alias = duplicate.id.clone(); + let hook = Arc::new(CompletedRetentionHook { + pause_before_publish: true, + ..Default::default() + }); + { + let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await; + hooks.insert(bucket.to_string(), Arc::clone(&hook)); + hooks.insert(task_id.clone(), Arc::clone(&hook)); + } + manager.submit_heal_request(request).await.expect("admit original"); + manager.submit_heal_request(duplicate).await.expect("admit alias"); + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(5), hook.started.notified()) + .await + .expect("scheduler starts"); + let task = manager.active_heals.lock().await.get(&task_id).cloned().expect("active task"); + task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096); + hook.execute.notify_one(); + tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified()) + .await + .expect("retry snapshot prepared"); + manager.cancel_task(&alias).await.expect("cancel wins active ownership"); + assert!(matches!(*hook.prepared_status.lock().await, Some(HealTaskStatus::Retrying { .. }))); + hook.publish.notify_one(); + tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified()) + .await + .expect("scheduler finishes handoff"); + for token in [&task_id, &alias] { + let report = manager.get_task_report(token).await.expect("cancelled token retained"); + assert_eq!(report.status, HealTaskStatus::Cancelled); + assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1); + } + assert!(!manager.retrying_heals.lock().await.contains_key(&task_id)); + assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id)); + hook.finish.notify_one(); + COMPLETED_RETENTION_HOOKS + .lock() + .await + .retain(|key, _| key != bucket && key != &task_id); +} + +#[tokio::test] +async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_handoff() { + for outcome in ["success", "failed", "cancelled"] { + let bucket = format!("completed-retention-{outcome}"); + let hook = Arc::new(CompletedRetentionHook::default()); + let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None)); + let request = HealRequest::object(bucket.clone(), "object".to_string(), None); + let task_id = request.id.clone(); + let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None); + let alias = duplicate.id.clone(); + { + let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await; + hooks.insert(bucket.clone(), Arc::clone(&hook)); + hooks.insert(task_id.clone(), Arc::clone(&hook)); + } + manager.submit_heal_request(request).await.expect("admit original"); + manager.submit_heal_request(duplicate).await.expect("admit alias"); + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(5), hook.started.notified()) + .await + .expect("scheduler reaches storage"); + let task = manager + .active_heals + .lock() + .await + .get(&task_id) + .cloned() + .expect("task is active"); + task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096); + let before = manager.get_task_report(&alias).await.expect("alias resolves active progress"); + assert_eq!(before.progress.as_ref().expect("active progress").objects_scanned, 1); + let poll_manager = Arc::clone(&manager); + let poll_alias = alias.clone(); + let stop = CancellationToken::new(); + let poll_stop = stop.clone(); + let polling = tokio::spawn(async move { + while !poll_stop.is_cancelled() { + let report = poll_manager + .get_task_report(&poll_alias) + .await + .expect("handoff must never return NotFound"); + assert!(report.progress.expect("progress never disappears").objects_scanned >= 1); + tokio::task::yield_now().await; + } + }); + if outcome == "cancelled" { + manager.cancel_task(&alias).await.expect("cancel active task by alias"); + } else { + hook.execute.notify_one(); + } + tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified()) + .await + .expect("scheduler archives terminal"); + assert!(!manager.active_heals.lock().await.contains_key(&task_id)); + let expected = task.get_progress().await; + for token in [&task_id, &alias] { + assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected); + let report = manager + .get_task_report_for_path_since(&format!("{bucket}/object"), token, Some(u64::MAX)) + .await + .expect("terminal token remains queryable at handoff"); + assert_eq!(report.progress.as_ref(), Some(&expected)); + assert!(report.result_items.is_empty()); + match outcome { + "success" => assert_eq!(report.status, HealTaskStatus::Completed), + "failed" => assert!(matches!(report.status, HealTaskStatus::Failed { .. })), + _ => assert_eq!(report.status, HealTaskStatus::Cancelled), + } + } + let retained = manager.completed_heals.lock().await; + assert!(Arc::ptr_eq(&retained[&task_id], &retained[&alias])); + drop(retained); + stop.cancel(); + polling.await.expect("concurrent polling succeeds"); + // Archived progress must not alias a mutable live progress object. + task.progress.write().await.objects_scanned = 999; + assert_eq!(manager.get_task_report(&alias).await.expect("frozen report").progress, Some(expected)); + hook.finish.notify_one(); + COMPLETED_RETENTION_HOOKS + .lock() + .await + .retain(|key, _| key != &bucket && key != &task_id); + } +} + #[async_trait::async_trait] impl HealStorageAPI for MockStorage { async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result> { @@ -123,6 +443,12 @@ impl HealStorageAPI for MockStorage { } async fn object_exists(&self, bucket: &str, _object: &str) -> Result { + let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(bucket).cloned(); + if let Some(hook) = hook { + hook.started.notify_one(); + hook.execute.notified().await; + return Ok(true); + } Ok(bucket == "retry-transition") } @@ -133,13 +459,18 @@ impl HealStorageAPI for MockStorage { _version_id: Option<&str>, _opts: &HealOpts, ) -> Result<(HealResultItem, Option)> { + if bucket == "completed-retention-failed" { + return Err(Error::TaskExecutionFailed { + message: "retention fixture failure".to_string(), + }); + } if let Some(hook) = manager_recovery_test_hook() { *hook .heal_object_calls .lock() .expect("manager recovery object call lock should not poison") += 1; } - if bucket == "retry-transition" { + if matches!(bucket, "retry-transition" | "completed-retention-retry-cancel") { return Ok(( HealResultItem::default(), Some(Error::Storage(EcstoreError::InsufficientReadQuorum( @@ -1145,7 +1476,13 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() { .expect("duplicate token should cancel merged active task"); assert!(manager.active_heals.lock().await.get(&active_task_id).is_none()); - assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. }))); + assert_eq!( + manager + .get_task_status(&active_task_id) + .await + .expect("cancelled task remains queryable"), + HealTaskStatus::Cancelled + ); } #[tokio::test] @@ -1638,6 +1975,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) -> manager.completed_heals.lock().await.insert( task_id, Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type, status: HealTaskStatus::Retrying { error: "Lock acquisition timeout".to_string(), @@ -2053,7 +2392,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() { "the overlapping admin task must be cancelled (removed from the active table) before the new one starts" ); assert!( - matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })), + matches!(manager.get_task_status(&old_id).await, Ok(HealTaskStatus::Cancelled)), "a cancelled task must no longer resolve as an active heal" ); } @@ -2360,6 +2699,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() { manager.completed_heals.lock().await.insert( task_id.clone(), Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type.clone(), status: HealTaskStatus::Retrying { error: "transient disk failure".to_string(), @@ -2395,6 +2736,8 @@ async fn test_get_task_status_reads_recent_completed_status() { manager.completed_heals.lock().await.insert( "completed-token".to_string(), Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Bucket { bucket: "bucket".to_string(), }, @@ -2424,6 +2767,8 @@ async fn test_get_task_report_for_path_reads_completed_items() { manager.completed_heals.lock().await.insert( "completed-token".to_string(), Arc::new(CompletedHealStatus { + progress: None, + retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Object { bucket: "bucket".to_string(), object: "object".to_string(), diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index c95770834..c4c1d902b 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -999,7 +999,7 @@ impl HealTask { let items = match since { None => result_items.iter().map(|(_, item)| item.clone()).collect::>(), Some(cursor) => { - if cursor + 1 < min_seq { + if cursor.saturating_add(1) < min_seq { lagged = true; } result_items From e6bf2a464660ac5cd605fda090f6e73f8035efd0 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 16:45:45 +0800 Subject: [PATCH 20/40] fix(admin): report partial background heal coverage (#7178) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(admin): report partial background heal coverage Refs rustfs/backlog#2035 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/madmin/src/client.rs | 72 +++++++++++ rustfs/src/admin/handlers/heal.rs | 197 +++++++++++++++++++++++++++--- 2 files changed, 249 insertions(+), 20 deletions(-) diff --git a/crates/madmin/src/client.rs b/crates/madmin/src/client.rs index 83f6c33d7..b49fde280 100644 --- a/crates/madmin/src/client.rs +++ b/crates/madmin/src/client.rs @@ -182,6 +182,9 @@ pub struct BackgroundHealStatus { pub heal_active_tasks: u64, #[serde(default)] pub cluster_status_complete: bool, + /// Missing on older servers; absent coverage or counts mean unknown. + #[serde(default)] + pub coverage: Option, #[serde(default)] pub progress: Option, /// Remaining wire fields (flattened `BackgroundHealInfo` plus the @@ -190,6 +193,22 @@ pub struct BackgroundHealStatus { pub extra: serde_json::Map, } +/// Node coverage of a background heal status snapshot. Counters describe only +/// nodes with usable snapshots; unknown peers may still be running heal work. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BackgroundHealCoverage { + #[serde(default)] + pub expected: Option, + #[serde(default)] + pub responded: Option, + #[serde(default)] + pub unknown: Option, + /// Stable reason codes; unknown future codes are preserved verbatim. + #[serde(default)] + pub reasons: Vec, +} + /// `GET /v3/scanner/status` response, typed at the fields operators branch /// on; everything else passes through verbatim. #[derive(Debug, Clone, Deserialize)] @@ -630,9 +649,39 @@ mod tests { assert_eq!(status.state, "active"); assert_eq!(status.heal_queue_length, 3); assert!(status.cluster_status_complete); + assert!(status.coverage.is_none(), "legacy payloads have unknown coverage"); assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through"); } + #[test] + fn background_heal_status_missing_coverage_fields_remain_unknown() { + for raw in [json!({"state": "degraded"}), json!({"state": "degraded", "coverage": {}})] { + let status: BackgroundHealStatus = serde_json::from_value(raw).expect("partial legacy payload decodes"); + assert!(!status.cluster_status_complete); + if let Some(coverage) = status.coverage { + assert_eq!(coverage.expected, None); + assert_eq!(coverage.responded, None); + assert_eq!(coverage.unknown, None); + } + } + } + + #[test] + fn background_heal_status_preserves_future_fields_and_reasons() { + let raw = json!({ + "state": "degraded", "clusterStatusComplete": false, + "coverage": {"expected": 3, "responded": 1, "unknown": 2, "reasons": ["future_reason"], "futureCoverage": true}, + "futureStatus": {"value": 7} + }); + let status: BackgroundHealStatus = serde_json::from_value(raw).expect("future additive fields decode"); + assert_eq!(status.extra["futureStatus"]["value"], 7); + let coverage = status.coverage.expect("coverage supplied"); + assert_eq!(coverage.expected, Some(3)); + assert_eq!(coverage.responded, Some(1)); + assert_eq!(coverage.unknown, Some(2)); + assert_eq!(coverage.reasons, ["future_reason"]); + } + #[test] fn scanner_status_defaults_freshness_to_unknown() { let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}}); @@ -721,6 +770,7 @@ mod tests { let status = client.background_heal_status().await.expect("status decodes"); assert_eq!(status.state, "idle"); + assert!(status.coverage.is_none(), "older HTTP responses retain unknown coverage"); let request = server.recorded(); // The server registers this route POST-only; a GET here answers 405. assert_eq!(request.method, "POST"); @@ -728,6 +778,28 @@ mod tests { assert_eq!(request.query, ""); } + #[tokio::test] + async fn background_heal_status_decodes_partial_coverage_over_http() { + let body = r#"{"state":"degraded","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":false,"coverage":{"expected":3,"responded":1,"unknown":2,"reasons":["notification_system_unavailable"]},"futureStatus":true}"#; + let server = TestServer::spawn(body, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("client builds"); + let status = client + .background_heal_status() + .await + .expect("partial status is a successful response"); + assert_eq!(status.state, "degraded"); + assert!(!status.cluster_status_complete); + assert_eq!(status.extra["futureStatus"], true); + let coverage = status.coverage.expect("partial coverage supplied"); + assert_eq!(coverage.expected, Some(3)); + assert_eq!(coverage.responded, Some(1)); + assert_eq!(coverage.unknown, Some(2)); + assert_eq!(coverage.reasons, ["notification_system_unavailable"]); + let request = server.recorded(); + assert_eq!(request.method, "POST"); + assert_eq!(request.query, "", "reading status must not send heal control parameters"); + } + #[tokio::test] async fn http_error_status_maps_to_a_typed_error_with_body() { let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await; diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index f10ad9123..74a2da980 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -39,7 +39,7 @@ use rustfs_utils::path::path_join; use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE}; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::future::Future; use std::path::PathBuf; use std::sync::Arc; @@ -261,6 +261,7 @@ struct BackgroundHealStatus<'a> { heal_active_tasks: u64, heal_operations: rustfs_heal::HealOperationsSnapshot, cluster_status_complete: bool, + coverage: &'a BackgroundHealCoverage, #[serde(skip_serializing_if = "Option::is_none")] progress: Option, } @@ -300,6 +301,23 @@ fn background_heal_runtime_state( type BackgroundHealProgress = rustfs_heal::HealProgress; +#[derive(Debug, Serialize)] +struct BackgroundHealCoverage { + expected: usize, + responded: usize, + unknown: usize, + reasons: BTreeSet, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +enum BackgroundHealCoverageReason { + NotificationSystemUnavailable, + PeerTopologyIncomplete, + PeerStatusUnsupported, + PeerStatusUnavailable, +} + #[derive(Debug)] struct ClusterHealStatusSnapshot { info: BackgroundHealInfo, @@ -307,6 +325,7 @@ struct ClusterHealStatusSnapshot { operations: rustfs_heal::HealOperationsSnapshot, progress: Option, complete: bool, + coverage: BackgroundHealCoverage, } fn add_priority_counts(total: &mut rustfs_heal::HealPriorityCounts, next: rustfs_heal::HealPriorityCounts) { @@ -338,6 +357,7 @@ fn add_operations(total: &mut rustfs_heal::HealOperationsSnapshot, next: rustfs_ } fn aggregate_cluster_heal_status(snapshots: Vec) -> ClusterHealStatusSnapshot { + let responded = snapshots.len(); let mut info = BackgroundHealInfo::default(); let mut operations = rustfs_heal::HealOperationsSnapshot::default(); let mut progress = Vec::new(); @@ -379,6 +399,12 @@ fn aggregate_cluster_heal_status(snapshots: Vec) -> Clus operations, progress, complete: true, + coverage: BackgroundHealCoverage { + expected: responded, + responded, + unknown: 0, + reasons: BTreeSet::new(), + }, } } @@ -413,12 +439,14 @@ fn merge_peer_heal_statuses( mut snapshots: Vec, peer_statuses: Vec, String>>, expected_nodes: usize, - topology_complete: bool, + coverage_reason: Option, ) -> S3Result { + let mut reasons: BTreeSet<_> = coverage_reason.into_iter().collect(); for peer_status in peer_statuses { match peer_status { Ok(Some(snapshot)) => snapshots.push(snapshot), Ok(None) => { + reasons.insert(BackgroundHealCoverageReason::PeerStatusUnsupported); warn!( event = EVENT_ADMIN_REQUEST_FAILED, component = LOG_COMPONENT_ADMIN_API, @@ -430,6 +458,7 @@ fn merge_peer_heal_statuses( ); } Err(err) => { + reasons.insert(BackgroundHealCoverageReason::PeerStatusUnavailable); warn!( event = EVENT_ADMIN_REQUEST_FAILED, component = LOG_COMPONENT_ADMIN_API, @@ -452,9 +481,12 @@ fn merge_peer_heal_statuses( // so during a reconfiguration the count can equal `expected_nodes` while // the topology is known-incomplete. Counting alone would report a // definitive answer precisely when the membership itself is in doubt. - let complete = topology_complete && snapshots.len() == expected_nodes; + let complete = reasons.is_empty() && snapshots.len() == expected_nodes; let mut status = aggregate_cluster_heal_status(snapshots); status.complete = complete; + status.coverage.expected = expected_nodes; + status.coverage.unknown = expected_nodes.saturating_sub(status.coverage.responded); + status.coverage.reasons = reasons; // A partial answer must never be mistakable for a definitive verdict: an // unreachable peer might be mid-heal, so reporting the reachable nodes' // "idle" (or disabled/uninitialized) as the cluster state would falsely @@ -497,7 +529,12 @@ async fn read_cluster_heal_status( return Ok(aggregate_cluster_heal_status(snapshots)); } let Some(notification_system) = notification_system else { - return Err(cluster_heal_status_unavailable("notification_system_unavailable")); + return merge_peer_heal_statuses( + snapshots, + Vec::new(), + expected_nodes, + Some(BackgroundHealCoverageReason::NotificationSystemUnavailable), + ); }; // An incomplete peer topology (a down member's client slot, a rolling // upgrade) previously failed the whole endpoint here, before any peer was @@ -540,7 +577,12 @@ async fn read_cluster_heal_status( })) .await; - merge_peer_heal_statuses(snapshots, peer_statuses, expected_nodes, topology_complete) + merge_peer_heal_statuses( + snapshots, + peer_statuses, + expected_nodes, + (!topology_complete).then_some(BackgroundHealCoverageReason::PeerTopologyIncomplete), + ) } async fn query_peer_replacement_recovery_status( @@ -1164,6 +1206,7 @@ fn encode_background_heal_status( heal_operations: rustfs_heal::HealOperationsSnapshot, progress: Option, cluster_status_complete: bool, + coverage: &BackgroundHealCoverage, ) -> S3Result> { let status = BackgroundHealStatus { info, @@ -1172,6 +1215,7 @@ fn encode_background_heal_status( heal_active_tasks: heal_operations.active_tasks, heal_operations, cluster_status_complete, + coverage, progress, }; serde_json::to_vec(&status).map_err(|e| { @@ -1461,6 +1505,7 @@ impl Operation for BackgroundHealStatusHandler { cluster_status.operations, cluster_status.progress, cluster_status.complete, + &cluster_status.coverage, )?; info!( event = EVENT_ADMIN_RESPONSE_EMITTED, @@ -1515,13 +1560,14 @@ impl Operation for ReplacementRecoveryStatusHandler { mod tests { use super::extract_heal_init_params; use super::{ - BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState, aggregate_cluster_heal_status, - aggregate_replacement_recovery_cluster_status, background_heal_runtime_state, build_heal_channel_request, - build_replacement_recovery_status_response, encode_background_heal_status, encode_heal_control_path, - encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability, heal_channel_response_items, - heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, json_response, - map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status, - query_peer_replacement_recovery_status, reject_heal_admission, validate_heal_request_mode, validate_heal_target, + BackgroundHealCoverage, BackgroundHealCoverageReason, BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState, + aggregate_cluster_heal_status, aggregate_replacement_recovery_cluster_status, background_heal_runtime_state, + build_heal_channel_request, build_replacement_recovery_status_response, encode_background_heal_status, + encode_heal_control_path, encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability, + heal_channel_response_items, heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, + json_response, map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status, + query_peer_replacement_recovery_status, read_cluster_heal_status, reject_heal_admission, validate_heal_request_mode, + validate_heal_target, }; use crate::storage::rpc::node_service::heal::{ NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status, @@ -2175,7 +2221,13 @@ mod tests { ..Default::default() }; - let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true) + let coverage = BackgroundHealCoverage { + expected: 1, + responded: 1, + unknown: 0, + reasons: Default::default(), + }; + let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true, &coverage) .expect("background heal info should serialize"); let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize"); @@ -2225,6 +2277,12 @@ mod tests { rustfs_heal::HealOperationsSnapshot::default(), Some(progress), true, + &BackgroundHealCoverage { + expected: 1, + responded: 1, + unknown: 0, + reasons: Default::default(), + }, ) .expect("background heal info should serialize"); let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize"); @@ -2398,6 +2456,84 @@ mod tests { assert!(peer_topology_complete(1, 0, 0, 1, 0)); } + #[tokio::test] + async fn test_background_heal_status_without_notification_preserves_local_snapshot() { + let initialized = rustfs_heal::heal_runtime_initialized(); + let info = BackgroundHealInfo { + bitrot_start_cycle: 37, + current_scan_mode: HealScanMode::Deep, + ..Default::default() + }; + for expected in [1, 3] { + let status = tokio::time::timeout(Duration::from_secs(1), read_cluster_heal_status(info.clone(), None, expected)) + .await + .expect("local status must not wait for remote peers") + .expect("missing notification must retain the local snapshot"); + assert_eq!(status.info.bitrot_start_cycle, 37); + assert_eq!(status.info.current_scan_mode, HealScanMode::Deep); + assert_eq!(status.complete, expected == 1); + assert_eq!(status.coverage.expected, expected); + assert_eq!(status.coverage.responded, 1); + assert_eq!(status.coverage.unknown, expected - 1); + if expected == 1 { + assert!(status.coverage.reasons.is_empty()); + } else { + assert!(matches!(status.state, HealRuntimeState::Degraded | HealRuntimeState::Active)); + assert_eq!( + status.coverage.reasons, + [BackgroundHealCoverageReason::NotificationSystemUnavailable].into() + ); + } + let encoded = encode_background_heal_status( + &status.info, + status.state, + status.operations, + status.progress, + status.complete, + &status.coverage, + ) + .expect("fallback status must encode"); + let decoded: rustfs_madmin::client::BackgroundHealStatus = + serde_json::from_slice(&encoded).expect("the actual madmin client must decode the server response"); + assert_eq!(decoded.cluster_status_complete, expected == 1); + let coverage = decoded.coverage.expect("new server supplies coverage"); + assert_eq!(coverage.expected, Some(expected)); + assert_eq!(coverage.responded, Some(1)); + assert_eq!(coverage.unknown, Some(expected - 1)); + if expected > 1 { + assert_eq!(coverage.reasons, ["notification_system_unavailable"]); + } + } + assert_eq!( + rustfs_heal::heal_runtime_initialized(), + initialized, + "reading status must not initialize heal" + ); + } + + #[test] + fn test_background_heal_status_coverage_reasons_are_bounded() { + let local = NodeHealStatusSnapshot::for_test(true, true, BackgroundHealInfo::default(), Default::default(), None); + let peers = (0..100) + .map(|index| { + if index % 2 == 0 { + Ok(None) + } else { + Err("peer unavailable".to_owned()) + } + }) + .collect(); + let status = merge_peer_heal_statuses(vec![local], peers, 101, None).expect("local status remains available"); + assert_eq!(status.coverage.responded, 1); + assert_eq!(status.coverage.unknown, 100); + assert_eq!(status.coverage.reasons.len(), 2); + let encoded = serde_json::to_vec(&status.coverage).expect("coverage encodes"); + assert!(encoded.len() < 256, "coverage must not grow with peer failures"); + let decoded: rustfs_madmin::client::BackgroundHealCoverage = + serde_json::from_slice(&encoded).expect("client coverage decodes"); + assert_eq!(decoded.reasons, ["peer_status_unsupported", "peer_status_unavailable"]); + } + #[test] fn test_peer_status_merge_degrades_explicitly_and_never_claims_idle() { let local = || { @@ -2413,15 +2549,20 @@ mod tests { // but the safety property of the previous fail-closed behaviour is // preserved: the partial answer is labelled Degraded, never Idle, so // unknown peer work cannot be mistaken for "nothing is running". - let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, true) + let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, None) .expect("an unreachable peer degrades the answer instead of destroying it"); assert!(!partial.complete); assert_eq!(partial.state, HealRuntimeState::Degraded); + assert_eq!(partial.coverage.expected, 2); + assert_eq!(partial.coverage.responded, 1); + assert_eq!(partial.coverage.unknown, 1); + assert_eq!(partial.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnavailable].into()); - let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, true) + let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, None) .expect("an older peer degrades the answer instead of destroying it"); assert!(!older_peer.complete); assert_eq!(older_peer.state, HealRuntimeState::Degraded); + assert_eq!(older_peer.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnsupported].into()); let known_active = NodeHealStatusSnapshot::for_test( true, @@ -2433,12 +2574,12 @@ mod tests { }, None, ); - let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, true) + let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, None) .expect("known active work may be reported as an explicit partial status"); assert!(!partial_active.complete); assert_eq!(partial_active.state, HealRuntimeState::Active); - merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, true) + merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, None) .expect_err("no snapshot at all still fails closed"); } @@ -2459,12 +2600,22 @@ mod tests { None, ) }; - let full_count_incomplete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, false) - .expect("incomplete topology degrades the answer instead of destroying it"); + let full_count_incomplete_topology = merge_peer_heal_statuses( + vec![snapshot()], + vec![Ok(Some(snapshot()))], + 2, + Some(BackgroundHealCoverageReason::PeerTopologyIncomplete), + ) + .expect("incomplete topology degrades the answer instead of destroying it"); assert!(!full_count_incomplete_topology.complete); assert_eq!(full_count_incomplete_topology.state, HealRuntimeState::Degraded); + assert_eq!(full_count_incomplete_topology.coverage.unknown, 0); + assert_eq!( + full_count_incomplete_topology.coverage.reasons, + [BackgroundHealCoverageReason::PeerTopologyIncomplete].into() + ); - let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, true) + let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, None) .expect("complete topology and full count is a definitive answer"); assert!(full_count_complete_topology.complete); assert_eq!(full_count_complete_topology.state, HealRuntimeState::Idle); @@ -2478,6 +2629,12 @@ mod tests { rustfs_heal::HealOperationsSnapshot::default(), None, false, + &BackgroundHealCoverage { + expected: 2, + responded: 1, + unknown: 1, + reasons: [BackgroundHealCoverageReason::PeerStatusUnavailable].into(), + }, ) .expect("degraded status must serialize"); let json: serde_json::Value = serde_json::from_slice(&encoded).expect("valid json"); From 42c32381b60d09a46e38554a37664352e6c66ec8 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 16:45:53 +0800 Subject: [PATCH 21/40] fix(scanner): make reset cleanup safely reentrant (#7180) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): make reset cleanup safely reentrant Refs rustfs/backlog#2264 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/scanner/src/scanner/cycle_state.rs | 331 ++++++++++++++++--- crates/scanner/src/scanner/leadership.rs | 8 +- crates/scanner/src/scanner/tests.rs | 380 +++++++++++++++++++++- 3 files changed, 664 insertions(+), 55 deletions(-) diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index dc29e3331..6913ed836 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -379,6 +379,12 @@ pub(super) fn decode_recovery_marker_for_reset( if !matches!(marker_revision, DataUsageCacheRevision::Etag(_)) { return Err(ScannerError::Other("cycle recovery marker has no object revision".to_string())); } + if let Ok(value) = serde_json::from_slice::(data) + && let Some(state) = value.get("state") + && !matches!(state.as_str(), Some("blocked" | "cleanup-pending")) + { + return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string())); + } let compat = serde_json::from_slice::(data).ok(); let _schema_version = compat.as_ref().and_then(|marker| marker.schema_version); let primary_revision = compat @@ -406,7 +412,10 @@ pub(super) fn decode_recovery_marker_for_reset( }; let state = match compat.as_ref().and_then(|marker| marker.state.as_deref()) { Some("cleanup-pending") => "cleanup-pending", - _ => "blocked", + Some("blocked") | None => "blocked", + Some(_) => { + return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string())); + } }; let now = unix_now_secs(); Ok(ScannerCycleRecoveryMarker { @@ -721,17 +730,19 @@ async fn mark_cycle_recovery_cleanup_pending( mut marker: ScannerCycleRecoveryMarker, marker_revision: &DataUsageCacheRevision, expected_epoch: u64, + owns_reset: &(impl Fn() -> bool + Sync), ) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> { marker.state = "cleanup-pending".to_string(); marker.last_attempt_at_unix_secs = unix_now_secs(); let bytes = serde_json::to_vec(&marker) .map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?; - let info = save_config_with_publication_admission_for_epoch( + let info = save_reset_config( storeapi.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), bytes, marker_revision.preconditions(), expected_epoch, + owns_reset, ) .await .map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?; @@ -933,6 +944,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< .get_write_lock_quiet(Duration::from_secs(5)) .await .map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?; + let owns_reset = || !guard.is_lock_lost() && !ctx.is_cancelled(); if guard.is_lock_lost() { return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string())); @@ -952,7 +964,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< } Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))), }; - let marker_data = marker_data.ok_or_else(|| ScannerError::Other("scanner cycle recovery marker is absent".to_string()))?; + let Some(marker_data) = marker_data else { + // A delete may commit before its reply is lost. Confirm both durable + // fences before treating a retry without its marker as completed. + let (cycle, epoch, revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?; + let floor = persisted_usage_floor(storeapi.clone()).await?; + if !matches!(revision, DataUsageCacheRevision::Etag(_)) + || epoch < floor.leader_epoch + || cycle.next < floor.next_cycle + || !owns_reset() + || scanner_publication_admission_for_epoch(storeapi.clone(), reset_epoch) + .await + .is_none() + { + return Err(ScannerError::Other( + "scanner cycle recovery marker is absent without a completed reset fence".to_string(), + )); + } + set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); + super::notify_scanner_cycle_recovery_wake(); + return Ok(()); + }; let (marker, force_full_rescan) = match serde_json::from_slice::(&marker_data) { Ok(marker) if validate_recovery_marker(&marker).is_ok() => (marker, false), _ => (decode_recovery_marker_for_reset(&marker_data, &marker_revision)?, true), @@ -1026,8 +1058,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< } }; if let Some((primary_cycle, primary_epoch)) = primary_state { + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; let (cleanup_marker, cleanup_marker_revision) = - mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?; + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch, &owns_reset) + .await?; set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending")); let usage_floor = persisted_usage_floor(storeapi.clone()).await?; let fence_epoch = primary_epoch @@ -1047,12 +1081,14 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "preserved scanner cycle state exceeds the bounded object size".to_string(), )); } - let preserved_info = save_config_with_publication_admission_for_epoch( + verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?; + let preserved_info = save_reset_config( storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), preserved_data, primary_revision.preconditions(), reset_epoch, + &owns_reset, ) .await .map_err(|err| { @@ -1072,9 +1108,17 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner leader lock was lost after fencing newer cycle state".to_string(), )); } - fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch), false) - .await - .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?; + verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?; + fence_scanner_usage_epoch_with_expected_epoch( + &ctx, + storeapi.clone(), + fence_epoch, + Some(reset_epoch), + false, + &owns_reset, + ) + .await + .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?; if guard.is_lock_lost() { return Err(ScannerError::Other( "scanner leader lock was lost after fencing newer cycle state".to_string(), @@ -1088,7 +1132,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner cycle state changed before recovery marker cleanup".to_string(), )); } - delete_config_with_publication_admission_for_epoch( + verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?; + delete_reset_config( storeapi.clone(), RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), @@ -1100,6 +1145,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< ..Default::default() }, reset_epoch, + &owns_reset, ) .await .map_err(|err| { @@ -1149,17 +1195,20 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< // Persist the cleanup-pending phase before rewriting the primary. If the // process dies after the rewrite, startup still sees a durable fence and // cannot mistake the partially completed reset for a healthy state. + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; let (marker, marker_revision) = if marker.state == "cleanup-pending" { (marker, marker_revision) } else { - mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await? + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch, &owns_reset).await? }; - let rebuilt_info = save_config_with_publication_admission_for_epoch( + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; + let rebuilt_info = save_reset_config( storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), data, primary_revision.preconditions(), reset_epoch, + &owns_reset, ) .await .map_err(|err| { @@ -1178,8 +1227,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner leader lock was lost after rebuilding cycle state".to_string(), )); } + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; if let Err(err) = - fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false).await + fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false, &owns_reset) + .await { set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { path: DATA_USAGE_BLOOM_NAME_PATH.clone(), @@ -1249,7 +1300,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< )); } - if let Err(err) = delete_config_with_publication_admission_for_epoch( + verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?; + if let Err(err) = delete_reset_config( storeapi.clone(), RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), @@ -1261,6 +1313,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< ..Default::default() }, reset_epoch, + &owns_reset, ) .await { @@ -1310,6 +1363,57 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< Ok(()) } +async fn verify_cycle_reset_intent( + storeapi: Arc, + expected_revision: &DataUsageCacheRevision, + owns_reset: &(impl Fn() -> bool + Sync), +) -> Result<(), ScannerError> { + let revision = read_config_revision(storeapi, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to verify scanner cycle reset intent: {err}")))?; + if &revision != expected_revision { + return Err(ScannerError::Other("scanner cycle reset intent changed".to_string())); + } + if !owns_reset() { + return Err(ScannerError::Other("scanner cycle reset ownership was lost".to_string())); + } + Ok(()) +} + +async fn save_reset_config( + storeapi: Arc, + path: &str, + data: Vec, + preconditions: crate::HTTPPreconditions, + expected_epoch: u64, + owns_reset: &(impl Fn() -> bool + Sync), +) -> Result { + let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + if !owns_reset() { + return Err(EcstoreError::other("scanner reset ownership was lost before write")); + } + save_config_with_preconditions(storeapi, path, data, preconditions).await +} + +async fn delete_reset_config( + storeapi: Arc, + bucket: &str, + path: &str, + options: ScannerObjectOptions, + expected_epoch: u64, + owns_reset: &(impl Fn() -> bool + Sync), +) -> Result { + let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + if !owns_reset() { + return Err(EcstoreError::other("scanner reset ownership was lost before delete")); + } + storeapi.delete_config_object(bucket, path, options).await +} + fn scanner_usage_state_reset_paths() -> Vec { vec![ DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), @@ -1333,8 +1437,14 @@ pub(super) async fn read_usage_state_reset_slots( Ok(slots) } -fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result { - let mut floor = PersistedUsageFloor::default(); +enum ScannerUsageResetFloor { + Missing, + Trusted(PersistedUsageFloor), + Corrupt, +} + +fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result { + let mut floor = None; for slot in slots { let Some(data) = slot.data.as_deref() else { continue; @@ -1342,9 +1452,21 @@ fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result(data) else { continue; }; - update_persisted_usage_floor(&mut floor, &usage, &slot.path)?; + if !data_usage_info_has_persisted_baseline_identity(&usage) + && !(slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str() && data_usage_info_is_bootstrap_pending(&usage)) + && legacy_incomplete_usage_fence(data, &usage) + .and_then(|fence| fence.claimable_epoch()) + .is_none() + { + continue; + } + update_persisted_usage_floor(floor.get_or_insert_with(PersistedUsageFloor::default), &usage, &slot.path)?; } - Ok(floor) + Ok(match floor { + Some(floor) => ScannerUsageResetFloor::Trusted(floor), + None if slots.iter().any(|slot| slot.data.is_some()) => ScannerUsageResetFloor::Corrupt, + None => ScannerUsageResetFloor::Missing, + }) } async fn read_cycle_state_for_usage_reset( @@ -1401,11 +1523,12 @@ async fn delete_usage_state_reset_slot( storeapi: Arc, slot: &ScannerUsageStateResetSlot, expected_epoch: u64, + owns_reset: &(impl Fn() -> bool + Sync), ) -> Result { if matches!(slot.revision, DataUsageCacheRevision::Missing) { return Ok(false); } - let delete_result = delete_config_with_publication_admission_for_epoch( + let delete_result = delete_reset_config( storeapi.clone(), RUSTFS_META_BUCKET, &slot.path, @@ -1415,6 +1538,7 @@ async fn delete_usage_state_reset_slot( ..Default::default() }, expected_epoch, + owns_reset, ) .await; match delete_result { @@ -1486,21 +1610,24 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary( expected_publication_epoch: u64, leader_epoch: Option, context: ScannerUsageBootstrapPublishContext, + owns_publication: impl Fn() -> bool + Sync, ) -> Result<(), ScannerError> { async fn inner( storeapi: Arc, expected_revision: &DataUsageCacheRevision, expected_publication_epoch: u64, leader_epoch: Option, + owns_publication: &(impl Fn() -> bool + Sync), ) -> Result<(), ScannerUsageBootstrapPublishError> { let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::now(), leader_epoch); let data = serde_json::to_vec(&marker).map_err(ScannerUsageBootstrapPublishError::Encode)?; - let save_result = save_config_with_publication_admission_for_epoch( + let save_result = save_reset_config( storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data.clone(), expected_revision.preconditions(), expected_publication_epoch, + owns_publication, ) .await; if save_result @@ -1524,7 +1651,7 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary( }) } - inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch) + inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch, &owns_publication) .await .map_err(|err| err.into_scanner_error(context)) } @@ -1534,32 +1661,108 @@ pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild( slots: &[ScannerUsageStateResetSlot], expected_epoch: u64, leader_epoch: u64, + owns_reset: impl Fn() -> bool + Sync, ) -> Result, ScannerError> { let mut reset_paths = Vec::new(); let primary = slots .iter() .find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str()) .ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?; - publish_scanner_usage_bootstrap_primary( - storeapi.clone(), - &primary.revision, - expected_epoch, - Some(leader_epoch), - ScannerUsageBootstrapPublishContext::Reset, - ) - .await?; + if !owns_reset() { + return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string())); + } + let resume_epoch = usage_state_reset_resume_epoch(slots)?; + match resume_epoch { + Some(epoch) if epoch == leader_epoch => {} + Some(_) => return Err(ScannerError::Other("scanner usage reset bootstrap epoch changed".to_string())), + None => { + publish_scanner_usage_bootstrap_primary( + storeapi.clone(), + &primary.revision, + expected_epoch, + Some(leader_epoch), + ScannerUsageBootstrapPublishContext::Reset, + &owns_reset, + ) + .await?; + } + } + let (data, intent_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to inspect scanner usage reset intent: {err}")))?; + data.as_deref() + .and_then(|data| serde_json::from_slice::(data).ok()) + .filter(|usage| data_usage_info_is_bootstrap_pending(usage) && usage.scanner_epoch == Some(leader_epoch)) + .ok_or_else(|| ScannerError::Other("scanner usage reset intent changed before cleanup".to_string()))?; + if !matches!(intent_revision, DataUsageCacheRevision::Etag(_)) + || (resume_epoch.is_some() && intent_revision != primary.revision) + { + return Err(ScannerError::Other("scanner usage reset intent revision changed".to_string())); + } reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string()); for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) { - if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch).await? { + if let Some(usage) = slot + .data + .as_deref() + .and_then(|data| serde_json::from_slice::(data).ok()) + && usage_epoch(&usage) >= leader_epoch + { + return Err(ScannerError::Other(format!( + "scanner usage reset slot is not older than its intent: {}", + slot.path + ))); + } + let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to verify scanner usage reset intent: {err}")))?; + if revision != intent_revision { + return Err(ScannerError::Other("scanner usage reset intent changed during cleanup".to_string())); + } + if !owns_reset() { + return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string())); + } + if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch, &owns_reset).await? { reset_paths.push(slot.path.clone()); } } + let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to confirm scanner usage reset intent: {err}")))?; + if revision != intent_revision || !owns_reset() { + return Err(ScannerError::Other( + "scanner usage reset intent or ownership changed before completion".to_string(), + )); + } invalidate_admin_data_usage_snapshot_cache().await; invalidate_data_usage_snapshot_cache().await; Ok(reset_paths) } +fn usage_state_reset_resume_epoch(slots: &[ScannerUsageStateResetSlot]) -> Result, ScannerError> { + let primary = slots.iter().find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str()); + let usage = primary + .and_then(|slot| slot.data.as_deref()) + .and_then(|data| serde_json::from_slice::(data).ok()); + match usage { + Some(usage) if usage.usage_snapshot_bootstrap_pending => { + if !data_usage_info_is_bootstrap_pending(&usage) { + return Err(ScannerError::Other("scanner usage reset bootstrap is invalid".to_string())); + } + if usage.scanner_epoch.is_none() { + // Initial bootstrap has no reset owner yet. + return Ok(None); + } + usage + .scanner_epoch + .filter(|epoch| *epoch > 0 && *epoch < u64::MAX) + .map(Some) + .ok_or_else(|| ScannerError::Other("scanner usage reset bootstrap has no valid epoch".to_string())) + } + _ => Ok(None), + } +} + pub async fn reset_scanner_usage_state_for_full_rebuild( ctx: CancellationToken, storeapi: Arc, @@ -1584,12 +1787,31 @@ pub async fn reset_scanner_usage_state_for_full_rebuild( }; let (cycle, cycle_epoch, cycle_revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?; let slots = read_usage_state_reset_slots(storeapi.clone()).await?; - let usage_floor = usage_state_reset_floor(&slots)?; - let leader_epoch = cycle_epoch - .max(usage_floor.leader_epoch) - .checked_add(1) - .filter(|epoch| *epoch < u64::MAX) - .ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?; + let usage_floor = match usage_state_reset_floor(&slots)? { + ScannerUsageResetFloor::Trusted(floor) => floor, + ScannerUsageResetFloor::Corrupt if matches!(cycle_revision, DataUsageCacheRevision::Missing) => { + return Err(ScannerError::Other("scanner usage reset has no trusted cycle or usage floor".to_string())); + } + ScannerUsageResetFloor::Missing | ScannerUsageResetFloor::Corrupt => PersistedUsageFloor { + next_cycle: cycle.next, + leader_epoch: cycle_epoch, + }, + }; + let resume_epoch = usage_state_reset_resume_epoch(&slots)?; + let leader_epoch = if let Some(epoch) = resume_epoch { + if epoch != cycle_epoch || usage_floor.leader_epoch > epoch || usage_floor.next_cycle > cycle.next { + return Err(ScannerError::Other( + "scanner usage reset bootstrap conflicts with the persisted cycle fence".to_string(), + )); + } + epoch + } else { + cycle_epoch + .max(usage_floor.leader_epoch) + .checked_add(1) + .filter(|epoch| *epoch < u64::MAX) + .ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))? + }; let rebuilt_cycle = CurrentCycle { next: cycle.next.max(usage_floor.next_cycle), ..Default::default() @@ -1602,21 +1824,24 @@ pub async fn reset_scanner_usage_state_for_full_rebuild( "scanner leader lock was lost before fencing usage reset cycle state".to_string(), )); } - save_config_with_publication_admission_for_epoch( - storeapi.clone(), - DATA_USAGE_BLOOM_NAME_PATH.as_str(), - cycle_data, - cycle_revision.preconditions(), - reset_epoch, - ) - .await - .map_err(|err| { - if scanner_publication_epoch_changed(&err) { - ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string()) - } else { - ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}")) - } - })?; + if resume_epoch.is_none() { + save_reset_config( + storeapi.clone(), + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + cycle_data, + cycle_revision.preconditions(), + reset_epoch, + &|| !guard.is_lock_lost() && !ctx.is_cancelled(), + ) + .await + .map_err(|err| { + if scanner_publication_epoch_changed(&err) { + ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string()) + } else { + ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}")) + } + })?; + } if guard.is_lock_lost() { return Err(ScannerError::Other( @@ -1624,7 +1849,10 @@ pub async fn reset_scanner_usage_state_for_full_rebuild( )); } let reset_paths = - reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch).await?; + reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch, || { + !guard.is_lock_lost() && !ctx.is_cancelled() + }) + .await?; if guard.is_lock_lost() { return Err(ScannerError::Other( "scanner leader lock was lost after publishing usage reset marker".to_string(), @@ -2135,6 +2363,7 @@ async fn recover_legacy_incomplete_usage_floor( expected_publication_epoch, Some(primary.epoch), ScannerUsageBootstrapPublishContext::Recovery, + || true, ) .await?; warn!( diff --git a/crates/scanner/src/scanner/leadership.rs b/crates/scanner/src/scanner/leadership.rs index 8283a987a..260476e7a 100644 --- a/crates/scanner/src/scanner/leadership.rs +++ b/crates/scanner/src/scanner/leadership.rs @@ -191,6 +191,7 @@ pub(super) async fn initialize_usage_baseline_bootstrap( expected_epoch, None, ScannerUsageBootstrapPublishContext::Initial, + || true, ) .await } @@ -201,9 +202,10 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch( claimed_epoch: u64, expected_publication_epoch: Option, allow_bootstrap_pending: bool, + owns_fence: impl Fn() -> bool, ) -> Result<(), ScannerError> { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { - if ctx.is_cancelled() { + if ctx.is_cancelled() || !owns_fence() { return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string())); } @@ -264,6 +266,9 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch( "scanner usage epoch fence changed while preparing its conditional write".to_string(), )); }; + if ctx.is_cancelled() || !owns_fence() { + return Err(ScannerError::Other("scanner leadership was lost before usage fencing".to_string())); + } save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions()) .await }; @@ -319,6 +324,7 @@ pub(super) async fn complete_scanner_leadership_claim( claimed_epoch, expected_publication_epoch, allow_bootstrap_pending, + || true, ) .await { diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 244e9088f..b9b4cd299 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -634,6 +634,8 @@ struct MemoryConfigStore { cancel_after_successful_puts: Mutex>, replace_after_successful_puts: Mutex)>>, error_after_commit_deletes: Mutex>, + cancel_after_deletes: Mutex>, + pause_next_publication_admission: Mutex, Arc)>>, put_counts: Mutex>, publication_admission_blocked: AtomicBool, block_publication_after_admissions: AtomicUsize, @@ -4081,6 +4083,9 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore { revisions.remove(&key); drop(revisions); drop(objects); + if let Some(token) = self.cancel_after_deletes.lock().await.remove(&key) { + token.cancel(); + } if self.error_after_commit_deletes.lock().await.remove(&key) { return Err(EcstoreError::other("injected delete error after commit")); } @@ -4088,6 +4093,11 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore { } async fn scanner_data_usage_publication_admission(&self) -> Option { + let pause = self.pause_next_publication_admission.lock().await.take(); + if let Some((entered, resume)) = pause { + entered.notify_one(); + resume.notified().await; + } if self.publication_admission_blocked.load(Ordering::Acquire) { return None; } @@ -4589,7 +4599,7 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met .expect("publication must also read the intact backup"); assert_eq!(baseline.data.as_deref(), Some(data.as_slice())); assert_eq!(baseline.revision, DataUsageCacheRevision::Missing); - fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false) + fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false, || true) .await .expect("legacy backup must be fenced into v2"); let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) @@ -4818,6 +4828,28 @@ async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() { ); } + let cycle_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("cycle should remain before retry"); + let marker_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("bootstrap should remain before retry"); + let retry = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone()) + .await + .expect("completed cleanup should be reentrant"); + assert_eq!(retry.leader_epoch, result.leader_epoch); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("cycle should remain"), + cycle_before_retry + ); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("bootstrap should remain"), + marker_before_retry + ); let (floor, state) = persisted_usage_floor_for_startup(store, false) .await .expect("reset marker should be resumable"); @@ -4973,7 +5005,7 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() { store.objects.lock().await.insert(key.clone(), b"newer-json".to_vec()); store.revisions.lock().await.insert(key, 2); - let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3) + let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true) .await .expect_err("stale primary revision must not be overwritten"); assert!( @@ -4983,6 +5015,348 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() { ); } +#[tokio::test] +async fn scanner_usage_state_reset_resumes_every_cleanup_boundary_without_rewriting_intent() { + for completed in 0..=4 { + let store = Arc::new(MemoryConfigStore::default()); + let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str(); + let cleanup_paths = [ + format!("{primary_path}.bkp"), + LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), + format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(), + ]; + for path in std::iter::once(primary_path).chain(cleanup_paths.iter().map(String::as_str)) { + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.scanner_epoch = Some(1); + save_config(store.clone(), path, serde_json::to_vec(&usage).expect("fixture should encode")) + .await + .expect("fixture should persist"); + } + // These objects belong to other owners, even when reset cleanup resumes. + for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] { + save_config(store.clone(), path, b"retain".to_vec()) + .await + .expect("unrelated state should persist"); + } + let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load"); + let cancelled = CancellationToken::new(); + if completed == 0 { + store + .cancel_after_successful_puts + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, primary_path), (2, cancelled.clone())); + } else { + store + .cancel_after_deletes + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, &cleanup_paths[completed - 1]), cancelled.clone()); + } + let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled()) + .await + .expect_err("interruption should stop cleanup"); + assert!(err.to_string().contains("ownership"), "boundary {completed}: {err}"); + for (index, path) in cleanup_paths.iter().enumerate() { + assert_eq!( + store + .objects + .lock() + .await + .contains_key(&memory_config_key(RUSTFS_META_BUCKET, path)), + index >= completed, + "boundary {completed}, slot {index}" + ); + } + let intent = read_config_with_revision(store.clone(), primary_path) + .await + .expect("intent should persist"); + let slots = read_usage_state_reset_slots(store.clone()) + .await + .expect("restart should reload slots"); + reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true) + .await + .expect("restart should complete the same intent"); + assert_eq!( + read_config_with_revision(store.clone(), primary_path) + .await + .expect("intent should remain"), + intent + ); + assert_eq!(store.put_counts.lock().await[&memory_config_key(RUSTFS_META_BUCKET, primary_path)], 2); + for path in cleanup_paths { + assert!( + !store + .objects + .lock() + .await + .contains_key(&memory_config_key(RUSTFS_META_BUCKET, &path)) + ); + } + for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] { + assert_eq!(read_config(store.clone(), path).await.expect("unrelated state should remain"), b"retain"); + } + } +} + +#[tokio::test] +async fn scanner_usage_state_reset_stops_usage_fence_after_owner_loss() { + let store = Arc::new(MemoryConfigStore::default()); + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.scanner_epoch = Some(1); + let bytes = serde_json::to_vec(&usage).expect("baseline should encode"); + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone()) + .await + .expect("baseline should persist"); + let checks = AtomicUsize::new(0); + let err = fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 3, Some(0), false, || { + checks.fetch_add(1, Ordering::SeqCst) == 0 + }) + .await + .expect_err("ownership lost during reads must prevent the write"); + assert!(err.to_string().contains("leadership was lost"), "{err}"); + assert_eq!( + read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("baseline should remain"), + bytes + ); +} + +#[tokio::test] +async fn scanner_usage_state_reset_cancels_during_publication_admission() { + for resuming in [false, true] { + let store = Arc::new(MemoryConfigStore::default()); + let usage = if resuming { + scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3)) + } else { + complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0) + }; + save_config( + store.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + serde_json::to_vec(&usage).expect("primary should encode"), + ) + .await + .expect("primary should persist"); + save_config(store.clone(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), b"corrupt".to_vec()) + .await + .expect("cleanup target should persist"); + let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load"); + let before = store.objects.lock().await.clone(); + let revisions_before = store.revisions.lock().await.clone(); + let entered = Arc::new(tokio::sync::Notify::new()); + let resume = Arc::new(tokio::sync::Notify::new()); + *store.pause_next_publication_admission.lock().await = Some((entered.clone(), resume.clone())); + let cancelled = CancellationToken::new(); + let (result, ()) = tokio::join!( + reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled()), + async { + entered.notified().await; + cancelled.cancel(); + resume.notify_one(); + } + ); + let err = result.expect_err("losing ownership during admission must prevent mutation"); + assert!(err.to_string().contains("ownership was lost"), "resuming={resuming}: {err}"); + assert_eq!(*store.objects.lock().await, before); + assert_eq!(*store.revisions.lock().await, revisions_before); + } +} + +#[tokio::test] +#[serial] +async fn scanner_usage_state_reset_rejects_corruption_without_a_trusted_floor() { + let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await; + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), b"{corrupt".to_vec()) + .await + .expect("corrupt primary should persist"); + let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("evidence should load"); + let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone()) + .await + .expect_err("corruption must not become a zero floor"); + assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}"); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("evidence should remain"), + before + ); + assert!(matches!( + read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); + let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + backup.scanner_epoch = Some(7); + backup.scanner_cycle = Some(40); + save_config( + store.clone(), + &format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&backup).expect("backup should encode"), + ) + .await + .expect("valid backup should persist"); + let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store) + .await + .expect("valid backup should supply the recovery floor"); + assert_eq!(result.leader_epoch, 8); + assert_eq!(result.next_cycle, 41); +} + +#[tokio::test] +async fn scanner_usage_state_reset_rejects_replaced_intent_and_newer_cleanup_slot() { + let store = Arc::new(MemoryConfigStore::default()); + let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3)); + let bytes = serde_json::to_vec(&marker).expect("marker should encode"); + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone()) + .await + .expect("intent should persist"); + let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load"); + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes) + .await + .expect("another intent should persist"); + let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true) + .await + .expect_err("same epoch cannot replace an intent revision"); + assert!(err.to_string().contains("intent revision changed"), "{err}"); + + let mut newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + newer.scanner_epoch = Some(3); + let path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let bytes = serde_json::to_vec(&newer).expect("newer snapshot should encode"); + save_config(store.clone(), &path, bytes.clone()) + .await + .expect("newer snapshot should persist"); + let slots = read_usage_state_reset_slots(store.clone()) + .await + .expect("slots should reload"); + let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true) + .await + .expect_err("cleanup cannot delete same-epoch progress"); + assert!(err.to_string().contains("not older than its intent"), "{err}"); + assert_eq!(read_config(store, &path).await.expect("newer snapshot should remain"), bytes); +} + +#[tokio::test] +#[serial] +async fn scanner_usage_state_reset_rejects_decodable_untrusted_floor() { + let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await; + let invalid_identity = DataUsageInfo { + usage_snapshot_complete: true, + buckets_count: 1, + last_update: Some(std::time::SystemTime::UNIX_EPOCH), + ..Default::default() + }; + for usage in [DataUsageInfo::default(), invalid_identity] { + save_config( + store.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + serde_json::to_vec(&usage).expect("fixture should encode"), + ) + .await + .expect("untrusted primary should persist"); + let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("primary should load"); + let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone()) + .await + .expect_err("valid JSON alone cannot prove a usage floor"); + assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}"); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("evidence should remain"), + before + ); + assert!(matches!( + read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); + } +} + +#[test] +fn full_rescan_reset_rejects_unknown_marker_phase_even_with_invalid_compat_fields() { + for state in [serde_json::json!("rewrite-v2"), serde_json::json!(7), serde_json::Value::Null] { + let marker = serde_json::json!({"state": state, "retry_count": "future-type", "schema_version": 99}); + let err = super::cycle_state::decode_recovery_marker_for_reset( + &serde_json::to_vec(&marker).expect("future marker should encode"), + &DataUsageCacheRevision::Etag("intent-1".to_string()), + ) + .expect_err("unknown persistent phases must remain fenced"); + assert!(err.to_string().contains("state is unsupported"), "{err}"); + } +} + +#[tokio::test] +#[serial] +async fn full_rescan_reset_preserves_unknown_phase_and_retries_completed_cleanup() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt".to_vec()) + .await + .expect("corrupt primary should persist"); + save_config( + store.clone(), + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + br#"{"state":"future-rewrite"}"#.to_vec(), + ) + .await + .expect("future marker should persist"); + let primary_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("primary should load"); + let marker_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("marker should load"); + let err = reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect_err("unknown phase must block explicit reset"); + assert!(err.to_string().contains("state is unsupported"), "{err}"); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("primary should remain"), + primary_before + ); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()) + .await + .expect("marker should remain"), + marker_before + ); + + save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{malformed".to_vec()) + .await + .expect("recoverable marker should persist"); + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("reset should complete"); + let primary = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt primary should load"); + let usage = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("fenced usage should load"); + reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()) + .await + .expect("retry after marker deletion should complete"); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("rebuilt primary should remain"), + primary + ); + assert_eq!( + read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("fenced usage should remain"), + usage + ); +} + #[tokio::test] async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() { let store = Arc::new(MemoryConfigStore::default()); @@ -4993,7 +5367,7 @@ async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() { .expect("usage reset slots should be inspected"); store.publication_admission_blocked.store(true, Ordering::Release); - let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3) + let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true) .await .expect_err("movement admission loss must defer reset"); assert!( From 0d1b312673d062e7ca236d2f444728a6afcfed5e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 16:49:58 +0800 Subject: [PATCH 22/40] fix(ci): require fresh successful scheduled validations (#7192) --- .../scheduled-validation-freshness.yml | 1 + .../check_scheduled_validation_freshness.py | 398 +++++++++++++----- 2 files changed, 293 insertions(+), 106 deletions(-) diff --git a/.github/workflows/scheduled-validation-freshness.yml b/.github/workflows/scheduled-validation-freshness.yml index eef340869..69ddc844c 100644 --- a/.github/workflows/scheduled-validation-freshness.yml +++ b/.github/workflows/scheduled-validation-freshness.yml @@ -42,6 +42,7 @@ jobs: - name: Check latest scheduled runs env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | set +e python3 scripts/check_scheduled_validation_freshness.py \ diff --git a/scripts/check_scheduled_validation_freshness.py b/scripts/check_scheduled_validation_freshness.py index a5812c7a3..c694e3c89 100644 --- a/scripts/check_scheduled_validation_freshness.py +++ b/scripts/check_scheduled_validation_freshness.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -"""Fail when a critical scheduled validation has not started recently.""" +"""Require recent scheduled attempts and completed successes on the default branch.""" from __future__ import annotations import argparse from datetime import datetime, timedelta, timezone +import io import json import os from pathlib import Path @@ -13,7 +14,7 @@ import sys import tempfile import unittest from unittest import mock -from urllib.parse import quote, urlencode +from urllib.parse import parse_qs, quote, urlencode, urlsplit from urllib.request import Request, urlopen @@ -75,15 +76,8 @@ def stale_reason( run: dict[str, object] | None, now: datetime, max_age_hours: int, - never_ran_grace_until: datetime | None = None, ) -> str | None: if run is None: - # The grace deadline only covers a workflow whose first scheduled slot - # has not arrived yet (for example a monthly cron enabled mid-month). - # A recorded-but-old run proves the schedule used to fire and stopped, - # so the grace never masks that case. - if never_ran_grace_until is not None and now <= never_ran_grace_until: - return None return "no scheduled run has been recorded" created_at = parse_timestamp(run.get("created_at")) age = now - created_at @@ -93,14 +87,23 @@ def stale_reason( def fetch_latest_scheduled_run( - repository: str, workflow: str, token: str, api_url: str + repository: str, + workflow: str, + token: str, + api_url: str, + default_branch: str, + successful: bool = False, ) -> dict[str, object] | None: owner, repo = repository.split("/", 1) workflow_name = Path(workflow).name + query = {"event": "schedule", "branch": default_branch, "per_page": 1} + if successful: + # Filter on the server: the last success may be beyond a page of failures. + query["status"] = "success" endpoint = ( f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}" f"/actions/workflows/{quote(workflow_name, safe='')}/runs?" - + urlencode({"event": "schedule", "per_page": 1}) + + urlencode(query) ) request = Request( endpoint, @@ -110,57 +113,104 @@ def fetch_latest_scheduled_run( "X-GitHub-Api-Version": "2022-11-28", }, ) - with urlopen(request, timeout=30) as response: + # Two requests per manifest entry must fit the watchdog's ten-minute job. + with urlopen(request, timeout=15) as response: payload = json.load(response) - runs = payload.get("workflow_runs") + runs = payload.get("workflow_runs") if isinstance(payload, dict) else None if not isinstance(runs, list): raise ValueError(f"GitHub returned no workflow_runs list for {workflow}") + total_count = payload.get("total_count") + if not isinstance(total_count, int) or isinstance(total_count, bool) or total_count < len(runs): + raise ValueError(f"GitHub returned an invalid run count for {workflow}") if not runs: + if total_count: + raise ValueError(f"GitHub returned an empty first page with recorded runs for {workflow}") return None - if not isinstance(runs[0], dict): + run = runs[0] + if not isinstance(run, dict): raise ValueError(f"GitHub returned an invalid workflow run for {workflow}") - return runs[0] + if run.get("event") != "schedule" or run.get("head_branch") != default_branch: + raise ValueError(f"GitHub returned a run outside the scheduled default-branch query for {workflow}") + if not isinstance(run.get("status"), str) or not run["status"]: + raise ValueError(f"GitHub returned no run status for {workflow}") + conclusion = run.get("conclusion") + if (conclusion is not None and not isinstance(conclusion, str)) or ( + run["status"] == "completed" and not conclusion + ): + raise ValueError(f"GitHub returned an invalid run conclusion for {workflow}") + if successful and (run["status"] != "completed" or conclusion != "success"): + raise ValueError(f"GitHub returned a run without a completed success for {workflow}") + parse_timestamp(run.get("created_at")) + if not isinstance(run.get("html_url"), str) or not run["html_url"]: + raise ValueError(f"GitHub returned no run URL for {workflow}") + return run -def write_report(path: Path, failures: list[tuple[str, int, str, str]]) -> None: - lines = ["## Scheduled validation freshness"] - if not failures: - lines.append("") - lines.append("All critical scheduled validations have a recent scheduled run.") - else: - lines.extend( - [ - "", - "The following critical validations are stale or could not be inspected:", - "", - "| Workflow | Limit | Result | Last run |", - "| --- | ---: | --- | --- |", - ] - ) - for workflow, max_age_hours, reason, run_url in failures: - link = f"[open]({run_url})" if run_url else "—" - lines.append(f"| `{workflow}` | {max_age_hours}h | {reason} | {link} |") +def describe_run(run: dict[str, object] | None) -> str: + if run is None: + return "No recorded run" + outcome = run["status"] + if run.get("conclusion"): + outcome = f"{outcome}/{run['conclusion']}" + return f"[{outcome}]({run['html_url']}) — created {run['created_at']}" + + +def write_report(path: Path, rows: list[tuple[str, int, str, str, str]], default_branch: str) -> None: + lines = [ + "## Scheduled validation freshness", + "", + f"Default branch: `{default_branch}`. Ages use scheduled-run creation time; rerunning an old commit does not refresh its evidence.", + "Attempt outcomes are shown independently of successful-run freshness.", + "Success is the GitHub workflow run conclusion; suite completeness remains the responsibility of each workflow.", + "", + "| Workflow | Limit | Freshness | Last attempt | Last completed success |", + "| --- | ---: | --- | --- | --- |", + ] + for workflow, max_age_hours, result, attempt, success in rows: + cells = [f"`{workflow}`", f"{max_age_hours}h", result, attempt, success] + lines.append("| " + " | ".join(cell.replace("|", "\\|").replace("\n", " ") for cell in cells) + " |") path.write_text("\n".join(lines) + "\n") def check_freshness( - config: Path, report: Path, repository: str, token: str, api_url: str + config: Path, report: Path, repository: str, token: str, api_url: str, default_branch: str ) -> int: now = datetime.now(timezone.utc) - failures: list[tuple[str, int, str, str]] = [] + rows: list[tuple[str, int, str, str, str]] = [] + failed = False for workflow, max_age_hours, never_ran_grace_until in load_validations(config): - try: - run = fetch_latest_scheduled_run(repository, workflow, token, api_url) - reason = stale_reason(run, now, max_age_hours, never_ran_grace_until) - if reason is not None: - run_url = str(run.get("html_url", "")) if run else "" - failures.append((workflow, max_age_hours, reason, run_url)) - except Exception as error: - failures.append( - (workflow, max_age_hours, f"inspection failed: {error}", "") - ) - write_report(report, failures) - return 1 if failures else 0 + runs: dict[str, dict[str, object] | None] = {} + reasons: list[str] = [] + for label, successful in (("Last attempt", False), ("Last completed success", True)): + try: + runs[label] = fetch_latest_scheduled_run( + repository, workflow, token, api_url, default_branch, successful + ) + except Exception as error: + reasons.append(f"{label}: inspection failed: {error}") + # A failed inspection or any recorded attempt ends first-run grace. + initial_grace = ( + len(runs) == 2 + and all(run is None for run in runs.values()) + and never_ran_grace_until is not None + and now <= never_ran_grace_until + ) + if not initial_grace: + for label, run in runs.items(): + reason = stale_reason(run, now, max_age_hours) + if reason is not None: + reasons.append(f"{label}: {reason}") + failed |= bool(reasons) + result = "; ".join(reasons) if reasons else "Fresh" + if initial_grace: + result = f"Initial grace until {never_ran_grace_until.isoformat()}" + evidence = [ + describe_run(runs[label]) if label in runs else "Inspection failed" + for label in ("Last attempt", "Last completed success") + ] + rows.append((workflow, max_age_hours, result, *evidence)) + write_report(report, rows, default_branch) + return 1 if failed else 0 class SelfTests(unittest.TestCase): @@ -173,15 +223,6 @@ class SelfTests(unittest.TestCase): self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36)) self.assertIsNotNone(stale_reason(None, self.NOW, 36)) - def test_never_ran_grace_only_covers_missing_runs(self) -> None: - future_grace = self.NOW + timedelta(hours=1) - past_grace = self.NOW - timedelta(seconds=1) - self.assertIsNone(stale_reason(None, self.NOW, 36, future_grace)) - self.assertIsNone(stale_reason(None, self.NOW, 36, self.NOW)) - self.assertIsNotNone(stale_reason(None, self.NOW, 36, past_grace)) - stale_run = {"created_at": "2026-08-20T23:59:59Z"} - self.assertIsNotNone(stale_reason(stale_run, self.NOW, 36, future_grace)) - def test_config_rejects_duplicate_and_invalid_entries(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "validations.json" @@ -238,63 +279,205 @@ class SelfTests(unittest.TestCase): ], ) - def test_check_reports_missing_runs(self) -> None: + @staticmethod + def run_fixture(**overrides: object) -> dict[str, object]: + return { + "status": "completed", + "conclusion": "success", + "event": "schedule", + "head_branch": "release/current", + "created_at": "2026-08-22T00:00:00Z", + "html_url": "https://github.test/rustfs/rustfs/actions/runs/1", + **overrides, + } + + def check_payloads( + self, payloads: list[object], *, grace: str | None = None, workflows: int = 1 + ) -> tuple[int, str, list]: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) config = root / "validations.json" report = root / "report.md" - config.write_text( - json.dumps( - [ - {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}, - {"workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36}, - {"workflow": ".github/workflows/mint.yml", "max_age_hours": 36}, - ] - ) - ) - with mock.patch( - __name__ + ".fetch_latest_scheduled_run", - side_effect=[ - {"created_at": "2999-01-01T00:00:00Z"}, - None, - RuntimeError("API unavailable"), - ], + entries = [ + {"workflow": f".github/workflows/check-{index}.yml", "max_age_hours": 36} + for index in range(workflows) + ] + if grace is not None: + entries[0]["never_ran_grace_until"] = grace + config.write_text(json.dumps(entries)) + responses = [] + for payload in payloads: + if isinstance(payload, dict) and isinstance(payload.get("workflow_runs"), list): + payload = {"total_count": len(payload["workflow_runs"]), **payload} + responses.append(payload if isinstance(payload, Exception) else io.StringIO(json.dumps(payload))) + with ( + mock.patch(__name__ + ".urlopen", side_effect=responses) as request, + mock.patch(__name__ + ".datetime", wraps=datetime) as clock, ): - self.assertEqual( - check_freshness( - config, - report, - "rustfs/rustfs", - "token", - "https://api.github.test", - ), - 1, + clock.now.return_value = self.NOW + status = check_freshness( + config, report, "rustfs/rustfs", "test-token", + "https://api.github.test", "release/current", ) - contents = report.read_text() - self.assertIn(".github/workflows/fuzz.yml", contents) - self.assertIn("inspection failed: API unavailable", contents) - self.assertNotIn(".github/workflows/ci.yml`", contents) + return status, report.read_text(), request.call_args_list - config.write_text( - json.dumps( - [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}] + def test_requests_filter_schedule_default_branch_and_success_on_server(self) -> None: + attempt = self.run_fixture(status="in_progress", conclusion=None) + success = self.run_fixture(html_url="https://github.test/rustfs/rustfs/actions/runs/2") + status, report, calls = self.check_payloads([ + {"workflow_runs": [attempt], "total_count": 1001}, + {"workflow_runs": [success], "total_count": 1}, + ]) + self.assertEqual(status, 0) + self.assertEqual(len(calls), 2) + for call, successful in zip(calls, (False, True)): + request = call.args[0] + url = urlsplit(request.full_url) + self.assertEqual(url.path, "/repos/rustfs/rustfs/actions/workflows/check-0.yml/runs") + expected = {"event": ["schedule"], "branch": ["release/current"], "per_page": ["1"]} + if successful: + expected["status"] = ["success"] + self.assertEqual(parse_qs(url.query), expected) + self.assertEqual(request.get_header("Authorization"), "Bearer test-token") + self.assertEqual(call.kwargs, {"timeout": 15}) + self.assertIn("[in_progress]", report) + self.assertIn(str(attempt["html_url"]), report) + self.assertIn(str(success["html_url"]), report) + + def test_cancelled_attempt_cannot_refresh_expired_success(self) -> None: + attempt = self.run_fixture(conclusion="cancelled") + success = self.run_fixture( + created_at="2026-08-20T23:59:59Z", updated_at="2026-08-22T11:59:59Z", + html_url="https://github.test/rustfs/rustfs/actions/runs/2", + ) + status, report, _ = self.check_payloads([ + {"workflow_runs": [attempt]}, {"workflow_runs": [success]}, + ]) + self.assertEqual(status, 1) + self.assertIn("Last completed success: last scheduled run is", report) + self.assertIn("[completed/cancelled]", report) + for run in (attempt, success): + self.assertIn(str(run["html_url"]), report) + self.assertIn(str(run["created_at"]), report) + + def test_attempt_outcome_does_not_replace_recent_success(self) -> None: + success = self.run_fixture(created_at="2026-08-21T00:00:00Z") + for state, conclusion in ( + ("completed", "failure"), ("completed", "cancelled"), + ("completed", "timed_out"), ("completed", "success"), + ("queued", None), ("in_progress", None), + ): + with self.subTest(state=state, conclusion=conclusion): + status, report, _ = self.check_payloads([ + {"workflow_runs": [self.run_fixture(status=state, conclusion=conclusion)]}, + {"workflow_runs": [success]}, + ]) + self.assertEqual(status, 0) + self.assertIn(f"[{state}" + (f"/{conclusion}" if conclusion else "") + "]", report) + self.assertIn("Fresh", report) + self.assertNotIn("All critical scheduled validations", report) + + def test_grace_requires_two_successful_queries_with_no_history(self) -> None: + for attempt, success, grace, expected in ( + (None, None, "2026-08-22T12:00:00Z", 0), + (None, None, "2026-08-22T11:59:59Z", 1), + (self.run_fixture(conclusion="failure"), None, "2026-08-23T00:00:00Z", 1), + (self.run_fixture(status="queued", conclusion=None), None, "2026-08-23T00:00:00Z", 1), + (None, self.run_fixture(), "2026-08-23T00:00:00Z", 1), + ): + with self.subTest(attempt=attempt, success=success, grace=grace): + status, report, _ = self.check_payloads([ + {"workflow_runs": [] if attempt is None else [attempt]}, + {"workflow_runs": [] if success is None else [success]}, + ], grace=grace) + self.assertEqual(status, expected) + self.assertEqual("Initial grace until" in report, expected == 0) + + def test_api_failures_preserve_other_evidence_and_never_enter_grace(self) -> None: + good = {"workflow_runs": [self.run_fixture()]} + for first, second in ( + (RuntimeError("API unavailable"), good), + (good, RuntimeError("API unavailable")), + (RuntimeError("API unavailable"), {"workflow_runs": []}), + ): + with self.subTest(first=first, second=second): + status, report, calls = self.check_payloads( + [first, second], grace="2026-08-23T00:00:00Z" ) - ) - with mock.patch( - __name__ + ".fetch_latest_scheduled_run", - return_value={"created_at": "2999-01-01T00:00:00Z"}, - ): - self.assertEqual( - check_freshness( - config, - report, - "rustfs/rustfs", - "token", - "https://api.github.test", - ), - 0, - ) - self.assertIn("All critical scheduled validations", report.read_text()) + self.assertEqual(status, 1) + self.assertEqual(len(calls), 2) + self.assertIn("inspection failed: API unavailable", report) + self.assertNotIn("Initial grace until", report) + if first is good or second is good: + self.assertIn(str(self.run_fixture()["html_url"]), report) + + def test_invalid_api_evidence_fails_closed(self) -> None: + malformed = [ + [], {}, {"workflow_runs": {}}, {"workflow_runs": [None]}, + {"workflow_runs": [], "total_count": 1}, + {"workflow_runs": [], "total_count": -1}, + {"workflow_runs": [], "total_count": None}, + {"workflow_runs": [], "total_count": True}, + *({"workflow_runs": [self.run_fixture(**override)]} for override in ( + {"event": "workflow_dispatch"}, {"head_branch": "other"}, + {"created_at": "invalid"}, {"created_at": "2026-08-22T00:00:00"}, + {"status": None}, {"conclusion": None}, {"conclusion": 1}, + {"html_url": ""}, + )), + ] + for payload in malformed: + for index, label in enumerate(("Last attempt", "Last completed success")): + with self.subTest(payload=payload, label=label): + payloads = [{"workflow_runs": [self.run_fixture()]} for _ in range(2)] + payloads[index] = payload + status, report, _ = self.check_payloads(payloads, grace="2026-08-23T00:00:00Z") + self.assertEqual(status, 1) + self.assertIn(f"{label}: inspection failed", report) + self.assertNotIn("Initial grace until", report) + self.assertIn(str(self.run_fixture()["html_url"]), report) + for state, conclusion in (("in_progress", "success"), ("completed", "failure"), ("completed", "skipped")): + with self.subTest(state=state, conclusion=conclusion): + status, report, _ = self.check_payloads([ + {"workflow_runs": [self.run_fixture()]}, + {"workflow_runs": [self.run_fixture(status=state, conclusion=conclusion)]}, + ]) + self.assertEqual(status, 1) + self.assertIn("without a completed success", report) + + def test_report_retains_every_workflow(self) -> None: + status, report, calls = self.check_payloads([ + {"workflow_runs": [self.run_fixture()]}, {"workflow_runs": [self.run_fixture()]}, + {"workflow_runs": []}, {"workflow_runs": []}, + RuntimeError("API unavailable"), {"workflow_runs": [self.run_fixture()]}, + ], workflows=3) + self.assertEqual(status, 1) + self.assertEqual(len(calls), 6) + for index in range(3): + self.assertEqual(report.count(f"`.github/workflows/check-{index}.yml`"), 1) + self.assertIn("No recorded run", report) + self.assertIn("Inspection failed", report) + + def test_cli_requires_the_repository_default_branch(self) -> None: + from check_test_wiring import yaml_block + + workflow = (ROOT / ".github/workflows/scheduled-validation-freshness.yml").read_text().splitlines() + job = yaml_block(workflow, "check-freshness", 2) + self.assertIsNotNone(job) + start = job.index(" - name: Check latest scheduled runs") + end = next((index for index in range(start + 1, len(job)) if job[index].startswith(" - ")), len(job)) + environment = yaml_block(job[start:end], "env", 8) + self.assertIsNotNone(environment) + self.assertIn(" RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}", environment) + + with ( + mock.patch.dict(os.environ, {"GITHUB_REPOSITORY": "rustfs/rustfs", "GH_TOKEN": "test-token"}, clear=True), + mock.patch.object(sys, "argv", ["checker", "--report", "unused.md"]), + mock.patch("sys.stderr", new=io.StringIO()) as stderr, + self.assertRaises(SystemExit) as error, + ): + main() + self.assertEqual(error.exception.code, 2) + self.assertIn("RUSTFS_DEFAULT_BRANCH", stderr.getvalue()) def main() -> int: @@ -318,11 +501,14 @@ def main() -> int: repository = os.environ.get("GITHUB_REPOSITORY", "") token = os.environ.get("GH_TOKEN", "") api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com") + default_branch = os.environ.get("RUSTFS_DEFAULT_BRANCH", "") if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository): parser.error("GITHUB_REPOSITORY must be owner/repository") if not token: parser.error("GH_TOKEN is required") - return check_freshness(args.config, args.report, repository, token, api_url) + if not default_branch or any(character.isspace() for character in default_branch): + parser.error("RUSTFS_DEFAULT_BRANCH is required and must name the repository default branch") + return check_freshness(args.config, args.report, repository, token, api_url, default_branch) if __name__ == "__main__": From acfeef55abc1a68c285d23b5dd301ed08f822337 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 16:54:55 +0800 Subject: [PATCH 23/40] feat(scanner): add bounded incarnation-scoped ACK receiver (#7182) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * feat(scanner): add bounded incarnation-scoped ACK receiver Refs rustfs/backlog#2265 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/api/mod.rs | 19 +- crates/ecstore/src/bucket/metadata_sys.rs | 121 +++++++- crates/ecstore/src/cluster/rpc/client.rs | 19 ++ .../src/cluster/rpc/peer_rest_client.rs | 47 +++ .../src/generated/proto_gen/node_service.rs | 286 ++++++++++++++++++ crates/protos/src/lib.rs | 2 + crates/protos/src/node.proto | 34 +++ crates/protos/src/scoped_dirty_usage.rs | 213 +++++++++++++ crates/scanner/src/lib.rs | 5 +- crates/scanner/src/scanner_io.rs | 5 +- crates/scanner/src/scanner_io/dirty_usage.rs | 106 +++++++ crates/scanner/src/storage_api.rs | 4 +- rustfs/src/server/http.rs | 47 +++ rustfs/src/storage/rpc/node_service.rs | 200 ++++++++++++ rustfs/src/storage/storage_api.rs | 10 +- rustfs/src/storage/tonic_service.rs | 1 + rustfs/src/storage_api.rs | 2 +- 17 files changed, 1097 insertions(+), 24 deletions(-) create mode 100644 crates/protos/src/scoped_dirty_usage.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 34e784288..437f5c458 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -196,15 +196,16 @@ pub mod bucket { pub use crate::bucket::metadata_sys::ConfigWriteLockProbe; pub use crate::bucket::metadata_sys::{ BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, - acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete, - delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, - get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config, - get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config, - get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config, - get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, - get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, - remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, - update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock, + acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence, + capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get, + get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, + get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, + get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, + get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, + get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, + reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update, + update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation, + update_under_transaction_lock, }; } diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index eb40ed8c9..4a53927f9 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -655,6 +655,12 @@ pub struct BucketMetadataMutationGuard { } impl BucketMetadataMutationGuard { + /// Returns the storage-verified identity while both incarnation fences remain valid. + pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> { + self.ensure_valid(&self.bucket)?; + Ok((&self.bucket, self.incarnation_id)) + } + fn ensure_valid(&self, bucket: &str) -> Result<()> { if self.bucket != bucket { return Err(Error::other("bucket metadata mutation guard does not match bucket")); @@ -674,6 +680,29 @@ async fn acquire_config_write_guard_for_incarnation( sys: Arc>, bucket: &str, expected_incarnation_id: Option, +) -> Result { + acquire_config_write_guard_with_migration(sys, bucket, expected_incarnation_id, true).await +} + +/// Scanner probes must not create an incarnation to make a capability available. +pub async fn acquire_scanner_bucket_incarnation_fence( + bucket: &str, + expected_incarnation_id: Uuid, + expected_owner_id: Uuid, +) -> Result { + super::utils::check_valid_bucket_name(bucket)?; + let sys = get_bucket_metadata_sys()?; + if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() { + return Err(Error::other("scanner bucket incarnation owner does not match")); + } + acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await +} + +async fn acquire_config_write_guard_with_migration( + sys: Arc>, + bucket: &str, + expected_incarnation_id: Option, + migrate: bool, ) -> Result { let metadata_sys = sys.read().await.clone(); let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?; @@ -681,13 +710,15 @@ async fn acquire_config_write_guard_for_incarnation( // Legacy buckets are migrated while the lifecycle fence prevents a // same-name replacement. The second read under the write transaction is // the CAS source of truth for the actual rewrite. - await_bucket_namespace_operation( - Some(&lifecycle_guard), - bucket, - "bucket config incarnation migration", - metadata_sys.get_bucket_incarnation_id(bucket), - ) - .await?; + if migrate { + await_bucket_namespace_operation( + Some(&lifecycle_guard), + bucket, + "bucket config incarnation migration", + metadata_sys.get_bucket_incarnation_id(bucket), + ) + .await?; + } let transaction_guard = await_bucket_namespace_operation( Some(&lifecycle_guard), bucket, @@ -3176,6 +3207,82 @@ mod tests { ); } + #[tokio::test] + async fn scoped_dirty_usage_incarnation_probe_does_not_migrate_legacy_metadata() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let sys = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone()))); + let bucket = "scoped-ack-legacy"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("create legacy bucket"); + } + let mut metadata = BucketMetadata::new(bucket); + metadata.bucket_incarnation_id = Uuid::nil(); + sys.read() + .await + .persist_and_set(metadata) + .await + .expect("persist legacy metadata"); + assert!( + acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(Uuid::new_v4()), false) + .await + .is_err() + ); + assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none()); + assert!( + sys.read() + .await + .get_config_from_disk(bucket) + .await + .expect("read metadata") + .bucket_incarnation_id + .is_nil() + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn scoped_dirty_usage_incarnation_rejects_deleted_and_recreated_bucket() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner"); + let bucket = "scoped-ack-recreated"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let old = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation"); + let guard = acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false) + .await + .expect("trusted incarnation fence"); + assert_eq!(guard.checked_bucket_incarnation().expect("valid fences"), (bucket, old)); + drop(guard); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("delete bucket"); + assert!( + acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false) + .await + .is_err() + ); + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("recreate bucket"); + let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation"); + assert_ne!(old, new); + assert!( + acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false) + .await + .is_err() + ); + assert!( + acquire_config_write_guard_with_migration(sys, bucket, Some(new), false) + .await + .is_ok() + ); + } + #[tokio::test] async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() { let (dirs, ecstore) = isolated_store_over_temp_disks().await; diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 435cbf64c..c4bf786f9 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -30,6 +30,7 @@ use rustfs_protos::{ ChannelClass, create_new_channel, get_channel_for_class, proto_gen::node_service::{ heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient, + scanner_control_service_client::ScannerControlServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient, }, }; @@ -60,6 +61,24 @@ pub async fn node_service_time_out_client( node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await } +pub(crate) async fn scanner_control_time_out_client( + addr: &str, + interceptor: TonicInterceptor, +) -> crate::error::Result>> { + let interceptor = interceptor.with_rpc_audience(addr)?; + let channel = match runtime_sources::cached_node_channel(addr).await { + Some(channel) => channel, + None => create_new_channel(addr) + .await + .map_err(|err| crate::error::Error::other(err.to_string()))?, + }; + let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); + let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize; + Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor) + .max_decoding_message_size(limit) + .max_encoding_message_size(limit)) +} + pub async fn heal_control_time_out_client( addr: &str, interceptor: TonicInterceptor, diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 5316199e3..9d57c4647 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -2050,6 +2050,53 @@ impl PeerRestClient { .await } + /// Probe only: scoped ACK production requires a durable per-bucket proof. + pub async fn scanner_scoped_dirty_usage_capability( + &self, + owner_id: String, + instance_id: String, + entries: Vec, + ) -> Result { + use rustfs_protos::scoped_dirty_usage::*; + let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest { + challenge: Uuid::new_v4().as_bytes().to_vec().into(), + protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION, + owner_id, + instance_id, + scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE, + probe_only: true, + entries, + }; + let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?; + self.finalize_result( + async { + let mut client = super::client::scanner_control_time_out_client( + &self.grid_host, + TonicInterceptor::Signature(gen_tonic_signature_interceptor()), + ) + .await?; + let mut request = Request::new(payload.clone()); + set_tonic_canonical_body_digest(&mut request, &canonical)?; + let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner(); + let body = canonical_scoped_dirty_usage_response(&canonical, &response) + .map_err(|_| Error::other("scoped dirty usage capability response is too large"))?; + verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?; + if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION + || response.owner_id != payload.owner_id + || response.instance_id != payload.instance_id + || response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES + || response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES + || response.cleared != 0 + { + return Err(Error::other("scoped dirty usage capability response does not match request")); + } + Ok(response.supported) + } + .await, + ) + .await + } + pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result { let result = self .scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION) diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index f015305f0..fe04d93ab 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -1283,6 +1283,54 @@ pub struct ScannerDirtyUsageSnapshotResponse { #[prost(bytes = "bytes", tag = "7")] pub response_proof: ::prost::bytes::Bytes, } +/// Receiver-only protocol. Producers must retain whole-cycle ACK until they +/// have a durable per-bucket publication proof. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScannerScopedDirtyUsageEntry { + #[prost(string, tag = "1")] + pub bucket: ::prost::alloc::string::String, + #[prost(bytes = "bytes", tag = "2")] + pub bucket_incarnation: ::prost::bytes::Bytes, + #[prost(uint64, tag = "3")] + pub generation: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ScannerScopedDirtyUsageAckRequest { + #[prost(bytes = "bytes", tag = "1")] + pub challenge: ::prost::bytes::Bytes, + #[prost(uint32, tag = "2")] + pub protocol_version: u32, + #[prost(string, tag = "3")] + pub owner_id: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub instance_id: ::prost::alloc::string::String, + /// Only scope 1 (a complete bucket) is supported; zero is invalid. + #[prost(uint32, tag = "5")] + pub scope: u32, + #[prost(bool, tag = "6")] + pub probe_only: bool, + #[prost(message, repeated, tag = "7")] + pub entries: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScannerScopedDirtyUsageAckResponse { + #[prost(uint32, tag = "1")] + pub protocol_version: u32, + #[prost(string, tag = "2")] + pub owner_id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub instance_id: ::prost::alloc::string::String, + #[prost(bool, tag = "4")] + pub supported: bool, + #[prost(uint32, tag = "5")] + pub max_entries: u32, + #[prost(uint32, tag = "6")] + pub max_request_bytes: u32, + #[prost(uint64, tag = "7")] + pub cleared: u64, + #[prost(bytes = "bytes", tag = "8")] + pub response_proof: ::prost::bytes::Bytes, +} /// A short-lived storage-owned read admission used only around a final /// scanner metadata publication. It is intentionally separate from the /// ScannerActivity observation wire so v6/v7 rolling compatibility remains @@ -6282,6 +6330,244 @@ pub mod node_service_server { } } /// Generated client implementations. +pub mod scanner_control_service_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct ScannerControlServiceClient { + inner: tonic::client::Grpc, + } + impl ScannerControlServiceClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl ScannerControlServiceClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor(inner: T, interceptor: F) -> ScannerControlServiceClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response<>::ResponseBody>, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + ScannerControlServiceClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn scanner_scoped_dirty_usage_ack( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.ScannerControlService", "ScannerScopedDirtyUsageAck")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod scanner_control_service_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with ScannerControlServiceServer. + #[async_trait] + pub trait ScannerControlService: std::marker::Send + std::marker::Sync + 'static { + async fn scanner_scoped_dirty_usage_ack( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct ScannerControlServiceServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl ScannerControlServiceServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for ScannerControlServiceServer + where + T: ScannerControlService, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck" => { + #[allow(non_camel_case_types)] + struct ScannerScopedDirtyUsageAckSvc(pub Arc); + impl tonic::server::UnaryService + for ScannerScopedDirtyUsageAckSvc + { + type Response = super::ScannerScopedDirtyUsageAckResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::scanner_scoped_dirty_usage_ack(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ScannerScopedDirtyUsageAckSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into()); + headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE); + Ok(response) + }), + } + } + } + impl Clone for ScannerControlServiceServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "node_service.ScannerControlService"; + impl tonic::server::NamedService for ScannerControlServiceServer { + const NAME: &'static str = SERVICE_NAME; + } +} +/// Generated client implementations. pub mod heal_control_service_client { #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] use tonic::codegen::http::Uri; diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index fd8704382..81ead7d65 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -541,6 +541,8 @@ pub fn canonical_scanner_activity_v7_response_body( Ok(body) } +pub mod scoped_dirty_usage; + pub fn canonical_scanner_dirty_usage_snapshot_request_body( request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest, ) -> Result, std::num::TryFromIntError> { diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index d3796991c..abc030adc 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -903,6 +903,36 @@ message ScannerDirtyUsageSnapshotResponse { bytes response_proof = 7; } +// Receiver-only protocol. Producers must retain whole-cycle ACK until they +// have a durable per-bucket publication proof. +message ScannerScopedDirtyUsageEntry { + string bucket = 1; + bytes bucket_incarnation = 2; + uint64 generation = 3; +} + +message ScannerScopedDirtyUsageAckRequest { + bytes challenge = 1; + uint32 protocol_version = 2; + string owner_id = 3; + string instance_id = 4; + // Only scope 1 (a complete bucket) is supported; zero is invalid. + uint32 scope = 5; + bool probe_only = 6; + repeated ScannerScopedDirtyUsageEntry entries = 7; +} + +message ScannerScopedDirtyUsageAckResponse { + uint32 protocol_version = 1; + string owner_id = 2; + string instance_id = 3; + bool supported = 4; + uint32 max_entries = 5; + uint32 max_request_bytes = 6; + uint64 cleared = 7; + bytes response_proof = 8; +} + // A short-lived storage-owned read admission used only around a final // scanner metadata publication. It is intentionally separate from the // ScannerActivity observation wire so v6/v7 rolling compatibility remains @@ -1245,6 +1275,10 @@ service NodeService { rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only } +service ScannerControlService { + rpc ScannerScopedDirtyUsageAck(ScannerScopedDirtyUsageAckRequest) returns (ScannerScopedDirtyUsageAckResponse) {}; // auth-policy: body-bound +} + service HealControlService { rpc HealControl(HealControlRequest) returns (HealControlResponse) {}; } diff --git a/crates/protos/src/scoped_dirty_usage.rs b/crates/protos/src/scoped_dirty_usage.rs new file mode 100644 index 000000000..ac3effaf7 --- /dev/null +++ b/crates/protos/src/scoped_dirty_usage.rs @@ -0,0 +1,213 @@ +// Copyright 2024 RustFS Team +// Licensed under the Apache License, Version 2.0. + +//! Bounded, authenticated receiver contract for per-bucket dirty acknowledgements. + +use crate::CanonicalBodyBuilder; +use crate::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse}; +use prost::Message; + +pub const SCOPED_DIRTY_USAGE_PROTOCOL_VERSION: u32 = 1; +pub const SCOPED_DIRTY_USAGE_BUCKET_SCOPE: u32 = 1; +pub const SCOPED_DIRTY_USAGE_MAX_ENTRIES: u32 = 32; +pub const SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES: u32 = 8192; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScopedDirtyUsageRequestError { + UnsupportedProtocol, + UnsupportedScope, + InvalidIdentity, + InvalidGeneration, + InvalidEntries, + TooLarge, +} + +impl std::fmt::Display for ScopedDirtyUsageRequestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::UnsupportedProtocol => "unsupported scoped dirty usage protocol", + Self::UnsupportedScope => "unsupported scoped dirty usage scope", + Self::InvalidIdentity => "invalid scoped dirty usage identity", + Self::InvalidGeneration => "invalid scoped dirty usage generation", + Self::InvalidEntries => "scoped dirty usage entries must be nonempty and strictly ordered", + Self::TooLarge => "scoped dirty usage request exceeds its budget", + }) + } +} + +impl std::error::Error for ScopedDirtyUsageRequestError {} + +pub fn validate_scoped_dirty_usage_request( + request: &ScannerScopedDirtyUsageAckRequest, +) -> Result<(), ScopedDirtyUsageRequestError> { + use ScopedDirtyUsageRequestError as E; + if request.entries.len() > SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize + || request.encoded_len() > SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize + { + return Err(E::TooLarge); + } + if request.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION { + return Err(E::UnsupportedProtocol); + } + if request.scope != SCOPED_DIRTY_USAGE_BUCKET_SCOPE { + return Err(E::UnsupportedScope); + } + if request.challenge.len() != 16 || request.owner_id.len() != 36 || request.instance_id.len() != 32 { + return Err(E::InvalidIdentity); + } + if request.entries.is_empty() || request.entries.windows(2).any(|pair| pair[0].bucket >= pair[1].bucket) { + return Err(E::InvalidEntries); + } + for entry in &request.entries { + if entry.bucket.is_empty() + || entry.bucket.len() > 63 + || entry.bucket_incarnation.len() != 16 + || entry.bucket_incarnation.iter().all(|byte| *byte == 0) + { + return Err(E::InvalidIdentity); + } + if entry.generation == 0 || entry.generation == u64::MAX { + return Err(E::InvalidGeneration); + } + } + Ok(()) +} + +pub fn canonical_scoped_dirty_usage_request( + request: &ScannerScopedDirtyUsageAckRequest, +) -> Result, ScopedDirtyUsageRequestError> { + validate_scoped_dirty_usage_request(request)?; + let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-request-v1\0"); + let encode = |_: std::num::TryFromIntError| ScopedDirtyUsageRequestError::TooLarge; + body.push_bytes(request.challenge.as_ref()).map_err(encode)?; + body.push_u32(request.protocol_version); + body.push_str(&request.owner_id).map_err(encode)?; + body.push_str(&request.instance_id).map_err(encode)?; + body.push_u32(request.scope); + body.push_bool(request.probe_only); + body.push_count(request.entries.len()).map_err(encode)?; + for entry in &request.entries { + body.push_str(&entry.bucket).map_err(encode)?; + body.push_bytes(entry.bucket_incarnation.as_ref()).map_err(encode)?; + body.push_u64(entry.generation); + } + Ok(body.finish()) +} + +pub fn canonical_scoped_dirty_usage_response( + request_body: &[u8], + response: &ScannerScopedDirtyUsageAckResponse, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-response-v1\0"); + body.push_bytes(request_body)?; + body.push_u32(response.protocol_version); + body.push_str(&response.owner_id)?; + body.push_str(&response.instance_id)?; + body.push_bool(response.supported); + body.push_u32(response.max_entries); + body.push_u32(response.max_request_bytes); + body.push_u64(response.cleared); + Ok(body.finish()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::proto_gen::node_service::ScannerScopedDirtyUsageEntry; + + fn request() -> ScannerScopedDirtyUsageAckRequest { + ScannerScopedDirtyUsageAckRequest { + challenge: vec![1; 16].into(), + protocol_version: 1, + owner_id: "11111111-1111-1111-1111-111111111111".into(), + instance_id: "a".repeat(32), + scope: 1, + probe_only: false, + entries: vec![ScannerScopedDirtyUsageEntry { + bucket: "photos".into(), + bucket_incarnation: vec![2; 16].into(), + generation: 8, + }], + } + } + + #[test] + fn scoped_dirty_usage_binds_every_request_field() { + let base = request(); + let baseline = canonical_scoped_dirty_usage_request(&base).expect("valid request"); + for field in 0..9 { + let mut changed = base.clone(); + match field { + 0 => changed.challenge = vec![3; 16].into(), + 1 => changed.protocol_version += 1, + 2 => changed.owner_id = "22222222-2222-2222-2222-222222222222".into(), + 3 => changed.instance_id = "b".repeat(32), + 4 => changed.scope += 1, + 5 => changed.probe_only = true, + 6 => changed.entries[0].bucket = "videos".into(), + 7 => changed.entries[0].bucket_incarnation = vec![3; 16].into(), + _ => changed.entries[0].generation += 1, + } + assert!(canonical_scoped_dirty_usage_request(&changed).map_or(true, |body| body != baseline)); + } + } + + #[test] + fn scoped_dirty_usage_binds_capability_and_ack_to_exact_request() { + let request = canonical_scoped_dirty_usage_request(&request()).expect("valid request"); + let response = ScannerScopedDirtyUsageAckResponse { + protocol_version: 1, + owner_id: "owner".into(), + instance_id: "process".into(), + supported: true, + max_entries: 32, + max_request_bytes: 8192, + cleared: 1, + response_proof: vec![1; 32].into(), + }; + let baseline = canonical_scoped_dirty_usage_response(&request, &response).expect("valid response"); + for field in 0..7 { + let mut changed = response.clone(); + match field { + 0 => changed.protocol_version += 1, + 1 => changed.owner_id.push('x'), + 2 => changed.instance_id.push('x'), + 3 => changed.supported = false, + 4 => changed.max_entries += 1, + 5 => changed.max_request_bytes += 1, + _ => changed.cleared += 1, + } + assert_ne!( + canonical_scoped_dirty_usage_response(&request, &changed).expect("response variant"), + baseline + ); + } + assert_ne!( + canonical_scoped_dirty_usage_response(b"another request", &response).expect("request variant"), + baseline + ); + } + + #[test] + fn scoped_dirty_usage_rejects_overflow_unknown_and_duplicate_entries() { + let base = request(); + let mut invalid = base.clone(); + invalid.entries = vec![base.entries[0].clone(); SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize + 1]; + assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge)); + invalid = base.clone(); + invalid.entries[0].bucket = "x".repeat(SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize); + assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge)); + invalid = base.clone(); + invalid.entries.push(base.entries[0].clone()); + assert_eq!( + validate_scoped_dirty_usage_request(&invalid), + Err(ScopedDirtyUsageRequestError::InvalidEntries) + ); + invalid = base; + invalid.entries[0].bucket_incarnation = vec![0; 16].into(); + assert_eq!( + validate_scoped_dirty_usage_request(&invalid), + Err(ScopedDirtyUsageRequestError::InvalidIdentity) + ); + } +} diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index f85a2e0b6..28cf638e9 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -90,8 +90,9 @@ pub use scanner::{ }; pub use scanner_io::{ ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, - acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, - scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation, + acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket, + record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, + scanner_maintenance_generation, }; pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER}; use std::sync::atomic::{AtomicU64, Ordering}; diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index c8353bae9..0d5e82eeb 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -883,8 +883,9 @@ pub(crate) use cache::{ }; pub use dirty_usage::{ ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, - acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, - scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation, + acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket, + record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, + scanner_maintenance_generation, }; #[cfg(test)] pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests}; diff --git a/crates/scanner/src/scanner_io/dirty_usage.rs b/crates/scanner/src/scanner_io/dirty_usage.rs index a5978e263..18db78a81 100644 --- a/crates/scanner/src/scanner_io/dirty_usage.rs +++ b/crates/scanner/src/scanner_io/dirty_usage.rs @@ -52,6 +52,112 @@ pub enum ScannerDirtyUsageAckError { ProcessChanged, #[error("scanner dirty usage generation cannot be acknowledged")] InvalidGeneration, + #[error("scanner dirty usage bucket incarnation fence is unavailable")] + IncarnationUnavailable, +} + +/// A scoped ACK requires storage-owned lifecycle and incarnation fences. +/// Callers must only send ACKs backed by durable per-bucket publication. +pub fn acknowledge_scoped_dirty_usage( + instance_id: &str, + entries: &[(&crate::storage_api::EcstoreBucketMetadataMutationGuard, u64)], + probe_only: bool, +) -> std::result::Result { + // Lock order: sorted bucket lifecycle/metadata fences (caller), then dirty map. + // No await or storage operation occurs while the dirty map is locked. + let (cleared, pending) = { + let mut dirty = dirty_usage_buckets(); + let checked = entries + .iter() + .map(|(guard, generation)| { + guard + .checked_bucket_incarnation() + .map(|(bucket, _)| (bucket, *generation)) + .map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable) + }) + .collect::, _>>()?; + let cleared = apply_scoped_dirty_usage_ack( + instance_id, + scanner_activity_epoch(), + DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire), + &mut dirty, + &checked, + probe_only, + )?; + if cleared > 0 { + advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); + } + (cleared, dirty.len()) + }; + if !probe_only { + global_metrics().record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared), usize_to_u64_saturated(pending)); + } + Ok(usize_to_u64_saturated(cleared)) +} + +fn apply_scoped_dirty_usage_ack( + instance_id: &str, + current_instance: &str, + current_generation: u64, + dirty: &mut DirtyUsageBuckets, + entries: &[(&str, u64)], + probe_only: bool, +) -> std::result::Result { + if instance_id != current_instance { + return Err(ScannerDirtyUsageAckError::ProcessChanged); + } + if current_generation == u64::MAX + || entries + .iter() + .any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation) + { + return Err(ScannerDirtyUsageAckError::InvalidGeneration); + } + let mut cleared = 0; + if !probe_only { + for (bucket, generation) in entries { + if dirty.get(*bucket) == Some(generation) { + dirty.remove(*bucket); + cleared += 1; + } + } + } + Ok(cleared) +} + +#[cfg(test)] +mod scoped_dirty_usage_tests { + use super::*; + + #[test] + fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() { + let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]); + assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], true), Ok(0)); + assert_eq!(dirty.len(), 2); + assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(1)); + assert_eq!(dirty.get("hot"), Some(&7)); + assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(0)); + dirty.insert("cold".to_string(), 9); + assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 9, &mut dirty, &[("cold", 8)], false), Ok(0)); + assert_eq!(dirty.get("cold"), Some(&9)); + } + + #[test] + fn scoped_dirty_usage_rejects_restart_and_invalid_batch_before_clearing() { + let original = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]); + let mut dirty = original.clone(); + assert_eq!( + apply_scoped_dirty_usage_ack("old", "new", 8, &mut dirty, &[("cold", 8)], false), + Err(ScannerDirtyUsageAckError::ProcessChanged) + ); + for generation in [0, 9, u64::MAX] { + assert_eq!( + apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8), ("hot", generation)], false), + Err(ScannerDirtyUsageAckError::InvalidGeneration) + ); + assert_eq!(dirty, original); + } + } } pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> { diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index f81065d59..7b2cda493 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -38,8 +38,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_fr #[cfg(test)] pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys; pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{ - get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config, - get_replication_config as ecstore_get_replication_config, + BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard, get_lifecycle_config as ecstore_get_lifecycle_config, + get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config, }; pub(crate) use rustfs_ecstore::api::bucket::replication::{ ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig, diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 78ce81e5b..d82d67f20 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -208,6 +208,7 @@ const EVENT_PEER_ADDR_UNAVAILABLE: &str = "peer_addr_unavailable"; const EVENT_RPC_SIGNATURE_VERIFICATION_FAILED: &str = "rpc_signature_verification_failed"; const EVENT_GRPC_TRACE_CONTEXT_PROPAGATION_FAILED: &str = "grpc_trace_context_propagation_failed"; const HEAL_CONTROL_TONIC_RPC_PATH: &str = "/node_service.HealControlService/HealControl"; +const SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH: &str = "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"; const TIER_MUTATION_PREPARE_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/PrepareTierMutation"; const TIER_MUTATION_COMMIT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/CommitTierMutation"; const TIER_MUTATION_ABORT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/AbortTierMutation"; @@ -1856,6 +1857,7 @@ fn process_connection( ); let rpc_service = RpcRequestPathService::new( Routes::new(node_service) + .add_service(InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth)) .add_service(heal_control_service) .add_service(tier_mutation_control_service) .prepare(), @@ -2259,6 +2261,7 @@ fn check_auth(req: Request<()>) -> std::result::Result, Status> { .strip_prefix(TONIC_RPC_PREFIX) .and_then(|suffix| suffix.strip_prefix('/')) .or_else(|| (target.uri.path() == HEAL_CONTROL_TONIC_RPC_PATH).then_some("HealControl")) + .or_else(|| (target.uri.path() == SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH).then_some("ScannerScopedDirtyUsageAck")) .or_else(|| (target.uri.path() == TIER_MUTATION_PREPARE_TONIC_RPC_PATH).then_some("PrepareTierMutation")) .or_else(|| (target.uri.path() == TIER_MUTATION_COMMIT_TONIC_RPC_PATH).then_some("CommitTierMutation")) .or_else(|| (target.uri.path() == TIER_MUTATION_ABORT_TONIC_RPC_PATH).then_some("AbortTierMutation")) @@ -3427,6 +3430,50 @@ mod tests { rustfs_common::set_global_local_node_name(&previous_node_name).await; } + #[tokio::test] + #[serial_test::serial] + async fn scoped_dirty_usage_peer_probe_reaches_handler_through_production_auth() { + let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string()); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scoped ACK auth test"); + let addr = listener.local_addr().expect("listener address"); + let previous_node_name = rustfs_common::get_global_local_node_name().await; + rustfs_common::set_global_local_node_name(&addr.to_string()).await; + let node = InterceptedService::new(NodeServiceServer::new(make_server()), check_auth); + let scanner = InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth); + let service = RpcRequestPathService::new(Routes::new(node).add_service(scanner).prepare()); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.expect("accept test connection"); + ConnBuilder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(socket), TowerToHyperService::new(service)) + .await + .expect("serve scoped ACK auth test"); + }); + let host = rustfs_utils::XHost::try_from(addr.to_string()).expect("peer address"); + let client = storage::PeerRestClient::new(host, format!("http://{addr}")); + let result = client + .scanner_scoped_dirty_usage_capability( + "11111111-1111-1111-1111-111111111111".to_string(), + "a".repeat(32), + vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry { + bucket: "photos".into(), + bucket_incarnation: vec![1; 16].into(), + generation: 8, + }], + ) + .await; + client.evict_connection().await; + server.abort(); + let _ = server.await; + rustfs_common::set_global_local_node_name(&previous_node_name).await; + let error = result + .expect_err("probe must fail closed without the requested storage owner") + .to_string(); + assert!( + error.contains("storage layer is not initialized") || error.contains("scoped dirty usage peer or process changed"), + "signed probe must pass production path authentication and reach owner validation: {error}" + ); + } + #[tokio::test] #[serial_test::serial] async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() { diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 0dd18424e..111726af3 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -493,6 +493,13 @@ impl std::fmt::Debug for NodeService { } } +pub(crate) fn make_scanner_control_server() -> scanner_control_service_server::ScannerControlServiceServer { + let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize; + scanner_control_service_server::ScannerControlServiceServer::new(make_server()) + .max_decoding_message_size(limit) + .max_encoding_message_size(limit) +} + pub fn make_server() -> NodeService { let context = runtime_sources::current_app_context(); make_server_for_context(context) @@ -1087,6 +1094,74 @@ impl NodeService { } } +#[tonic::async_trait] +impl scanner_control_service_server::ScannerControlService for NodeService { + async fn scanner_scoped_dirty_usage_ack( + &self, + request: Request, + ) -> Result, Status> { + use rustfs_protos::scoped_dirty_usage::*; + static ADMISSION: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); + + let canonical = + canonical_scoped_dirty_usage_request(request.get_ref()).map_err(|err| Status::invalid_argument(err.to_string()))?; + verify_tonic_canonical_body_digest(&request, &canonical) + .map_err(|_| Status::permission_denied("scoped dirty usage authentication failed"))?; + let _admission = ADMISSION + .try_acquire() + .map_err(|_| Status::resource_exhausted("scoped dirty usage receiver is busy"))?; + let request = request.into_inner(); + let store = self + .resolve_object_store() + .ok_or_else(|| Status::unavailable("storage layer is not initialized"))?; + if store.id.is_nil() + || request.owner_id != store.id.to_string() + || request.instance_id != rustfs_scanner::scanner_activity_epoch() + { + return Err(Status::failed_precondition("scoped dirty usage peer or process changed")); + } + let cleared = timeout(Duration::from_secs(30), async { + // Strict bucket order is validated before admission. Acquire every + // lifecycle/metadata fence before clearing any dirty record. + let mut guards = Vec::with_capacity(request.entries.len()); + for entry in &request.entries { + let incarnation = Uuid::from_slice(entry.bucket_incarnation.as_ref()) + .map_err(|_| Status::invalid_argument("invalid bucket incarnation"))?; + let guard = + crate::storage::storage_api::acquire_scanner_bucket_incarnation_fence(&entry.bucket, incarnation, store.id) + .await + .map_err(|_| Status::failed_precondition("trusted bucket incarnation is unavailable"))?; + guards.push(guard); + } + let entries = guards + .iter() + .zip(&request.entries) + .map(|(guard, entry)| (guard, entry.generation)) + .collect::>(); + rustfs_scanner::acknowledge_scoped_dirty_usage(&request.instance_id, &entries, request.probe_only) + .map_err(|err| Status::failed_precondition(err.to_string())) + }) + .await + .map_err(|_| Status::deadline_exceeded("scoped dirty usage incarnation validation timed out"))??; + let mut response = ScannerScopedDirtyUsageAckResponse { + protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION, + owner_id: request.owner_id, + instance_id: request.instance_id, + supported: true, + max_entries: SCOPED_DIRTY_USAGE_MAX_ENTRIES, + max_request_bytes: SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES, + cleared, + response_proof: Bytes::new(), + }; + let body = canonical_scoped_dirty_usage_response(&canonical, &response) + .map_err(|_| Status::internal("scoped dirty usage response is too large"))?; + response.response_proof = sign_tonic_rpc_response_proof(&body) + .map_err(|_| Status::unavailable("scoped dirty usage response authentication is unavailable"))? + .into(); + Ok(Response::new(response)) + } +} + #[tonic::async_trait] impl Node for NodeService { async fn ping(&self, request: Request) -> Result, Status> { @@ -2623,6 +2698,7 @@ mod tests { use rustfs_kms::KmsServiceManager; use rustfs_protos::CanonicalMutationBody as _; use rustfs_protos::models::PingBodyBuilder; + use rustfs_protos::proto_gen::node_service::scanner_control_service_server::ScannerControlService as _; use rustfs_protos::proto_gen::node_service::{ BackgroundHealStatusRequest, BatchGenerallyLockRequest, CancelDecommissionRequest, CheckPartsRequest, ClearDecommissionRequest, ControlPlaneErrorCode, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest, @@ -5990,6 +6066,74 @@ mod tests { ); } + fn scoped_dirty_usage_request() -> rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest { + rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest { + challenge: vec![7; 16].into(), + protocol_version: 1, + owner_id: "11111111-1111-1111-1111-111111111111".into(), + instance_id: "a".repeat(32), + scope: 1, + probe_only: false, + entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry { + bucket: "photos".into(), + bucket_incarnation: vec![1; 16].into(), + generation: 8, + }], + } + } + + #[tokio::test] + async fn scoped_dirty_usage_authenticates_before_storage_and_rejects_tampering() { + use rustfs_protos::scoped_dirty_usage::canonical_scoped_dirty_usage_request; + let service = create_test_node_service(); + let unsigned = service + .scanner_scoped_dirty_usage_ack(Request::new(scoped_dirty_usage_request())) + .await + .expect_err("unsigned ACK must not access storage"); + assert_eq!(unsigned.code(), tonic::Code::PermissionDenied); + for field in 0..9 { + let mut signed = Request::new(scoped_dirty_usage_request()); + let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request"); + set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest"); + mark_v2_authenticated(&mut signed); + match field { + 0 => signed.get_mut().challenge = vec![3; 16].into(), + 1 => signed.get_mut().owner_id = "22222222-2222-2222-2222-222222222222".into(), + 2 => signed.get_mut().instance_id = "b".repeat(32), + 3 => signed.get_mut().probe_only = true, + 4 => signed.get_mut().entries[0].bucket = "videos".into(), + 5 => signed.get_mut().entries[0].bucket_incarnation = vec![2; 16].into(), + 6 => signed.get_mut().entries[0].generation += 1, + 7 => signed.get_mut().scope += 1, + _ => signed.get_mut().protocol_version += 1, + } + let error = service + .scanner_scoped_dirty_usage_ack(signed) + .await + .expect_err("tampered ACK must fail"); + assert_eq!( + error.code(), + if field < 7 { + tonic::Code::PermissionDenied + } else { + tonic::Code::InvalidArgument + } + ); + } + let mut signed = Request::new(scoped_dirty_usage_request()); + let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request"); + set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest"); + mark_v2_authenticated(&mut signed); + assert_eq!( + service + .scanner_scoped_dirty_usage_ack(signed) + .await + .expect_err("missing owner cannot advertise capability") + .code(), + tonic::Code::Unavailable + ); + } + #[tokio::test] async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() { let service = create_test_node_service(); @@ -6485,6 +6629,62 @@ mod tests { ) } + #[tokio::test] + async fn scoped_dirty_usage_transport_rejects_oversized_unknown_and_duplicate_fields() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind scoped ACK transport test"); + let addr = listener.local_addr().expect("test listener address"); + let (shutdown, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(super::make_scanner_control_server()) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + let _ = stopped.await; + }) + .await + .expect("scoped ACK transport server"); + }); + let client = reqwest::Client::builder() + .no_proxy() + .http2_prior_knowledge() + .build() + .expect("HTTP/2 client"); + let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize; + for tag in [0x78, 0x0a] { + // Unknown varint field 15, or repeated empty singular challenge: + // both decode to a tiny default struct despite the large wire body. + for oversized in [false, true] { + let mut payload = [tag, 0].repeat(if oversized { (limit - 4) / 2 } else { limit / 2 }); + if oversized { + // Unknown fixed32 field 15 makes a valid cap+1 protobuf. + payload.extend_from_slice(&[0x7d, 0, 0, 0, 0]); + } + assert_eq!(payload.len(), limit + usize::from(oversized)); + let mut frame = vec![0]; + frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test payload").to_be_bytes()); + frame.extend_from_slice(&payload); + let response = client + .post(format!("http://{addr}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck")) + .header("content-type", "application/grpc") + .header("te", "trailers") + .body(frame) + .send() + .await + .expect("send raw protobuf frame"); + let status = response.headers().get("grpc-status").expect("gRPC failure status"); + assert_eq!( + status.to_str().expect("status text"), + if oversized { "11" } else { "3" }, + "cap+1 must fail in the codec, while cap bytes reach request validation" + ); + } + } + drop(client); + shutdown.send(()).expect("stop test server"); + server.await.expect("join test server"); + } + #[tokio::test] async fn heal_control_transport_enforces_codec_limit_and_fails_closed() { let Some(mut client) = connect_test_heal_control_client().await else { diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 9e1e92dc4..2216db7f4 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -379,7 +379,7 @@ pub(crate) mod tonic_service_consumer { #[cfg(test)] pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source}; pub(crate) use super::super::tonic_service::{ - make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server, + make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server, }; } @@ -1704,6 +1704,14 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock( ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await } +pub(crate) async fn acquire_scanner_bucket_incarnation_fence( + bucket: &str, + incarnation: uuid::Uuid, + owner_id: uuid::Uuid, +) -> Result { + ecstore_bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence(bucket, incarnation, owner_id).await +} + pub(crate) async fn update_bucket_targets_under_transaction_lock( guard: &ecstore_bucket::metadata_sys::BucketMetadataMutationGuard, bucket: &str, diff --git a/rustfs/src/storage/tonic_service.rs b/rustfs/src/storage/tonic_service.rs index 539361507..c7d2ee28a 100644 --- a/rustfs/src/storage/tonic_service.rs +++ b/rustfs/src/storage/tonic_service.rs @@ -13,6 +13,7 @@ // limitations under the License. pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache; +pub(crate) use crate::storage::rpc::node_service::make_scanner_control_server; #[cfg(test)] pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source}; pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server}; diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index db459b72d..4f824f8da 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -176,7 +176,7 @@ pub(crate) mod server { heal_topology_fingerprint, make_heal_control_server_for_source, }; pub(crate) use crate::storage::storage_api::tonic_service_consumer::{ - make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server, + make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server, }; } } From 7ba5cd6888b367423357ef3e5211798c87bab3be Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 17:06:52 +0800 Subject: [PATCH 24/40] chore(deps): refresh SDKs and verify clock skew behavior (#7174) chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-authored-by: heihutu Co-authored-by: zhi22915 From e8a7f4bc4ab9f8e22f6234927f0dde9898d326d7 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 18:44:01 +0800 Subject: [PATCH 25/40] fix(ecstore): remove duplicate local rename implementation (#7190) * fix(ecstore): remove duplicate local rename implementation Keep the canonical commit module after concurrent storage changes merged. The control-write and rollback changes are already present there. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ci): satisfy new clippy lints --------- Co-authored-by: heihutu Co-authored-by: zhi22915 Co-authored-by: Zhengchao An --- crates/ecstore/src/disk/local.rs | 997 --------------------------- rustfs/src/app/object/shared.rs | 2 +- rustfs/src/site_replication/tests.rs | 5 +- 3 files changed, 3 insertions(+), 1001 deletions(-) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index bf2239a9f..9cd683578 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -9862,1004 +9862,7 @@ fn should_read_legacy_inline_part(fi: &FileInfo, storage_class_config: &crate::c storage_class_config.should_inline(shard_size, fi.erasure.data_blocks, versioned) } -/// Proof produced only when the local rename returns at an existing access -/// preflight, before metadata, backups, or object data can be published. -#[derive(Debug)] -pub(in crate::disk) struct LocalRenamePreflightRejection(()); - impl LocalDisk { - #[tracing::instrument(name = "rename_data", level = "trace", skip_all)] - async fn rename_data_inner( - &self, - src_volume: &str, - src_path: &str, - fi: FileInfo, - dst_volume: &str, - dst_path: &str, - preflight_rejection: &mut Option, - ) -> Result { - crate::hp_guard!("LocalDisk::rename_data"); - let mut fi = fi; - // A non-force DeleteBucket must not remove a directory while a local - // object commit is publishing into it. The peer's empty scan remains - // optimistic; this lease establishes the local commit/delete order and - // remains owned by any blocking syscall that outlives async cancellation. - let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; - let quota_fence_token = - match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { - Some(value) => { - let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; - Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) - } - None if rustfs_utils::http::metadata_compat::contains_key_str( - &fi.metadata, - QUOTA_MUTATION_FENCE_METADATA_SUFFIX, - ) => - { - return Err(DiskError::FileCorrupt); - } - None => None, - }; - rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); - let quota_fence_claim = match quota_fence_token { - Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), - None => None, - }; - let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; - if let Some(claim) = quota_fence_claim { - mutation_lease.attach_external_guard(claim); - } - if fi.is_legacy_indexed_delete_marker() { - fi.erasure.index = 0; - } - fi.validate_for_metadata_read()?; - // Snapshot the destination part paths before `fi` is consumed below. These - // are the descriptors a reader may hold for the version this call is about - // to replace (backlog#1145); readers build the identical string in - // `io_primitives`. An inline-data version has no parts and yields none. - let invalidate_part_paths: Vec = { - let data_dir = fi.data_dir.unwrap_or_default(); - fi.parts - .iter() - .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) - .collect() - }; - let src_volume_dir = self.io_get_bucket_path(src_volume)?; - if !skip_access_checks(src_volume) - && let Err(e) = super::fs::access_std(&src_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?src_volume_dir, - operation = "rename_data_src_access", - error = %e, - "Disk local access check failed" - ); - *preflight_rejection = Some(LocalRenamePreflightRejection(())); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; - if !skip_access_checks(dst_volume) - && let Err(e) = super::fs::access_std(&dst_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?dst_volume_dir, - operation = "rename_data_dst_access", - error = %e, - "Disk local access check failed" - ); - *preflight_rejection = Some(LocalRenamePreflightRejection(())); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - // xl.meta path - let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; - let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; - - // data_dir path - let has_data_dir_path = { - let has_data_dir = { - if !fi.is_remote() { - fi.data_dir - .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) - } else { - None - } - }; - - if let Some(data_dir) = has_data_dir { - let src_data_path = self.io_get_object_path( - src_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), - )?; - let dst_data_path = self.io_get_object_path( - dst_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), - )?; - - Some((src_data_path, dst_data_path)) - } else { - None - } - }; - - check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; - check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - - let no_inline = fi.data.is_none() && fi.size > 0; - // Captured before `fi` is consumed by add_version; gates the stale - // destination purge below. - let fi_healing = fi.is_healing(); - - // Resolved once for the whole commit so a concurrent configuration - // change can never leave a single rename_data half-synced. The tier is - // keyed on the destination volume: user data staged in scratch - // namespaces follows the configured tier, while commits into - // system-critical namespaces (IAM, config, bucket metadata) stay - // pinned to strict. - let durability = effective_durability(dst_volume); - - let src_file_parent = src_file_path - .parent() - .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; - let dst_file_parent = dst_file_path - .parent() - .ok_or_else(|| DiskError::other("missing object metadata parent"))?; - if !no_inline { - fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; - } - // Acquire the common trees before reading destination metadata. On - // Windows this pins the object directory identity across metadata - // preparation, data publication, rollback backup, and final commit. - let rename_commit_guard = lock_rename_commit_directories( - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; - - if no_inline { - // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta - let mut xlmeta = FileMeta::new(); - // An existing dst xl.meta that fails to parse leaves `xlmeta` empty - // and gets overwritten by the commit below (pre-existing behavior); - // track that so the old-size observation reports unknown instead of - // a false `Absent` (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(dst_buf) = has_dst_buf.as_ref() { - if FileMeta::is_xl2_v1_format(dst_buf) - && let Ok(nmeta) = FileMeta::load(dst_buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let mut skip_parent = dst_volume_dir.clone(); - if has_dst_buf.as_ref().is_some() - && let Some(parent) = dst_file_path.parent() - { - skip_parent = parent.to_path_buf(); - } - - let version_id = fi.version_id.unwrap_or_default(); - let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = has_old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - if let Some(old_data_dir) = has_old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *old_data_dir); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_dst_buf = xlmeta.marshal_msg()?; - - // This tmp xl.meta is renamed onto dst_file_path at the commit - // point below, so only its contents must be durable before the - // rename (SyncMode::FileOnly); the dst parent directory is fsynced - // after the commit rename, and a crash before the rename means the - // PUT was never acknowledged. A metadata commit: relaxed tiers - // leave it to the page cache. - let tmp_meta_sync = if durability.syncs_commit_metadata() { - SyncMode::FileOnly - } else { - SyncMode::None - }; - // The tmp xl.meta write and the shard-file fdatasync are independent - // (disjoint paths) and both only need to be durable before the commit - // renames below, so run them concurrently to drop a blocking - // round-trip from the PUT commit critical path (rustfs/backlog#922 - // step 2). The "contents durable -> rename -> dst dir fsync" ordering - // is unchanged — both futures complete before any rename — which the - // rename_data crash-consistency harness (backlog#935) exercises. - // - // Shard durability: once rename_data succeeds the write is - // acknowledged, so data must not live only in the page cache. - // Multipart parts were already synced during rename_part, so their - // fdatasync here is a cheap no-op. A missing source dir is left for the - // rename below to report through the existing rollback path. Payload - // durability is kept by both strict and relaxed. - let tmp_meta_write = { - let src_file_path = src_file_path.clone(); - let dst_file_path = dst_file_path.clone(); - let rename_commit_guard = rename_commit_guard.clone(); - let mutation_lease = mutation_lease.clone(); - async move { - os::run_blocking_namespace_operation(mutation_lease, move || { - #[cfg(test)] - run_owned_file_write_before_open(&src_file_path); - let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( - &src_file_path, - &dst_file_path, - &rename_commit_guard, - )?; - prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; - Ok(prepared_metadata_source) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from) - } - }; - let shard_sync = async { - if durability.syncs_data_shards() - && let Some((src_data_path, _)) = has_data_dir_path.as_ref() - && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await - && err.kind() != ErrorKind::NotFound - { - return Err::<(), DiskError>(to_file_error(err).into()); - } - Ok(()) - }; - let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); - // Surface a tmp-meta failure first (its prior serial position), then a - // shard-sync failure; either aborts before any rename, exactly as the - // sequential version did. - let prepared_metadata_source = tmp_meta_res?; - shard_sync_res?; - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - std::fs::remove_file(&src_file_path).map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - // Heal reuses the version's data_dir, so for in-place corruption - // the destination dir still exists — and rename(2) cannot replace - // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge - // it first, healing commits only; fresh PUTs mint a new data_dir - // and never collide. Best effort: a real failure surfaces in the - // rename below. - if fi_healing - && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = self.move_to_trash(dst_data_path, true, false).await - { - warn!( - event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - dst_path = ?dst_data_path, - error = ?err, - "Healing commit could not purge the stale destination data dir" - ); - } - if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = os::rename_all_with_commit_guard( - src_data_path, - dst_data_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_data_path_failed", - src_path = ?src_data_path, - dst_path = ?dst_data_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - #[cfg(test)] - if has_data_dir_path.is_some() { - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - } - - // Crash-consistency injection: hard power loss after the data dir - // is in place but before xl.meta commits. No cleanup — the harness - // reopens the disk and asserts the object still reads as the old - // version (the staged data dir is a harmless orphan for GC). - if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { - return Err(DiskError::Unexpected); - } - - if should_fail_before_old_metadata_backup(dst_path) { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "test_fail_before_old_metadata_backup", - "Disk local rename flow failed before metadata commit" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::Unexpected); - } - - // The rollback backup stays where it is written (no rename) and is - // the sole restore source for a later undo_write, so under strict - // it keeps SyncMode::FileAndDir: contents and directory entry both - // durable. It is part of the metadata commit machinery, so relaxed - // tiers leave it to the page cache like the xl.meta it mirrors. - let backup_sync = if durability.syncs_commit_metadata() { - SyncMode::FileAndDir - } else { - SyncMode::None - }; - if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { - let backup_parent = dst_file_parent.join(old_data_dir.to_string()); - #[cfg(not(windows))] - if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { - Ok(guard) => guard, - Err(err) => { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::from(to_file_error(err))); - } - }; - let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); - if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { - #[cfg(windows)] - drop(backup_path_guard); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_bytes = dst_buf.clone(); - // Keep the volume, commit-tree, and exact destination-path - // guards in this task until the backup write and durability - // sync finish. A detached spawn_blocking writer could survive - // cancellation and later truncate a newer transaction's - // deterministic rollback backup. - let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { - #[cfg(test)] - run_owned_file_write_before_open(&backup_path); - backup_path_guard.write_file_for_path_access( - &backup_path, - backup_bytes.as_ref(), - backup_sync != SyncMode::None, - backup_sync == SyncMode::FileAndDir, - ) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from); - if let Err(err) = write_result { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "write_old_metadata_backup_failed", - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - } - - // Crash-consistency injection: hard power loss after the rollback - // backup is durable but before the xl.meta commit rename. No - // cleanup — the harness asserts the object still reads as the old - // version, since the destination xl.meta is untouched here. - if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - if let Err(err) = os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_metadata_failed", - src_path = ?src_file_path, - dst_path = ?dst_file_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - - let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); - if should_fail_after_metadata_commit(dst_path) { - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - return Err(DiskError::Unexpected); - } - - // Crash-consistency injection: hard power loss immediately after the - // xl.meta commit rename but before the durability fsync. Unlike the - // graceful failpoint above, no rollback runs — the commit rename is - // already on disk, so the harness asserts the object reads back as - // the new version. - if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - // Persist the directory entries for both the data dir and xl.meta renames; - // without this the commit itself can vanish on power loss. Relaxed tiers - // accept that window (documented in docs/operations/durability-modes.md). - if durability.syncs_commit_metadata() - && let Some(parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // The commit rename changed the dst part inodes before this fsync - // failed and rolled them back; drop any fd cached during that - // window so readers re-open the restored inode (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // First PUT of an object creates its directory (and any missing prefix - // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The - // commit fsync above persists the object dir's *contents*, not its own - // entry in the bucket/prefix dir, so on power loss after ack the whole - // object dir could vanish (rustfs/backlog#922 step 4). For a new object - // (no prior xl.meta) fsync the ancestor chain from the object dir's - // parent up to and including the bucket so those new directory entries - // are durable. Overwrites already have a durable object dir. The - // starts_with guard bounds the walk to the bucket subtree. Relaxed/none - // accept the wider window, like the commit fsync above. - if has_dst_buf.is_none() && durability.syncs_commit_metadata() { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(dir) = ancestor { - if !dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dir(dir).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // Same post-commit rollback window as above — drop cached - // dst part fds so readers re-open the restored inode - // (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if dir == dst_volume_dir.as_path() { - break; - } - ancestor = dir.parent(); - } - } - - // Publication and every rollback-capable durability step are now - // complete. Do not retain the Windows object identity guard while - // cleaning staging paths or invalidating cached descriptors. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(src_file_path_parent) = src_file_path.parent() { - if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { - let _ = std::fs::remove_dir(src_file_path_parent); - } else { - let _ = self - .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) - .await; - } - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: has_old_data_dir, - rollback_data_dir, - cleanup_data_dir: has_old_data_dir, - sign: version_signature, - old_current_size, - }) - } else { - // Inline metadata preparation is blocking. The transaction lease is - // moved into that work so a timeout can release the async waiter without - // allowing a retry to reuse the deterministic staging path too early. - let src = src_file_path.clone(); - let dst = dst_file_path.clone(); - let cleanup_path = if src_volume == super::RUSTFS_META_MULTIPART_BUCKET { - src_file_path.parent().map(|p| p.to_path_buf()) - } else { - None - }; - let dst_path_for_failpoint = dst_path.to_string(); - #[cfg(windows)] - let source_parent = src_file_parent.to_path_buf(); - let rename_commit_guard_for_preparation = rename_commit_guard.clone(); - let sync = durability.syncs_commit_metadata(); - #[cfg(test)] - run_inline_before_file_sync_admission(dst_path); - let mut file_sync_admission = if sync { - Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ) - } else { - None - }; - let prepare_inline_metadata = move || { - let mut prepared_metadata_source = - os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; - #[cfg(windows)] - let source_metadata_guard = - rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; - let mut xlmeta = FileMeta::new(); - // Same as the non-inline branch: an unparsable existing dst - // xl.meta must surface as unknown, not `Absent` - // (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(ref buf) = has_dst_buf { - if FileMeta::is_xl2_v1_format(buf) - && let Ok(nmeta) = FileMeta::load(buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let version_id = fi.version_id.unwrap_or_default(); - let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - let mut staged_rollback_path = None; - if let Some(d) = old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *d); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_buf = xlmeta.marshal_msg()?; - // Write the staged xl.meta. Inline objects carry their data inside - // xl.meta, so this is the durable preparation for the metadata commit: - // relaxed tiers do no per-object fsync here at all (aligned - // with MinIO's default), trading a documented power-loss - // window for latency. - prepared_metadata_source.write_all(&new_buf, sync)?; - run_inline_preparation_before_backup(&dst_path_for_failpoint); - if let Some(ref old_metadata) = has_dst_buf - && (rollback_data_dir.is_some() || sync || cfg!(test)) - { - #[cfg(windows)] - let backup_path = { - let backup_path = src - .parent() - .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? - .join(STORAGE_FORMAT_FILE_BACKUP); - source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; - backup_path - }; - #[cfg(not(windows))] - let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; - #[cfg(not(windows))] - if sync { - std::fs::File::open(&backup_path)?.sync_data()?; - } - staged_rollback_path = Some(backup_path); - } - - Ok::<_, std::io::Error>(( - rollback_data_dir, - old_data_dir, - version_signature, - old_current_size, - staged_rollback_path, - has_dst_buf.is_none(), - prepared_metadata_source, - )) - }; - let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { - os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await - } else { - os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await - } - .map_err(to_file_error) - .map_err(DiskError::from); - - let ( - rollback_data_dir, - cleanup_data_dir, - version_signature, - old_current_size, - mut local_rollback_path, - destination_was_absent, - prepared_metadata_source, - ) = match inline_preparation { - Ok(prepared) => prepared, - Err(err) => { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - }; - - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - let remove_result = std::fs::remove_file(&src_file_path); - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - remove_result.map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { - let Some(dst_parent) = dst_file_path.parent() else { - return Err(DiskError::other("missing object metadata parent")); - }; - let backup_path = dst_parent - .join(rollback_data_dir.to_string()) - .join(STORAGE_FORMAT_FILE_BACKUP); - // rename_all acquires the backup path's namespace lease. Do not - // hold a disk admission while acquiring another namespace lock. - drop(file_sync_admission.take()); - if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { - let _ = remove_file_if_exists(staged_backup); - return Err(err); - } - #[cfg(test)] - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - if sync { - file_sync_admission = Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ); - } - if let Some(admission) = file_sync_admission.as_ref() - && let Some(backup_parent) = backup_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - return Err(DiskError::from(to_file_error(err))); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - } - local_rollback_path = None; - } - - let commit_result = if should_fail_commit_rename(dst_path) { - Err(DiskError::other("test fail during metadata commit rename")) - } else { - os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &dst_volume_dir, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - }; - if let Err(err) = commit_result { - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - - let post_commit = async { - if should_fail_after_metadata_commit(dst_path) { - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(std::io::Error::other("test fail after metadata commit")); - } - - // Persist the commit rename's directory entry across power loss. - if let Some(admission) = file_sync_admission.as_ref() - && let Some(dst_parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) - .await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // Same power-loss gap as the non-inline path (rustfs/backlog#922 - // step 4): a first PUT creates the object dir (and any missing - // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all - // never fsynced. The fsync above persists the object dir's contents, - // not its own entry, so for a new inline object fsync the ancestor - // chain up to and including the bucket. Overwrites already have a - // durable object dir; the starts_with guard bounds the walk. - if let Some(admission) = file_sync_admission.as_ref() - && destination_was_absent - { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(ancestor_dir) = ancestor { - if !ancestor_dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std( - &dst_file_path, - rollback_data_dir, - local_rollback_path.as_deref(), - )?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if ancestor_dir == dst_volume_dir.as_path() { - break; - } - ancestor = ancestor_dir.parent(); - } - } - - Ok::<(), std::io::Error>(()) - } - .await; - - // The disk admission protects the durability chain, not staging - // cleanup or cache invalidation after that chain has completed. - drop(file_sync_admission.take()); - - // A post-commit rollback (for example, a commit-metadata fsync - // failure under strict durability) restores the old metadata; drop any - // descriptors cached during the committed window before propagating the - // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so - // this is mostly defensive and keeps both commit branches consistent. - if let Err(err) = post_commit { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(DiskError::from(err)); - } - - // The commit no longer has a rollback path. Release the Windows - // object identity guard before best-effort staging cleanup. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - - // Cleanup - if let Some(ref cleanup) = cleanup_path { - let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; - } else if let Some(parent) = src_file_path.parent() { - let _ = std::fs::remove_dir(parent); - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: cleanup_data_dir, - rollback_data_dir, - cleanup_data_dir, - sign: version_signature, - old_current_size, - }) - } - } - - pub(in crate::disk) async fn rename_data_observed( - &self, - src_volume: &str, - src_path: &str, - fi: &FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> super::RenameDataObservation { - let mut preflight_rejection = None; - let result = self - .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection) - .await; - super::RenameDataObservation { - result, - preflight_rejection, - } - } - pub(crate) async fn rename_data_borrowed( &self, src_volume: &str, diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs index c35619edb..ffc9e7db2 100644 --- a/rustfs/src/app/object/shared.rs +++ b/rustfs/src/app/object/shared.rs @@ -309,7 +309,7 @@ fn classify_bucket_default_sse_lookup( ) -> S3Result> { match lookup { Ok(config) => Ok(Some(config)), - Err(err) if err == StorageError::ConfigNotFound => Ok(None), + Err(StorageError::ConfigNotFound) => Ok(None), Err(err) => { let api_error = ApiError::from(err); error!( diff --git a/rustfs/src/site_replication/tests.rs b/rustfs/src/site_replication/tests.rs index e2ab27bb7..3b2c48666 100644 --- a/rustfs/src/site_replication/tests.rs +++ b/rustfs/src/site_replication/tests.rs @@ -867,7 +867,6 @@ fn test_retry_drain_bounds_each_peer_round_to_one_small_request_chain() { r#type: "tags".to_string(), ..Default::default() }], - ..Default::default() }; let make = RetryDrainAction::BucketOpReplay { operation: SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING.to_string(), @@ -973,7 +972,7 @@ fn test_lightweight_bucket_retry_plan_orders_real_metadata_and_counts_it() { operator_replication.rules.push(operator_rule("operator-backup")); let mut bucket_with_operator_rule = bucket; bucket_with_operator_rule.replication_config = - Some(BASE64_STANDARD.encode_to_string(&serialize(&operator_replication).expect("operator replication config"))); + Some(BASE64_STANDARD.encode_to_string(serialize(&operator_replication).expect("operator replication config"))); let plan = site_replication_bucket_retry_plan_from_info(&bucket_with_operator_rule, false).expect("targeted retry plan"); assert!( plan.bucket_items.iter().any(|item| item.r#type == "replication-config"), @@ -1050,7 +1049,7 @@ fn test_reachable_probe_promotion_is_fenced_by_the_observed_event() { .peers .insert("remote".to_string(), peer("remote", "https://remote.example.com")); - assert_eq!(mark_reachable_deferred_retry_events(&mut state, &[recovered.clone()]), 1); + assert_eq!(mark_reachable_deferred_retry_events(&mut state, std::slice::from_ref(&recovered)), 1); assert_eq!(state.retry_queue[0].updated_at, None); assert!(!state.retry_queue[0].peer_unreachable); assert_eq!( From f053862aade5c30d81bb6fd8d268a66f8dc853fe Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 18:56:26 +0800 Subject: [PATCH 26/40] docs: request concrete behavior evidence in pull requests (#7196) --- .github/pull_request_template.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b6edfbe1e..b514de257 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,16 +10,16 @@ Use N/A when there is no related issue. ## Summary of Changes ## Verification ## Impact From 9e2545244ca7726c06e7367259ad3fc497727689 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 19:44:49 +0800 Subject: [PATCH 27/40] fix(odm): bound empty pagination chains with staged tokens (#7197) * fix(odm): add staged cross-request pagination progress budgets * fix(ecstore): remove duplicate local rename implementation Keep the canonical commit module after concurrent storage changes merged. The control-write and rollback changes are already present there. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(app): simplify absent SSE configuration matching * fix(tests): satisfy new clippy lints --------- Co-authored-by: houseme Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/api/mod.rs | 7 +- .../on_demand_migration/list_through.rs | 347 +++++++++++++++++- .../src/bucket/on_demand_migration/mod.rs | 7 +- docs/operations/on-demand-migration.md | 10 + rustfs/src/app/bucket_list_through.rs | 329 ++++++++++++++++- rustfs/src/app/storage_api.rs | 8 +- 6 files changed, 670 insertions(+), 38 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 437f5c458..9e6fd34be 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -167,9 +167,10 @@ pub mod bucket { idle_guarded_body, }; pub use crate::bucket::on_demand_migration::{ - FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, - ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, - SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan, + FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, + ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome, + MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, + decode_continuation_token, source_list_plan, }; pub mod backfill { pub use crate::bucket::on_demand_migration::backfill::{ diff --git a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs index 2720c7718..a2ee6fea3 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs @@ -25,8 +25,13 @@ use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use std::time::{Duration, Instant}; -/// The only continuation-token envelope version this build reads and writes. +/// The continuation-token version used by ordinary progressing pages. pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1; +const LIST_THROUGH_PROGRESS_TOKEN_VERSION: u32 = 2; + +/// The sixteenth consecutive merged page without a key or new EOF fails. +/// This also bounds legitimate sparse listings; it is not a cycle detector. +pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16; /// Envelope marker. A bucket that is *not* merging hands out the local /// listing's own marker, so the decoder needs a positive signal before it @@ -111,6 +116,10 @@ pub struct ListThroughToken { /// common prefix compares as itself, never as its members. #[serde(default)] pub last_key: Option, + /// Consecutive empty truncated merged pages, present only in v2 tokens. + /// Ordinary v1 tokens retain their original serialized shape. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub no_progress: Option, } impl ListThroughToken { @@ -123,6 +132,7 @@ impl ListThroughToken { source: source.token, source_done: source.done, last_key, + no_progress: None, } } @@ -170,7 +180,21 @@ pub fn decode_continuation_token(decoded: &str) -> Result {} + Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => { + // v1 readers reject this field even when it is null or zero. + if value.get("no_progress").is_some() { + return Err(ListThroughTokenError::Malformed); + } + } + Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => { + if !value + .get("no_progress") + .and_then(serde_json::Value::as_u64) + .is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count)) + { + return Err(ListThroughTokenError::Malformed); + } + } Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)), None => return Err(ListThroughTokenError::Malformed), } @@ -288,6 +312,8 @@ pub enum ListPageError { Empty, #[error("truncated listing repeats a continuation token")] Repeated, + #[error("listing exhausted its consecutive no-progress page budget")] + NoProgress(MergeSide), } pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> { @@ -352,6 +378,7 @@ pub struct MergeOutcome { #[derive(Debug)] pub struct ListThroughMerger { max_keys: usize, + no_progress: Option, last_key: Option, local: SideState, source: SideState, @@ -371,6 +398,7 @@ impl ListThroughMerger { }; Self { max_keys, + no_progress: token.and_then(|token| token.no_progress), last_key, local, source, @@ -436,13 +464,18 @@ impl ListThroughMerger { Ok(()) } - pub fn finish(self) -> MergeOutcome { + /// `issue_progress_tokens` allows a v1 chain to start carrying a budget. + /// An existing v2 budget is always enforced, including on reader-only nodes. + /// Borrowing lets a source failure re-merge the fetched local buffers. + pub fn finish(&self, issue_progress_tokens: bool) -> Result { let Self { max_keys, + no_progress, last_key, local, source, } = self; + let max_keys = *max_keys; // A side with more pages behind it can only be trusted up to the last // key it handed over: past that horizon the other side's entries could @@ -508,12 +541,44 @@ impl ListThroughMerger { let source_left = !source.disabled && (!source_cursor.done || consumed_source < source.entries.len()); let is_truncated = local_left || source_left; - let last_key = consumed_key.or(last_key); - MergeOutcome { + let reached_eof = (!local.start.done && local_cursor.done) || (!source.start.done && source_cursor.done); + let next_no_progress = if !is_truncated || !picks.is_empty() || reached_eof { + None + } else if max_keys == 0 { + // A zero-sized request cannot consume entries. Preserve an existing + // budget without spending it or starting a new one. + *no_progress + } else if issue_progress_tokens || no_progress.is_some() { + let count = no_progress.unwrap_or(0).saturating_add(1); + if count >= MAX_LIST_NO_PROGRESS_PAGES { + // An empty truncated side closes the merge horizon. Local + // failure takes precedence; disabling the source cannot fix it. + let side = if local.more && local.entries.is_empty() { + MergeSide::Local + } else if !source.disabled && source.more && source.entries.is_empty() { + MergeSide::Source + } else { + MergeSide::Local + }; + return Err(ListPageError::NoProgress(side)); + } + Some(count) + } else { + None + }; + let last_key = consumed_key.or_else(|| last_key.clone()); + Ok(MergeOutcome { picks, is_truncated, - next_token: is_truncated.then(|| ListThroughToken::new(local_cursor, source_cursor, last_key)), - } + next_token: is_truncated.then(|| { + let mut token = ListThroughToken::new(local_cursor, source_cursor, last_key); + if let Some(count) = next_no_progress { + token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION; + token.no_progress = Some(count); + } + token + }), + }) } } @@ -641,7 +706,7 @@ mod tests { .push_page(fetch.side, kept, truncated, next) .expect("reference provider pages must advance"); } - let outcome = merger.finish(); + let outcome = merger.finish(false).expect("valid merge outcome"); assert_eq!(outcome.is_truncated, outcome.next_token.is_some()); if outcome.is_truncated { assert_ne!(outcome.next_token, token, "every truncated merged page must make progress"); @@ -724,7 +789,7 @@ mod tests { .push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None) .expect("local EOF is valid"); assert_eq!(merger.next_fetch(), None); - let outcome = merger.finish(); + let outcome = merger.finish(false).expect("valid merge outcome"); assert_eq!(outcome.picks.len(), 1); assert!(!outcome.is_truncated); assert!(outcome.next_token.is_none()); @@ -740,6 +805,7 @@ mod tests { source: Some("source-1".to_string()), source_done: false, last_key: Some("a".to_string()), + no_progress: None, }; let mut merger = ListThroughMerger::new(1, Some(&resume)); merger.disable_source(); @@ -751,7 +817,7 @@ mod tests { Some("local-2".to_string()), ) .expect("local cursor advances"); - let outcome = merger.finish(); + let outcome = merger.finish(false).expect("valid merge outcome"); assert!(outcome.is_truncated); let token = outcome.next_token.expect("truncated page carries a token"); assert_eq!(token.source.as_deref(), Some("source-1"), "the source cursor must not move"); @@ -830,7 +896,7 @@ mod tests { .expect("opaque cursor advances regardless of sort order"); } assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget"); - let outcome = merger.finish(); + let outcome = merger.finish(false).expect("valid merge outcome"); assert!(outcome.picks.is_empty()); assert!(outcome.is_truncated); let token = outcome.next_token.expect("empty progressing page has a cursor"); @@ -840,7 +906,7 @@ mod tests { merger .push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None) .expect("source EOF"); - let outcome = merger.finish(); + let outcome = merger.finish(false).expect("valid merge outcome"); assert_eq!( outcome.picks, vec![MergePick { @@ -887,7 +953,7 @@ mod tests { Err(ListPageError::Repeated) ); merger.disable_source(); - let outcome = merger.finish(); + let outcome = merger.finish(false).expect("valid merge outcome"); assert_eq!( outcome.picks, vec![MergePick { @@ -979,8 +1045,8 @@ mod tests { let encoded = token.encode(); assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token)))); - let bumped = encoded.replace("\"v\":1", "\"v\":2"); - assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(2))); + let bumped = encoded.replace("\"v\":1", "\"v\":3"); + assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(3))); let extra = encoded.replace("{", "{\"x\":1,"); assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed)); @@ -992,6 +1058,257 @@ mod tests { assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed)); } + fn progress_token(count: Option, local_done: bool, source_done: bool) -> ListThroughToken { + let mut token = ListThroughToken::new( + SideCursor { + token: None, + done: local_done, + }, + SideCursor { + token: Some("A".into()), + done: source_done, + }, + Some("last-key".into()), + ); + if let Some(count) = count { + token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION; + token.no_progress = Some(count); + } + token + } + + fn push_empty_pages(merger: &mut ListThroughMerger, side: MergeSide) { + for _ in 0..MAX_LIST_FETCHES_PER_SIDE { + let fetch = merger.next_fetch().expect("empty truncated side must be fetched"); + assert_eq!(fetch.side, side); + let next = format!("{}:next", fetch.token.unwrap_or_default()); + merger + .push_page(side, vec![], true, Some(next)) + .expect("opaque cursor advances"); + } + } + + #[test] + fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() { + let token = progress_token(None, true, false); + assert_eq!( + token.encode(), + r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"# + ); + for count in 1..MAX_LIST_NO_PROGRESS_PAGES { + let token = progress_token(Some(count), true, false); + assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token)))); + } + for version in [1, 2] { + for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] { + let encoded = format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#); + assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}"); + } + } + for encoded in [ + r#"{"t":"odm-list","v":1,"no_progress":1}"#, + r#"{"t":"odm-list","v":2}"#, + r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#, + ] { + assert_eq!(decode_continuation_token(encoded), Err(ListThroughTokenError::Malformed), "{encoded}"); + } + } + + #[test] + fn reader_only_nodes_do_not_start_a_budget_but_mixed_readers_preserve_one() { + let mut token = progress_token(None, true, false); + for _ in 0..MAX_LIST_NO_PROGRESS_PAGES { + let mut merger = ListThroughMerger::new(2, Some(&token)); + push_empty_pages(&mut merger, MergeSide::Source); + token = merger + .finish(false) + .expect("reader-only v1 behavior") + .next_token + .expect("truncated cursor"); + assert_eq!(token.v, 1); + assert_eq!(token.no_progress, None); + } + for count in 1..=MAX_LIST_NO_PROGRESS_PAGES { + let mut merger = ListThroughMerger::new(2, Some(&token)); + push_empty_pages(&mut merger, MergeSide::Source); + assert!(merger.next_fetch().is_none(), "the per-request two-fetch limit stays intact"); + let outcome = merger.finish(count % 2 == 1); + if count == MAX_LIST_NO_PROGRESS_PAGES { + assert_eq!(outcome, Err(ListPageError::NoProgress(MergeSide::Source))); + break; + } + token = outcome.expect("budget not exhausted").next_token.expect("truncated cursor"); + assert_eq!(token.no_progress, Some(count)); + let ListThroughCursor::Merged(decoded) = decode_continuation_token(&token.encode()).expect("round-trip v2") else { + panic!("merged cursor expected"); + }; + token = *decoded; + } + } + + #[test] + fn objects_and_common_prefixes_reset_a_budget_at_the_boundary() { + for entry in [ListEntryKey::object("result"), ListEntryKey::prefix("result/")] { + for issue_tokens in [false, true] { + let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + merger + .push_page(MergeSide::Source, vec![], true, Some("B".into())) + .expect("empty advancing page"); + merger + .push_page(MergeSide::Source, vec![entry.clone()], true, Some("C".into())) + .expect("real progress"); + let outcome = merger + .finish(issue_tokens) + .expect("real progress does not exhaust the budget"); + assert_eq!( + outcome.picks, + vec![MergePick { + side: MergeSide::Source, + index: 0 + }] + ); + let next = outcome.next_token.expect("source remains truncated"); + assert_eq!(next.last_key.as_deref(), Some(entry.name.as_str())); + assert_eq!(next.v, 1); + assert_eq!(next.no_progress, None); + assert!(!next.encode().contains("no_progress")); + } + } + } + + #[test] + fn only_a_new_eof_transition_resets_the_empty_page_budget() { + for finished_side in [MergeSide::Local, MergeSide::Source] { + let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + if finished_side == MergeSide::Local { + merger + .push_page(MergeSide::Local, vec![], false, None) + .expect("new local EOF"); + push_empty_pages(&mut merger, MergeSide::Source); + } else { + push_empty_pages(&mut merger, MergeSide::Local); + merger + .push_page(MergeSide::Source, vec![], false, None) + .expect("new source EOF"); + } + let next = merger + .finish(false) + .expect("new EOF is progress") + .next_token + .expect("other side truncated"); + assert_eq!(next.no_progress, None); + assert_eq!(next.v, 1); + assert_eq!(next.local_done, finished_side == MergeSide::Local); + assert_eq!(next.source_done, finished_side == MergeSide::Source); + let mut merger = ListThroughMerger::new(2, Some(&next)); + let remaining = if finished_side == MergeSide::Local { + MergeSide::Source + } else { + MergeSide::Local + }; + push_empty_pages(&mut merger, remaining); + let next = merger + .finish(true) + .expect("a new budget starts") + .next_token + .expect("truncated"); + assert_eq!(next.no_progress, Some(1), "an already-done side cannot reset every page"); + } + let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + merger.push_page(MergeSide::Source, vec![], false, None).expect("final EOF"); + let outcome = merger.finish(false).expect("EOF succeeds at the budget boundary"); + assert!(!outcome.is_truncated); + assert!(outcome.next_token.is_none()); + } + + #[test] + fn filtered_duplicates_cannot_reset_the_no_progress_budget() { + let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + for next in ["B", "C"] { + let entries = [ListEntryKey::object("last-key"), ListEntryKey::object("earlier")] + .into_iter() + .filter(|entry| merger.accepts(&entry.name)) + .collect::>(); + assert!(entries.is_empty(), "both provider entries were already consumed"); + merger + .push_page(MergeSide::Source, entries, true, Some(next.into())) + .expect("advancing cursor"); + } + assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source))); + } + + #[test] + fn no_progress_is_attributed_to_local_when_source_cannot_unblock_it() { + for source_mode in ["disabled", "done", "empty", "data"] { + let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, source_mode == "done"); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + if source_mode == "disabled" { + merger.disable_source(); + } + push_empty_pages(&mut merger, MergeSide::Local); + match source_mode { + "empty" => push_empty_pages(&mut merger, MergeSide::Source), + "data" => merger + .push_page(MergeSide::Source, vec![ListEntryKey::object("source")], false, None) + .expect("source data"), + _ => {} + } + assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Local)), "{source_mode}"); + } + } + + #[test] + fn source_budget_failure_remerges_local_objects_and_prefixes_without_refetching() { + let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + merger + .push_page(MergeSide::Local, vec![ListEntryKey::object("local")], true, Some("L1".into())) + .expect("local object"); + merger + .push_page(MergeSide::Local, vec![ListEntryKey::prefix("prefix/")], true, Some("L2".into())) + .expect("local prefix"); + push_empty_pages(&mut merger, MergeSide::Source); + assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source))); + merger.disable_source(); + assert!(merger.next_fetch().is_none(), "fallback does not perform another fetch"); + let outcome = merger.finish(false).expect("local data makes progress"); + assert_eq!( + outcome.picks, + vec![ + MergePick { + side: MergeSide::Local, + index: 0 + }, + MergePick { + side: MergeSide::Local, + index: 1 + } + ] + ); + let token = outcome.next_token.expect("remaining local page"); + assert_eq!(token.local.as_deref(), Some("L2")); + assert_eq!(token.source.as_deref(), Some("A")); + assert_eq!(token.last_key.as_deref(), Some("prefix/")); + assert_eq!(token.no_progress, None); + assert_eq!(token.v, 1); + } + + #[test] + fn a_zero_sized_merge_preserves_an_existing_budget() { + let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false); + let mut merger = ListThroughMerger::new(0, Some(&resume)); + merger + .push_page(MergeSide::Source, vec![ListEntryKey::object("result")], true, Some("B".into())) + .expect("source page"); + let outcome = merger.finish(false).expect("a zero-sized request cannot consume entries"); + assert!(outcome.picks.is_empty()); + assert_eq!(outcome.next_token.expect("unconsumed source").no_progress, resume.no_progress); + } + #[test] fn a_plain_local_marker_stays_local() { assert_eq!( diff --git a/crates/ecstore/src/bucket/on_demand_migration/mod.rs b/crates/ecstore/src/bucket/on_demand_migration/mod.rs index 85a4d5a99..0ca4a29ce 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/mod.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/mod.rs @@ -40,9 +40,10 @@ pub use config::{ SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext, }; pub use list_through::{ - FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, - ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, - SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan, + FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, + ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome, MergePick, + MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, + decode_continuation_token, source_list_plan, }; pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache}; pub use pull::{ diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md index 7a29db72c..eb7cde57f 100644 --- a/docs/operations/on-demand-migration.md +++ b/docs/operations/on-demand-migration.md @@ -7,6 +7,16 @@ On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** t The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured. +## List continuation token rollout + +`RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape. + +Leave the switch off while deploying v2 reader support to every node that can receive a continuation request, including nodes behind other load-balancer routes. Then set it to `true` in each node's environment and restart those nodes to enable issuance. A v1-only binary rejects v2 with `400 InvalidArgument` before the source-error policy runs; neither `not_found` nor turning off list-through makes that old reader compatible. With issuance still off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. The default rollout does not claim to fix that chain until issuance is enabled. + +An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64 JSON, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries. + +For rollback, first turn issuance off on every node. Keep v2-capable readers available for outstanding v2 chains: switching issuance off does not erase their budgets, and tokens have no expiration that proves those chains have drained. Route those continuations to compatible readers or have clients explicitly restart their listings before restoring v1-only binaries. Restarting a listing is a new scan and can repeat entries. Do not roll back readers while assuming the issuance switch makes existing v2 tokens disappear. + ## Positioning | Capability | Direction | What it moves | Where the authoritative copy is | When to use it instead | diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs index a97470f9a..23b7513f2 100644 --- a/rustfs/src/app/bucket_list_through.rs +++ b/rustfs/src/app/bucket_list_through.rs @@ -26,8 +26,8 @@ use super::storage_api::bucket_usecase::ECStore; use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo; use super::storage_api::bucket_usecase::StorageObjectOptions; use super::storage_api::bucket_usecase::bucket::on_demand_migration::{ - BucketOdmState, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, MergeSide, - OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan, + BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, + MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan, SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan, }; use super::storage_api::bucket_usecase::bucket::versioning_sys::BucketVersioningSys; @@ -51,6 +51,9 @@ type ListObjectsV2Info = StorageListObjectsV2Info; /// yet, so the only class RustFS can vouch for is the default one. const SOURCE_STORAGE_CLASS: &str = "STANDARD"; +/// Enable only after every node serving continuation requests can read v2. +const ENV_LIST_PROGRESS_TOKENS: &str = "RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS"; + /// Concurrent local metadata probes when a versioned bucket has to check /// source-only keys for a shadowing delete marker. const DELETE_MARKER_PROBE_CONCURRENCY: usize = 32; @@ -264,7 +267,18 @@ pub(crate) async fn merged_list_objects_v2( buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some)); } - let outcome = merger.finish(); + let issue_progress_tokens = rustfs_utils::get_env_bool(ENV_LIST_PROGRESS_TOKENS, false); + let outcome = match merger.finish(issue_progress_tokens) { + Ok(outcome) => outcome, + Err(ListPageError::NoProgress(MergeSide::Source)) => { + degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?; + merger + .finish(issue_progress_tokens) + .map_err(|error| S3Error::with_message(S3ErrorCode::InternalError, error.to_string()))? + } + Err(error) => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())), + }; + drop(merger); let mut objects = Vec::with_capacity(outcome.picks.len()); let mut prefixes = Vec::new(); let mut source_only_keys = Vec::new(); @@ -430,7 +444,8 @@ mod tests { use crate::app::bucket_usecase::DefaultBucketUsecase; use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore}; use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{ - FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig, + FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, + SourceCredentials, TlsConfig, }; use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response}; use crate::app::storage_api::test::StoragePutObjReader; @@ -448,6 +463,7 @@ mod tests { source: Some("source-2".to_string()), source_done: false, last_key: Some("k".to_string()), + no_progress: None, } } @@ -515,6 +531,24 @@ mod tests { assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted)); } + #[test] + fn a_v2_token_keeps_the_local_cursor_when_list_through_is_turned_off() { + let mut resume = token(Some("local-2"), false); + resume.v = 2; + resume.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 1); + let encoded = resume.encode(); + let decoded = decode_list_cursor(Some(&encoded)).expect("a v2 envelope decodes"); + assert_eq!(decoded.as_ref(), Some(&resume)); + assert!(matches!( + local_cursor(Some(&encoded), decoded.as_ref()), + LocalListCursor::Token(Some(local)) if local == "local-2" + )); + resume.local_done = true; + let encoded = resume.encode(); + let decoded = decode_list_cursor(Some(&encoded)).expect("v2 with local EOF decodes"); + assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted)); + } + #[test] fn a_plain_local_token_is_passed_through_and_a_tampered_one_is_rejected() { assert!( @@ -551,14 +585,30 @@ mod tests { /// Serves exactly the scripted S3 pages and joins every connection before /// returning. A source retry or unexpected operation fails the test. async fn scripted_list_source(pages: Vec) -> (String, tokio_util::task::AbortOnDropHandle>) { + let (endpoint, server, _) = list_source(pages.into_iter()).await; + (endpoint, server) + } + + async fn list_source( + pages: impl Iterator + Send + 'static, + ) -> ( + String, + tokio_util::task::AbortOnDropHandle>, + tokio_util::sync::CancellationToken, + ) { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("bind listing source"); let address = listener.local_addr().expect("listing source address"); + let stop = tokio_util::sync::CancellationToken::new(); + let server_stop = stop.clone(); let server = tokio::spawn(async move { let mut requests = Vec::new(); for body in pages { - let (mut stream, _) = listener.accept().await.expect("accept source listing"); + let (mut stream, _) = tokio::select! { + _ = server_stop.cancelled() => break, + accepted = listener.accept() => accepted.expect("accept source listing"), + }; let mut request = Vec::new(); let mut chunk = [0; 4096]; while !request.windows(4).any(|window| window == b"\r\n\r\n") { @@ -588,7 +638,7 @@ mod tests { } requests }); - (format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server)) + (format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server), stop) } fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String { @@ -616,12 +666,12 @@ mod tests { } } - async fn source_policy_request( - pages: Vec, + async fn source_policy_input( + endpoint: String, policy: SourceErrorPolicy, resume_source: Option<&str>, filter_prefix: Option<&str>, - ) -> (S3Result>, Vec) { + ) -> (ListThroughTestState, ListObjectsV2Input) { let store = shared_gating_ecstore().await; crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple()); @@ -638,9 +688,8 @@ mod tests { ) .await .expect("seed real local listing"); - let (endpoint, server) = scripted_list_source(pages).await; let sys = OnDemandMigrationSys::get(); - let _state_guard = ListThroughTestState { + let state_guard = ListThroughTestState { bucket: bucket.clone(), module_enabled: sys.is_module_enabled(), }; @@ -685,6 +734,7 @@ mod tests { source: Some(source.into()), source_done: false, last_key: None, + no_progress: None, }; base64_simd::STANDARD.encode_to_string(token.encode().as_bytes()) }); @@ -701,6 +751,10 @@ mod tests { request_payer: None, start_after: None, }; + (state_guard, input) + } + + async fn execute_source_list(input: ListObjectsV2Input) -> S3Result> { let request = S3Request { input, method: http::Method::GET, @@ -712,12 +766,23 @@ mod tests { service: None, trailing_headers: None, }; - let result = tokio::time::timeout( + tokio::time::timeout( Duration::from_secs(10), DefaultBucketUsecase::from_global().execute_list_objects_v2(request), ) .await - .expect("listing must complete within its bounded source budget"); + .expect("listing must complete within its bounded source budget") + } + + async fn source_policy_request( + pages: Vec, + policy: SourceErrorPolicy, + resume_source: Option<&str>, + filter_prefix: Option<&str>, + ) -> (S3Result>, Vec) { + let (endpoint, server) = scripted_list_source(pages).await; + let (_state_guard, input) = source_policy_input(endpoint, policy, resume_source, filter_prefix).await; + let result = execute_source_list(input).await; let requests = tokio::time::timeout(Duration::from_secs(5), server) .await .expect("source connections must finish") @@ -842,6 +907,244 @@ mod tests { }); } + #[test] + #[serial_test::serial] + fn list_through_cross_request_empty_cursor_cycle_obeys_policy() { + run_large_stack_test("list-through-cross-request-cursor-cycle", || async { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, Some("true")), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] { + let pages = ["B", "C", "A"].map(|next| source_xml(Some(next), true, None)); + let (endpoint, server, stop) = list_source(pages.into_iter().cycle()).await; + let (_state_guard, mut input) = source_policy_input(endpoint, policy, Some("A"), None).await; + let mut seen = std::collections::HashSet::from([input + .continuation_token + .clone() + .expect("the first request resumes source cursor A")]); + let mut client_requests = 0; + let mut empty_pages = 0; + let terminal = tokio::time::timeout(Duration::from_secs(30), async { + loop { + client_requests += 1; + let response = match execute_source_list(input.clone()).await { + Ok(response) => response, + Err(error) => break Err(error), + }; + if response.headers.contains_key("x-rustfs-on-demand-migration-list") { + break Ok(response); + } + let output = response.output; + assert!(output.contents.as_ref().is_none_or(Vec::is_empty)); + assert!(output.common_prefixes.as_ref().is_none_or(Vec::is_empty)); + assert_eq!(output.key_count, Some(0)); + assert_eq!(output.is_truncated, Some(true)); + let next = output + .next_continuation_token + .expect("a truncated page must carry its cursor"); + assert!( + seen.insert(next.clone()), + "a cross-request source cursor cycle must not return an identical empty merged token" + ); + empty_pages += 1; + input.continuation_token = Some(next); + } + }) + .await + .expect("a source cursor cycle must terminate within a bounded client pagination chain"); + assert_eq!(empty_pages, usize::from(MAX_LIST_NO_PROGRESS_PAGES - 1)); + assert_eq!(client_requests, usize::from(MAX_LIST_NO_PROGRESS_PAGES)); + assert_source_policy_result(terminal, policy); + stop.cancel(); + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("cyclic source server must stop") + .expect("cyclic source server must not panic"); + assert_eq!(requests.len(), 2 * client_requests, "the sixteenth empty page exhausts the budget"); + for (index, request) in requests.iter().enumerate() { + let source_cursor = ["A", "B", "C"][index % 3]; + assert!( + request.contains(&format!("continuation-token={source_cursor}")), + "the real SDK must follow the returned source cursor: {request}" + ); + } + } + }, + ) + .await; + }); + } + + #[test] + #[serial_test::serial] + fn list_through_default_rollout_continues_v2_without_issuing_it_from_v1() { + run_large_stack_test("list-through-reader-first-rollout", || async { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, None), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] { + let pages = ["B", "C", "A"].map(|next| source_xml(Some(next), true, None)); + let (endpoint, server, stop) = list_source(pages.into_iter().cycle()).await; + let (_state_guard, mut input) = source_policy_input(endpoint, policy, Some("A"), None).await; + let original = input.continuation_token.clone(); + for _ in 0..3 { + let response = execute_source_list(input.clone()).await.expect("reader-only v1 behavior"); + assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list")); + assert_eq!(response.output.key_count, Some(0)); + assert_eq!(response.output.is_truncated, Some(true)); + let next = response.output.next_continuation_token.expect("resumable empty page"); + let raw = base64_simd::STANDARD.decode_to_vec(&next).expect("base64 continuation token"); + let decoded = std::str::from_utf8(&raw).expect("JSON token"); + let token = decode_list_cursor(Some(decoded)).expect("v1 reader").expect("merged token"); + assert_eq!(token.v, 1, "the default rollout cannot begin issuing v2"); + assert_eq!(token.no_progress, None); + assert!(!decoded.contains("no_progress"), "ordinary v1 wire shape stays unchanged"); + input.continuation_token = Some(next); + } + assert_eq!(input.continuation_token, original, "default rollout retains the known v1 limitation"); + + let raw = base64_simd::STANDARD + .decode_to_vec(input.continuation_token.as_ref().expect("v1 token")) + .expect("base64 continuation token"); + let mut token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token"))) + .expect("v1 reader") + .expect("merged token"); + token.v = 2; + token.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 2); + input.continuation_token = Some(base64_simd::STANDARD.encode_to_string(token.encode().as_bytes())); + let response = execute_source_list(input.clone()).await.expect("reader-only node resumes v2"); + assert_eq!(response.output.key_count, Some(0)); + assert_eq!(response.output.is_truncated, Some(true)); + let next = response.output.next_continuation_token.expect("last allowed empty cursor"); + let raw = base64_simd::STANDARD.decode_to_vec(&next).expect("base64 continuation token"); + let token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token"))) + .expect("v2 reader") + .expect("merged token"); + assert_eq!(token.v, 2); + assert_eq!(token.no_progress, Some(MAX_LIST_NO_PROGRESS_PAGES - 1)); + input.continuation_token = Some(next); + assert_source_policy_result(execute_source_list(input).await, policy); + stop.cancel(); + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("cyclic source server must stop") + .expect("source server must not panic"); + assert_eq!(requests.len(), 10, "five handler requests each fetched two source pages"); + for (index, request) in requests.iter().enumerate() { + let cursor = ["A", "B", "C"][index % 3]; + assert!(request.contains(&format!("continuation-token={cursor}")), "{request}"); + } + } + }, + ) + .await; + }); + } + + #[test] + #[serial_test::serial] + fn list_through_empty_advancing_pages_resume_across_handler_requests() { + run_large_stack_test("list-through-resumable-empty-pages", || async { + temp_env::async_with_vars( + [ + (ENV_LIST_PROGRESS_TOKENS, Some("true")), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + for filter_prefix in [None, Some("photos/2024/")] { + let source_key = filter_prefix.map_or("a-source", |_| "photos/2024/a-source"); + let (endpoint, server) = scripted_list_source(vec![ + source_xml(Some("A"), true, None), + source_xml(Some("B"), true, None), + source_xml(Some("C"), true, None), + source_xml(None, false, Some(source_key)), + ]) + .await; + let (_state_guard, mut input) = + source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, filter_prefix).await; + let first = execute_source_list(input.clone()) + .await + .expect("valid empty pages must remain resumable"); + assert!(!first.headers.contains_key("x-rustfs-on-demand-migration-list")); + assert!(first.output.contents.as_ref().is_none_or(Vec::is_empty)); + assert!(first.output.common_prefixes.as_ref().is_none_or(Vec::is_empty)); + assert_eq!(first.output.key_count, Some(0)); + assert_eq!(first.output.is_truncated, Some(true)); + input.continuation_token = Some(first.output.next_continuation_token.expect("empty advancing cursor")); + + let second = execute_source_list(input) + .await + .expect("a progressing empty chain must reach its data"); + assert!(!second.headers.contains_key("x-rustfs-on-demand-migration-list")); + let output = second.output; + let objects = output + .contents + .unwrap_or_default() + .into_iter() + .map(|object| object.key.expect("listed object key")) + .collect::>(); + let prefixes = output + .common_prefixes + .unwrap_or_default() + .into_iter() + .map(|prefix| prefix.prefix.expect("listed common prefix")) + .collect::>(); + if filter_prefix.is_some() { + assert_eq!(objects, vec!["z-local"]); + assert_eq!(prefixes, vec!["photos/"]); + } else { + assert_eq!(objects, vec!["a-source", "z-local"]); + assert!(prefixes.is_empty()); + } + assert_eq!(output.key_count, Some(2)); + assert_eq!(output.is_truncated, Some(false)); + assert!(output.next_continuation_token.is_none()); + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("finite source connections must finish") + .expect("finite source server must not panic"); + assert_eq!(requests.len(), 4); + assert!(!requests[0].contains("continuation-token=")); + for (request, cursor) in requests[1..].iter().zip(["A", "B", "C"]) { + assert!(request.contains(&format!("continuation-token={cursor}")), "{request}"); + } + } + }, + ) + .await; + }); + } + fn assert_source_policy_result(result: S3Result>, policy: SourceErrorPolicy) { match policy { SourceErrorPolicy::Propagate => { diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 9f543ac95..383df36a1 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -634,8 +634,8 @@ pub(crate) mod bucket { }; #[cfg(test)] pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{ - BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OnDemandMigrationConfig, PathStyle, Provider, SourceConfig, - SourceCredentials, TlsConfig, + BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, + PathStyle, Provider, SourceConfig, SourceCredentials, TlsConfig, }; pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{ BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OdmStateError, OnDemandMigrationSys, PolicyConfig, @@ -643,8 +643,8 @@ pub(crate) mod bucket { commit_inline, idle_guarded_body, }; pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{ - ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, MergeSide, - SOURCE_LIST_MAX_RATE_WAIT, SourceListPlan, decode_continuation_token, source_list_plan, + ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, + MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SourceListPlan, decode_continuation_token, source_list_plan, }; } From cc1ec6b9929c86844f0a6b90594bbc420192814c Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 20:03:59 +0800 Subject: [PATCH 28/40] fix(ci): share quick checks and lint workflows (#7194) * fix(ci): share quick checks and lint workflows * fix(ci): install actionlint from its verified release * fix(ci): reject dependencies on required quick checks --- .github/actions/quick-checks/action.yml | 115 +++++++++++++ .github/workflows/ci-docs-only.yml | 95 +---------- .github/workflows/ci.yml | 70 +------- scripts/check_test_wiring.py | 207 +++++++++++++++++++++++- 4 files changed, 325 insertions(+), 162 deletions(-) create mode 100644 .github/actions/quick-checks/action.yml diff --git a/.github/actions/quick-checks/action.yml b/.github/actions/quick-checks/action.yml new file mode 100644 index 000000000..e6cc59775 --- /dev/null +++ b/.github/actions/quick-checks/action.yml @@ -0,0 +1,115 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Quick Checks +description: Run the shared compile-free RustFS quality checks. + +runs: + using: composite + steps: + - name: Install quality tools + uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 + with: + tool: | + ripgrep@15.2.0 + shellcheck@0.11.0 + + - name: Install actionlint + shell: bash + run: | + actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")" + curl --fail --location --silent --show-error \ + --output "$actionlint_dir/actionlint.tar.gz" \ + https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz + echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status + tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint + rm "$actionlint_dir/actionlint.tar.gz" + echo "$actionlint_dir" >> "$GITHUB_PATH" + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + components: rustfmt + + - name: Check workflow syntax and shell scripts + shell: bash + run: shellcheck --version && actionlint + + - name: Check code formatting + shell: bash + run: cargo fmt --all --check + + - name: Check unsafe code allowances + shell: bash + run: ./scripts/check_unsafe_code_allowances.sh + + - name: Check layered dependencies + shell: bash + run: ./scripts/check_layer_dependencies.sh + + - name: Check architecture migration rules + shell: bash + run: ./scripts/check_architecture_migration_rules.sh + + - name: Check logging guardrails + shell: bash + run: ./scripts/check_logging_guardrails.sh + + - name: Check error other(format!) ratchet + shell: bash + run: ./scripts/check_error_other_format_ratchet.sh + + - name: Check tokio io-uring feature guard + shell: bash + run: ./scripts/check_no_tokio_io_uring.sh + + - name: Check extension schema boundaries + shell: bash + run: ./scripts/check_extension_schema_boundaries.sh + + - name: Check body-cache whitelist guard + shell: bash + run: ./scripts/check_body_cache_whitelist.sh + + - name: Check s3s footprint ratchet + shell: bash + run: ./scripts/check_s3s_footprint.sh + + - name: Check cryptographic capability wording + shell: bash + run: ./scripts/check_fips_wording.sh + + - name: Check no embedded secret material + shell: bash + run: ./scripts/check_embedded_secrets.sh + + - name: Check test wiring + shell: bash + run: | + python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/test_security_workflow.py + python3 ./scripts/check_test_wiring.py + + - name: Check no planning docs committed + shell: bash + run: ./scripts/check_no_planning_docs.sh + + - name: Check CI paths stay in sync + shell: bash + run: ./scripts/check_ci_paths_sync.sh + + - name: Check io_uring lane --lib precondition + shell: bash + run: ./scripts/check_uring_lane_lib_only.sh diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index 7c6ad22ec..e61db2aff 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -12,24 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Companion to ci.yml for required status checks. -# -# ci.yml skips docs-only pull requests via paths-ignore, but the branch ruleset -# requires a check named "Test and Lint" — without this workflow a docs-only PR -# would wait on it forever. This workflow triggers on exactly the paths ci.yml -# ignores and reports success under the same job name. Mixed PRs trigger both -# workflows and the real check still gates: a required check with any failing -# run blocks the merge. -# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks -# -# "Quick Checks" is mirrored here ahead of the ruleset change that will make it -# required too (rustfs/backlog#1599). Until that change lands this job is -# inert; mirroring it first is what lets the ruleset change happen without -# stranding docs-only PRs on a check nobody reports. -# -# Keep the paths list below in sync with the pull_request paths-ignore list -# in ci.yml, and keep the quick-checks steps below byte-identical to the -# quick-checks job in ci.yml. +# Reports the existing required checks for paths excluded by ci.yml. +# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared +# action to keep validation coverage aligned. Keep this paths list in sync with +# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh. name: Continuous Integration (docs only) @@ -59,19 +45,6 @@ permissions: contents: read jobs: - # Deliberately NOT a bare `echo`. Once "Quick Checks" becomes a required - # check, ci.yml gates every expensive job behind it, so a mixed PR reports - # two check runs with this name: the real one (45-51s) and this companion. - # GitHub has no written contract for how it picks between same-named - # required check runs ("latest wins" vs "any failure blocks"), so instead of - # relying on ordering we make both runs execute the same commands against - # the same merge ref — their conclusions are then necessarily identical and - # the choice does not matter. Keep these steps byte-identical to the - # quick-checks job in ci.yml (a guard script that asserts this, and the paths - # sync below, is tracked in rustfs/backlog#1603). - # - # For a genuinely docs-only PR this adds no strictness (no code changed, so - # fmt and the guards always pass) and costs ~50s of ubuntu-latest. quick-checks: name: Quick Checks runs-on: ubuntu-latest @@ -82,64 +55,8 @@ jobs: with: persist-credentials: false - - name: Install ripgrep - uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 - with: - tool: ripgrep@15.2.0 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - components: rustfmt - - - name: Check code formatting - run: cargo fmt --all --check - - - name: Check unsafe code allowances - run: ./scripts/check_unsafe_code_allowances.sh - - - name: Check layered dependencies - run: ./scripts/check_layer_dependencies.sh - - - name: Check architecture migration rules - run: ./scripts/check_architecture_migration_rules.sh - - - name: Check logging guardrails - run: ./scripts/check_logging_guardrails.sh - - - name: Check tokio io-uring feature guard - run: ./scripts/check_no_tokio_io_uring.sh - - - name: Check extension schema boundaries - run: ./scripts/check_extension_schema_boundaries.sh - - - name: Check body-cache whitelist guard - run: ./scripts/check_body_cache_whitelist.sh - - - name: Check s3s footprint ratchet - run: ./scripts/check_s3s_footprint.sh - - - name: Check cryptographic capability wording - run: ./scripts/check_fips_wording.sh - - - name: Check no embedded secret material - run: ./scripts/check_embedded_secrets.sh - - - name: Check test wiring - run: | - python3 ./scripts/check_test_wiring.py --self-test - python3 ./scripts/check_scheduled_validation_freshness.py --self-test - python3 ./scripts/test_security_workflow.py - python3 ./scripts/check_test_wiring.py - - - name: Check no planning docs committed - run: ./scripts/check_no_planning_docs.sh - - - name: Check CI paths stay in sync - run: ./scripts/check_ci_paths_sync.sh - - - name: Check io_uring lane --lib precondition - run: ./scripts/check_uring_lane_lib_only.sh + - name: Run shared quick checks + uses: ./.github/actions/quick-checks test-and-lint: name: Test and Lint diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 171f52380..5250f8bae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,12 +100,7 @@ jobs: - name: Typos check with custom config file uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master - # Fast, compile-free checks that fail early so contributors get feedback in - # ~1 minute instead of waiting for the full test job. - # - # These steps are mirrored byte-for-byte in ci-docs-only.yml so that a mixed - # PR, which reports two check runs named "Quick Checks", cannot get one red - # and one green. Edit both jobs together. + # Fail early with compile-free checks shared with docs-only CI. quick-checks: name: Quick Checks if: github.event_name != 'pull_request' || github.event.action != 'closed' @@ -117,67 +112,8 @@ jobs: with: persist-credentials: false - - name: Install ripgrep - uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 - with: - tool: ripgrep@15.2.0 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - components: rustfmt - - - name: Check code formatting - run: cargo fmt --all --check - - - name: Check unsafe code allowances - run: ./scripts/check_unsafe_code_allowances.sh - - - name: Check layered dependencies - run: ./scripts/check_layer_dependencies.sh - - - name: Check architecture migration rules - run: ./scripts/check_architecture_migration_rules.sh - - - name: Check logging guardrails - run: ./scripts/check_logging_guardrails.sh - - - name: Check error other(format!) ratchet - run: ./scripts/check_error_other_format_ratchet.sh - - - name: Check tokio io-uring feature guard - run: ./scripts/check_no_tokio_io_uring.sh - - - name: Check extension schema boundaries - run: ./scripts/check_extension_schema_boundaries.sh - - - name: Check body-cache whitelist guard - run: ./scripts/check_body_cache_whitelist.sh - - - name: Check s3s footprint ratchet - run: ./scripts/check_s3s_footprint.sh - - - name: Check cryptographic capability wording - run: ./scripts/check_fips_wording.sh - - - name: Check no embedded secret material - run: ./scripts/check_embedded_secrets.sh - - - name: Check test wiring - run: | - python3 ./scripts/check_test_wiring.py --self-test - python3 ./scripts/check_scheduled_validation_freshness.py --self-test - python3 ./scripts/test_security_workflow.py - python3 ./scripts/check_test_wiring.py - - - name: Check no planning docs committed - run: ./scripts/check_no_planning_docs.sh - - - name: Check CI paths stay in sync - run: ./scripts/check_ci_paths_sync.sh - - - name: Check io_uring lane --lib precondition - run: ./scripts/check_uring_lane_lib_only.sh + - name: Run shared quick checks + uses: ./.github/actions/quick-checks test-and-lint: name: Test and Lint diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 8b1cf1246..63de463d3 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -5,7 +5,9 @@ from __future__ import annotations import hashlib import json +import os import re +import subprocess import sys import tempfile import tomllib @@ -481,18 +483,20 @@ def yaml_block(lines: list[str], key: str, indent: int) -> list[str] | None: return lines[start:end] -def workflow_step_block(job_lines: list[str], action: str) -> tuple[int, list[str]] | None: +def workflow_step_block( + job_lines: list[str], value: str, key: str = "uses", indent: int = 6 +) -> tuple[int, list[str]] | None: uses_index = next( ( index for index, line in enumerate(job_lines) if ( - line.split("#", 1)[0].strip() == f"- uses: {action}" - and len(line) - len(line.lstrip()) == 6 + line.split("#", 1)[0].strip() == f"- {key}: {value}" + and len(line) - len(line.lstrip()) == indent ) or ( - line.split("#", 1)[0].strip() == f"uses: {action}" - and len(line) - len(line.lstrip()) == 8 + line.split("#", 1)[0].strip() == f"{key}: {value}" + and len(line) - len(line.lstrip()) == indent + 2 ) ), None, @@ -520,6 +524,67 @@ def workflow_step_block(job_lines: list[str], action: str) -> tuple[int, list[st return start, job_lines[start:end] +def yaml_scalar_continues(lines: list[str], index: int, indent: int) -> bool: + following = next( + (line for line in lines[index + 1:] if line.strip() and not line.lstrip().startswith("#")), None + ) + return following is not None and len(following) - len(following.lstrip()) > indent + + +def check_quick_checks(root: Path) -> list[str]: + errors: list[str] = [] + bypass_key = r'''(?:if|continue-on-error|needs|"if"|"continue-on-error"|"needs"|'if'|'continue-on-error'|'needs')\s*:''' + for name in ("ci.yml", "ci-docs-only.yml"): + relative = f".github/workflows/{name}" + path = root / relative + job = yaml_block(path.read_text().splitlines(), "quick-checks", 2) if path.is_file() else None + if job is None: + errors.append(f"{relative}: missing Quick Checks job") + continue + conditions = [index for index, line in enumerate(job) if re.match(rf"^ {bypass_key}", line)] + expected = ["if: github.event_name != 'pull_request' || github.event.action != 'closed'"] if name == "ci.yml" else [] + if [job[index].strip() for index in conditions] != expected or any( + yaml_scalar_continues(job, index, 4) for index in conditions + ): + errors.append(f"{relative}: Quick Checks job must not add dependencies, bypass failures, or change its event condition") + checkout = workflow_step_block(job, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0") + action = workflow_step_block(job, "./.github/actions/quick-checks") + if checkout is None or action is None: + errors.append(f"{relative}: Quick Checks requires checkout and the shared quick-checks action") + continue + if checkout[0] >= action[0]: + errors.append(f"{relative}: checkout must run before shared Quick Checks") + if " persist-credentials: false" not in checkout[1]: + errors.append(f"{relative}: Quick Checks checkout must disable persisted credentials") + for step in (checkout, action): + if any(re.match(rf"^\s+(?:- )?{bypass_key}", line) for line in step[1]): + errors.append(f"{relative}: Quick Checks checkout and shared action must run without bypasses") + + relative = ".github/actions/quick-checks/action.yml" + path = root / relative + runs = yaml_block(path.read_text().splitlines(), "runs", 0) if path.is_file() else None + if runs is None or " using: composite" not in runs: + errors.append(f"{relative}: missing composite action") + return errors + steps = yaml_block(runs, "steps", 2) or [] + for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"): + step = workflow_step_block(steps, command, key="run", indent=4) + if step is None: + errors.append(f"{relative}: missing direct execution of {command}") + continue + if " shell: bash" not in step[1] or any( + re.match(rf"^\s+(?:- )?{bypass_key}", line) for line in step[1] + ): + errors.append(f"{relative}: {command} must use bash without a condition or continue-on-error") + run_index = next( + index for index, line in enumerate(step[1]) + if line.split("#", 1)[0].rstrip() in (f" run: {command}", f" - run: {command}") + ) + if yaml_scalar_continues(step[1], run_index, 6): + errors.append(f"{relative}: {command} must remain a single-line run scalar") + return errors + + def alert_step_errors( job_lines: list[str], expected_action_if: str | None, @@ -820,10 +885,139 @@ def validate(root: Path) -> list[str]: errors.extend(check_workflow_readiness(root)) errors.extend(check_profile_definitions(root)) errors.extend(check_scheduled_alerts(root)) + errors.extend(check_quick_checks(root)) return errors class SelfTests(unittest.TestCase): + def test_quick_checks_rejects_caller_and_execution_bypasses(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + caller = ( + "jobs:\n quick-checks:\n steps:\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " with:\n persist-credentials: false\n" + " - uses: ./.github/actions/quick-checks\n" + ) + action = ( + "runs:\n using: composite\n steps:\n" + " - uses: taiki-e/install-action@pinned\n" + " with:\n tool: actionlint@1.7.12\n" + " - name: Lint workflows\n shell: bash\n run: shellcheck --version && actionlint\n" + " - name: Error format ratchet\n shell: bash\n" + " run: ./scripts/check_error_other_format_ratchet.sh\n" + ) + sources = { + ".github/workflows/ci.yml": caller.replace( + " steps:", " if: github.event_name != 'pull_request' || github.event.action != 'closed'\n steps:" + ), + ".github/workflows/ci-docs-only.yml": caller, + ".github/actions/quick-checks/action.yml": action, + } + for relative, source in sources.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + self.assertEqual(check_quick_checks(root), []) + for relative in (".github/workflows/ci.yml", ".github/workflows/ci-docs-only.yml"): + source = sources[relative] + mutations = { + "different action": source.replace("./.github/actions/quick-checks", "./.github/actions/other"), + "conditional call": source + " if: false\n", + "ignored call failure": source + " continue-on-error: true\n", + "conditional checkout": source.replace(" with:", " if: false\n with:"), + "ignored job failure": source.replace(" steps:", " continue-on-error: true\n steps:"), + "changed job condition": ( + source.replace("github.event_name != 'pull_request' || github.event.action != 'closed'", "false") + if relative.endswith("/ci.yml") else source.replace(" steps:", " if: false\n steps:") + ), + "persisted credentials": source.replace("persist-credentials: false", "persist-credentials: true"), + "late checkout": source.replace(" - uses: ./.github/actions/quick-checks\n", "").replace( + " steps:\n", " steps:\n - uses: ./.github/actions/quick-checks\n" + ), + "missing job": source.replace(" quick-checks:", " other-checks:"), + } + for key in ("'if' : false", '"if": false', "'continue-on-error': true", '"continue-on-error" : true'): + mutations[f"quoted call {key}"] = source + f" {key}\n" + mutations[f"quoted checkout {key}"] = source.replace(" with:", f" {key}\n with:") + job_source = source.replace( + " if: github.event_name != 'pull_request' || github.event.action != 'closed'\n", "" + ) if "if" in key else source + mutations[f"quoted job {key}"] = job_source.replace(" steps:", f" {key}\n steps:") + for dependency in ("needs: prerequisite", "needs: [prerequisite]", "needs:\n - prerequisite", "'needs' : [prerequisite]", '"needs": [prerequisite]'): + for condition in ("false", "true"): + prerequisite = f"\n prerequisite:\n if: {condition}\n runs-on: ubuntu-latest\n steps:\n - run: exit 1\n" + mutations[f"job dependency {dependency} if {condition}"] = source.replace(" steps:", f" {dependency}\n steps:") + prerequisite + if relative.endswith("/ci.yml"): + for separator in ("", "\n", " # continued condition\n"): + mutations[f"continued job condition {separator!r}"] = source.replace( + " steps:", f"{separator} && false\n steps:" + ) + for case, mutated in mutations.items(): + with self.subTest(path=relative, case=case): + (root / relative).write_text(mutated) + self.assertTrue(check_quick_checks(root)) + (root / relative).write_text(source) + relative = ".github/actions/quick-checks/action.yml" + mutations = { + "not composite": action.replace("using: composite", "using: node24"), + "only installed actionlint": action.replace("run: shellcheck --version && actionlint", "run: echo actionlint"), + "missing shellcheck preflight": action.replace("shellcheck --version && ", ""), + "missing ratchet": action.replace("run: ./scripts/check_error_other_format_ratchet.sh", "run: echo skipped"), + "swallowed lint failure": action.replace("&& actionlint", "&& actionlint || true"), + "swallowed ratchet failure": action.replace("ratchet.sh", "ratchet.sh || true"), + "conditional lint": action.replace("run: shellcheck", "if: false\n run: shellcheck"), + "ignored ratchet failure": action.replace("run: ./scripts/", "continue-on-error: true\n run: ./scripts/"), + "non-failing shell": action.replace("shell: bash", "shell: bash {0}"), + "run text in step name": action.replace( + "name: Lint workflows", "name: |\n run: shellcheck --version && actionlint" + ).replace("\n run: shellcheck --version && actionlint\n", "\n run: shellcheck --version && actionlint\n || true\n"), + } + for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"): + for key in ("'if' : false", '"if": false', "'continue-on-error': true", '"continue-on-error" : true'): + mutations[f"quoted {command} {key}"] = action.replace(f"run: {command}", f"{key}\n run: {command}") + for separator in ("", "\n", " # continued command\n"): + mutations[f"continued {command} {separator!r}"] = action.replace( + f"run: {command}\n", f"run: {command}\n{separator} || true\n" + ) + for case, mutated in mutations.items(): + with self.subTest(case=case): + (root / relative).write_text(mutated) + self.assertTrue(check_quick_checks(root)) + (root / relative).unlink() + self.assertTrue(check_quick_checks(root)) + + def test_quick_checks_commands_propagate_failure(self) -> None: + runs = yaml_block((ROOT / ".github/actions/quick-checks/action.yml").read_text().splitlines(), "runs", 0) + steps = yaml_block(runs or [], "steps", 2) or [] + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "scripts").mkdir() + commands = ("shellcheck", "actionlint", "./scripts/check_error_other_format_ratchet.sh") + for failing in commands: + with self.subTest(command=failing): + run = "shellcheck --version && actionlint" if failing != commands[-1] else failing + step = workflow_step_block(steps, run, key="run", indent=4) + self.assertIsNotNone(step) + run_index = next(index for index, line in enumerate(step[1]) if line.startswith(" run:")) + self.assertFalse(yaml_scalar_continues(step[1], run_index, 6)) + body = step[1][run_index].removeprefix(" run: ") + for command in commands: + shim = root / command + shim.write_text(f"#!/bin/sh\nexit {17 if command == failing else 0}\n") + shim.chmod(0o755) + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", body], + cwd=root, env=dict(os.environ, PATH=f"{root}{os.pathsep}{os.environ['PATH']}"), + capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 17, result.stderr) + + def test_validate_includes_quick_checks(self) -> None: + error = "Quick Checks wiring regression" + with mock.patch(__name__ + ".check_quick_checks", return_value=[error]): + self.assertIn(error, validate(ROOT)) + def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -1058,6 +1252,7 @@ class SelfTests(unittest.TestCase): mock.patch(__name__ + ".check_profile_definitions", return_value=[]), mock.patch(__name__ + ".check_ilm_build_budget", return_value=[]), mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]), + mock.patch(__name__ + ".check_quick_checks", return_value=[]), ): self.assertEqual(len(validate(root)), 1) @@ -1498,7 +1693,7 @@ def main() -> int: for error in errors: print(f"ERROR: {error}", file=sys.stderr) return 1 - print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired") + print("OK: e2e modules, runner selection, fuzz matrices, profiles, scheduled alerts, and Quick Checks are wired") return 0 From 3677871468355860e95d82c7efec6301a23c4e2d Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 20:16:46 +0800 Subject: [PATCH 29/40] chore(deps): bump zstd to 0.14 (#7173) Signed-off-by: houseme Co-authored-by: zhi22915 --- Cargo.lock | 44 +++++++++++++++++++++++++++++++------------- Cargo.toml | 10 +++++----- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8206b1951..f4cc41aae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -315,7 +315,7 @@ dependencies = [ "strum", "thiserror 2.0.20", "uuid", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -508,7 +508,7 @@ dependencies = [ "arrow-select", "flatbuffers", "lz4_flex", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -2249,8 +2249,8 @@ dependencies = [ "liblzma", "lz4", "memchr", - "zstd", - "zstd-safe", + "zstd 0.13.3", + "zstd-safe 7.3.0", ] [[package]] @@ -4067,7 +4067,7 @@ dependencies = [ "uuid", "walkdir", "zip", - "zstd", + "zstd 0.14.0", ] [[package]] @@ -5971,7 +5971,7 @@ dependencies = [ "lz4", "snap", "uuid", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -7658,7 +7658,7 @@ dependencies = [ "snap", "tokio", "twox-hash", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -9618,7 +9618,7 @@ dependencies = [ "x509-parser", "zeroize", "zip", - "zstd", + "zstd 0.14.0", ] [[package]] @@ -10228,7 +10228,7 @@ dependencies = [ "thiserror 2.0.20", "walkdir", "zip", - "zstd", + "zstd 0.14.0", ] [[package]] @@ -10395,7 +10395,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", - "zstd", + "zstd 0.14.0", ] [[package]] @@ -10985,7 +10985,7 @@ dependencies = [ "transform-stream", "url", "windows", - "zstd", + "zstd 0.14.0", ] [[package]] @@ -14095,7 +14095,7 @@ dependencies = [ "typed-path", "zeroize", "zopfli", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -14128,7 +14128,16 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe", + "zstd-safe 7.3.0", +] + +[[package]] +name = "zstd" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" +dependencies = [ + "zstd-safe 8.0.0", ] [[package]] @@ -14140,6 +14149,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.1.0+zstd.1.5.7" diff --git a/Cargo.toml b/Cargo.toml index a7ecf889b..871d8cb71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -199,10 +199,10 @@ serde_urlencoded = "0.7.1" # matching stable releases are not available yet, while previous stable lines # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable # releases. -aes-gcm = { version = "=0.11.1" } -argon2 = { version = "=0.6.0" } -blake2 = "=0.11.0" -chacha20poly1305 = { version = "=0.11.0" } +aes-gcm = { version = "0.11.1" } +argon2 = { version = "0.6.0" } +blake2 = "0.11.0" +chacha20poly1305 = { version = "0.11.0" } crc-fast = "1.10.0" hmac = { version = "0.13.0" } jsonwebtoken = { version = "11.0.0" } @@ -343,7 +343,7 @@ windows = { version = "0.62.2" } windows-sys = "0.61.2" xxhash-rust = { version = "0.8.18" } zip = "8.6.0" -zstd = "0.13.3" +zstd = "0.14.0" # Observability and Metrics metrics = "0.24.6" From 3e5d4ebb09ac7437f4a9a462d67a3511d103ecb8 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 20:20:50 +0800 Subject: [PATCH 30/40] fix(ecstore): release multipart disk snapshot before nested reads (#7184) * fix(ecstore): release multipart disk snapshot before nested reads * fix(ecstore): remove duplicate local rename implementation Keep the canonical commit module after concurrent storage changes merged. The control-write and rollback changes are already present there. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(app): simplify absent SSE configuration matching * fix(tests): satisfy new clippy lints --------- Co-authored-by: houseme Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/set_disk/ops/multipart.rs | 88 +++++++++++++++++++- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index d7d3a39ad..aaf6204b5 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -2452,10 +2452,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let write_quorum = fi.write_quorum(self.default_write_quorum()); let read_quorum = fi.read_quorum(self.default_read_quorum()); - let disks = self.disks.read().await; - - let disks = disks.clone(); - // let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution); + // Release the registry guard before recovery and cleanup read it again: + // a queued topology writer would otherwise deadlock those nested reads. + let disks = self.get_disks_internal().await; let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil())); self.recover_part_transactions(&part_path, read_quorum, write_quorum) @@ -6743,6 +6742,87 @@ mod tests { .await; } + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn complete_multipart_releases_disk_snapshot_before_cleanup() { + let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-topology-lock-bucket"; + let object = "object"; + let body = vec![0x65; 4096]; + make_bucket_on_all(&disk_stores, bucket).await; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, object, &body, &ObjectOptions::default()).await; + let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id); + for dir in &temp_dirs { + assert!( + dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(), + "the test must create real upload staging on every disk" + ); + } + let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication); + let complete_store = set_disks.clone(); + let complete_upload_id = upload_id.clone(); + let complete = tokio::spawn(async move { + complete_store + .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + + // Hold a separate read gate so the real writer queues even when completion + // correctly releases its snapshot guard. Polling Pending proves admission + // to Tokio's write-preferring queue before the cleanup attempts another read. + let read_gate = set_disks.disks.read().await; + let writer = set_disks.disks.write(); + tokio::pin!(writer); + assert!(matches!( + futures::poll!(tokio::task::unconstrained(writer.as_mut())), + std::task::Poll::Pending + )); + assert!( + set_disks.disks.try_read().is_err(), + "the pending writer must already block new readers before the cleanup resumes" + ); + drop(read_gate); + barrier.release(); + + let writer_guard = tokio::time::timeout(Duration::from_secs(5), writer) + .await + .expect("a queued topology writer must not deadlock with multipart cleanup's disk snapshot"); + // A reconnect can publish the same handles; this test isolates admission + // order without changing the disks that contain the committed object. + drop(writer_guard); + tokio::time::timeout(Duration::from_secs(10), complete) + .await + .expect("multipart cleanup must finish after the topology writer releases") + .expect("completion task should not panic") + .expect("completion should preserve the successful object commit"); + + let mut reader = tokio::time::timeout( + Duration::from_secs(10), + set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()), + ) + .await + .expect("GET should finish after completion") + .expect("the completed object should remain readable"); + let mut observed_body = Vec::new(); + tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut observed_body)) + .await + .expect("the completed object body should finish streaming") + .expect("the completed object body should be readable"); + assert_eq!(observed_body, body); + assert!(matches!( + set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await, + Err(StorageError::InvalidUploadID(..)) + )); + for dir in &temp_dirs { + assert!( + !dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(), + "successful completion must remove its upload staging from every disk" + ); + } + } + #[tokio::test(flavor = "multi_thread")] #[serial] async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() { From c589fd24399cf461a0b060f922cc34df5d4318b1 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 20:22:56 +0800 Subject: [PATCH 31/40] fix(dev): install a lightweight formatting commit hook (#7198) --- .config/make/pre-commit.mak | 5 ++-- .pre-commit-config.yaml | 6 ++--- CONTRIBUTING.md | 48 +++++++++---------------------------- 3 files changed, 17 insertions(+), 42 deletions(-) diff --git a/.config/make/pre-commit.mak b/.config/make/pre-commit.mak index 716b182f4..c23adc230 100644 --- a/.config/make/pre-commit.mak +++ b/.config/make/pre-commit.mak @@ -3,9 +3,10 @@ .NOTPARALLEL: pre-commit pre-pr dev-check .PHONY: setup-hooks -setup-hooks: ## Set up git hooks +setup-hooks: ## Install the configured pre-commit hooks @echo "🔧 Setting up git hooks..." - chmod +x .git/hooks/pre-commit + pre-commit validate-config + pre-commit install @echo "✅ Git hooks setup complete!" .PHONY: doc-paths-check diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 947c4ffbb..083bf9dff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,9 +3,9 @@ repos: - repo: local hooks: - - id: rustfs-dev-check - name: rustfs dev-check - entry: make dev-check + - id: rustfs-fmt-check + name: Rust formatting + entry: cargo fmt --all --check language: system types: [rust] pass_filenames: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 13bf35c08..4af869ef5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,24 +109,17 @@ affected boundaries and risks. CI still runs its configured repository gates. ### 🔒 Git Pre-commit Hooks (optional) -Git hooks are **not** versioned in this repository, so a fresh clone has no -active pre-commit hook. If you add your own `.git/hooks/pre-commit` (a good -choice is a one-liner that runs `make pre-commit`), you can mark it executable -with: +The optional hook uses the checked-in `.pre-commit-config.yaml`. Install [pre-commit](https://pre-commit.com/#installation), then run this from the checkout or a linked worktree: ```bash make setup-hooks ``` -Or manually: +The hook runs `cargo fmt --all --check` when staged files include Rust source. It does not compile the workspace or run tests. Fix formatting with `cargo fmt --all`, inspect and stage the result, then commit again. -```bash -chmod +x .git/hooks/pre-commit -``` +`pre-commit install` resolves Git's hook directory for linked worktrees and preserves an existing hook in migration mode. If you use `core.hooksPath`, keep that hook manager and integrate `pre-commit run` there; the installer refuses to silently replace that configuration. -With or without a hook, follow the verification tiers in `AGENTS.md`. Run the -applicable scoped checks, and reserve `make pre-pr` for broad cross-module -changes whose impact cannot be bounded by those checks. +A local hook provides early formatting feedback. With or without it, follow the verification tiers in `AGENTS.md`, run relevant behavioral tests, and satisfy the CI merge gates. `make pre-commit` and `make dev-check` remain explicit broader commands. ### 📝 Formatting Configuration @@ -138,31 +131,11 @@ fn_call_width = 90 single_line_let_else_max_width = 100 ``` -### 🚫 Commit Prevention - -If you set up a pre-commit hook and your code doesn't meet the formatting requirements, the hook will: - -1. **Block the commit** and show clear error messages -2. **Provide exact commands** to fix the issues -3. **Guide you through** the resolution process - -Example output when formatting fails: - -``` -❌ Code formatting check failed! -💡 Please run 'cargo fmt --all' to format your code before committing. - -🔧 Quick fix: - cargo fmt --all - git add . - git commit -``` - ### 🔄 Development Workflow 1. **Make your changes** 2. **Format your code**: `make fmt` or `cargo fmt --all` -3. **Run the fast gate**: `make pre-commit` (no clippy, no tests) +3. **Select relevant checks** using the validation tier in `AGENTS.md`; use `make pre-commit` when its broader fast gate adds useful coverage 4. **Commit your changes**: `git commit -m "your message"` 5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`) 6. **Run applicable scoped checks before opening/updating a PR**; consider @@ -206,11 +179,12 @@ Configure your IDE to: #### Pre-commit hook not running? ```bash -# Check if hook is executable -ls -la .git/hooks/pre-commit - -# Make it executable if needed -chmod +x .git/hooks/pre-commit +pre-commit validate-config +pre-commit run --all-files +# Inspect any configured hook manager; do not overwrite it. +git config --get core.hooksPath +# Install if no separate hook manager is configured. +make setup-hooks ``` #### Formatting issues? From 0a92a7d98c4f8cc8eb10f69e754596181af09c15 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 5 Sep 2026 20:41:11 +0800 Subject: [PATCH 32/40] fix(tier): bound remote transition requests (#7147) Co-authored-by: Zhengchao An --- crates/config/README.md | 15 + crates/config/src/constants/object.rs | 32 ++ .../ecstore/src/services/tier/warm_backend.rs | 26 +- .../src/services/tier/warm_backend_s3.rs | 5 +- crates/s3-client/src/api_get_object.rs | 252 +++++++++--- crates/s3-client/src/api_list.rs | 92 ++++- .../s3-client/src/api_put_object_multipart.rs | 12 +- crates/s3-client/src/api_remove.rs | 12 +- crates/s3-client/src/api_stat.rs | 83 +++- crates/s3-client/src/bucket_cache.rs | 15 +- crates/s3-client/src/transition_api.rs | 369 ++++++++++++++++-- .../replication-outbound-transport.md | 12 + 12 files changed, 783 insertions(+), 142 deletions(-) diff --git a/crates/config/README.md b/crates/config/README.md index c450c05b0..2313a41ec 100644 --- a/crates/config/README.md +++ b/crates/config/README.md @@ -130,6 +130,21 @@ Scanner cycle budget controls: - timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling. - this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`. +## Remote tier timeout environment variables + +- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS` + - remote tier TCP connect timeout. + - default is `10`. + - must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. +- `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS` + - remote tier request timeout through response headers. + - default is `86400` so large transition uploads keep a production-safe budget. + - must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. Very large values are accepted and act as a correspondingly long budget. +- `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS` + - maximum idle time between remote tier response-body chunks. + - default is `60`; the timer resets only when non-empty body data keeps progressing. + - must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. + ## Drive timeout environment variables - `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS` diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index 9af74f2a3..7ec459afd 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -137,6 +137,28 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false; const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE); const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED); +/// Environment variable for remote tier TCP connect timeout in seconds. +pub const ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS"; +/// Default remote tier TCP connect timeout in seconds. +pub const DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS: u64 = 10; + +/// Environment variable for the remote tier request timeout in seconds. +/// +/// This bounds upload/download request progress through response headers. The +/// default is intentionally large so multi-TiB transition uploads keep their +/// previous production budget while black-hole remotes no longer wait forever. +pub const ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS"; +/// Default remote tier request timeout in seconds. +pub const DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS: u64 = 24 * 60 * 60; + +/// Environment variable for remote tier response-body idle timeout in seconds. +/// +/// The timer is re-armed on every non-empty response-body chunk, so slow but +/// progressing remotes can continue while silent response bodies are cancelled. +pub const ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS"; +/// Default remote tier response-body idle timeout in seconds. +pub const DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: u64 = 60; + /// Request the object-transaction fencing contract used by storage-owned /// cleanup receipts and lock-window optimizations. /// @@ -812,6 +834,16 @@ mod remote_version_state_tests { ); } + #[test] + fn remote_tier_timeout_env_names_are_stable() { + assert_eq!(super::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS"); + assert_eq!(super::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS"); + assert_eq!( + super::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS, + "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS" + ); + } + #[test] fn data_movement_part_checksum_gate_uses_stable_environment_names() { assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE"); diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index b5cf4ab38..22134d744 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -37,7 +37,7 @@ use crate::services::tier::{ use bytes::Bytes; use http::StatusCode; use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value}; -use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore}; +use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore}; use rustfs_s3_client::{ admin_handler_utils::AdminError, api_error_response::to_error_response, @@ -320,6 +320,27 @@ pub(crate) fn endpoint_authority(url: &url::Url) -> Result Duration { + Duration::from_secs(rustfs_utils::get_env_u64(env_key, default_secs)) +} + +pub(crate) fn transition_client_timeouts_from_env() -> TransitionClientTimeouts { + TransitionClientTimeouts::new( + transition_timeout_from_env( + rustfs_config::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS, + rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS, + ), + transition_timeout_from_env( + rustfs_config::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS, + rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS, + ), + transition_timeout_from_env( + rustfs_config::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS, + rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS, + ), + ) +} + /// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers. /// /// Credential, bucket, and endpoint validation run in this order because the @@ -350,6 +371,7 @@ pub(crate) async fn new_s3_compatible_warm_backend( signer_type: SignatureType::SignatureV4, ..Default::default() })); + let timeouts = transition_client_timeouts_from_env(); let opts = Options { creds, secure: u.scheme() == "https", @@ -362,7 +384,7 @@ pub(crate) async fn new_s3_compatible_warm_backend( // Run the SSRF guard after the host-presence check so a host-less endpoint // keeps this constructor's stable error text. (params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; - let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?; + let client = TransitionClient::new_with_timeouts(&endpoint, opts, params.provider_tag, timeouts).await?; let client = Arc::new(client); let core = TransitionCore(Arc::clone(&client)); diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index b830ea7f2..268f28596 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -26,7 +26,7 @@ use crate::services::tier::{ tier_config::TierS3, warm_backend::{ TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts, - build_transition_put_options, endpoint_authority, + build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env, }, }; use http::HeaderMap; @@ -139,6 +139,7 @@ impl WarmBackendS3 { } else { return Err(std::io::Error::other("insufficient parameters for S3 backend authentication")); } + let timeouts = transition_client_timeouts_from_env(); let opts = Options { creds, secure: u.scheme() == "https", @@ -147,7 +148,7 @@ impl WarmBackendS3 { ..Default::default() }; let endpoint = endpoint_authority(&u)?; - let client = TransitionClient::new(&endpoint, opts, tier_type).await?; + let client = TransitionClient::new_with_timeouts(&endpoint, opts, tier_type, timeouts).await?; let client = Arc::new(client); let core = TransitionCore(Arc::clone(&client)); diff --git a/crates/s3-client/src/api_get_object.rs b/crates/s3-client/src/api_get_object.rs index 872c1a90d..6cbe2fc02 100644 --- a/crates/s3-client/src/api_get_object.rs +++ b/crates/s3-client/src/api_get_object.rs @@ -120,18 +120,10 @@ impl TransitionClient { let h = resp.headers().clone(); - let mut body = resp.into_body(); let body_vec = if let Some(limit) = max_response_bytes { - collect_response_body(body, limit).await? + self.collect_response_body(resp.into_body(), limit).await? } else { - let mut body_vec = Vec::new(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; - if let Some(data) = frame.data_ref() { - body_vec.extend_from_slice(data); - } - } - body_vec + self.collect_response_body_unbounded(resp.into_body()).await? }; Ok((object_stat, h, BufReader::new(Cursor::new(body_vec)))) } @@ -143,7 +135,7 @@ mod bounded_response_tests { use crate::{ api_get_options::GetObjectOptions, credentials::{Credentials, SignatureType, Static, Value}, - transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body}, + transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, collect_response_body}, }; use http_body_util::Full; use hyper::body::Bytes; @@ -175,7 +167,31 @@ mod bounded_response_tests { assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); } - async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle)> { + fn test_options() -> Options { + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + } + } + + async fn client_for_endpoint(endpoint: &str, timeouts: TransitionClientTimeouts) -> TransitionClient { + TransitionClient::new_with_timeouts(endpoint, test_options(), "", timeouts) + .await + .expect("fixture client should build") + } + + async fn bounded_get_fixture_with_timeouts( + body: &'static [u8], + timeouts: TransitionClientTimeouts, + ) -> Option<(TransitionClient, tokio::task::JoinHandle)> { let listener = match TcpListener::bind("127.0.0.1:0").await { Ok(listener) => listener, Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, @@ -209,27 +225,14 @@ mod bounded_response_tests { stream.write_all(body).await.expect("fixture should write response body"); request }); - let client = TransitionClient::new( - &endpoint, - Options { - creds: Credentials::new(Static(Value { - access_key_id: "access-key".to_string(), - secret_access_key: "secret-key".to_string(), - signer_type: SignatureType::SignatureV4, - ..Default::default() - })), - region: "us-east-1".to_string(), - bucket_lookup: BucketLookupType::BucketLookupPath, - max_retries: 1, - ..Default::default() - }, - "", - ) - .await - .expect("fixture client should build"); + let client = client_for_endpoint(&endpoint, timeouts).await; Some((client, request)) } + async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle)> { + bounded_get_fixture_with_timeouts(body, TransitionClientTimeouts::default()).await + } + #[tokio::test] async fn real_transport_accepts_the_exact_closed_range_length() { let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else { @@ -292,24 +295,7 @@ mod bounded_response_tests { .local_addr() .expect("listener local address should be available") .to_string(); - let client = TransitionClient::new( - &endpoint, - Options { - creds: Credentials::new(Static(Value { - access_key_id: "access-key".to_string(), - secret_access_key: "secret-key".to_string(), - signer_type: SignatureType::SignatureV4, - ..Default::default() - })), - region: "us-east-1".to_string(), - bucket_lookup: BucketLookupType::BucketLookupPath, - max_retries: 1, - ..Default::default() - }, - "", - ) - .await - .expect("fixture client should build"); + let client = client_for_endpoint(&endpoint, TransitionClientTimeouts::default()).await; let mut opts = GetObjectOptions::default(); opts.headers .insert("range".to_string(), "bytes=0-18446744073709551615".to_string()); @@ -326,6 +312,176 @@ mod bounded_response_tests { .is_err() ); } + + #[tokio::test] + async fn connection_refused_returns_without_waiting_for_the_request_timeout() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + drop(listener); + + let client = client_for_endpoint( + &endpoint, + TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(5), Duration::from_secs(1)), + ) + .await; + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + + let result = tokio::time::timeout(Duration::from_secs(2), client.get_object_inner("bucket", "probe", &opts)) + .await + .expect("connection refused should return before the broader request timeout"); + + assert!(result.is_err(), "connection refused must fail instead of hanging"); + } + + #[tokio::test] + async fn response_header_stall_returns_timed_out() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let fixture = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + loop { + let read = stream.read(&mut buffer).await.expect("fixture should read request headers"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + tokio::time::sleep(Duration::from_millis(200)).await; + }); + let client = client_for_endpoint( + &endpoint, + TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)), + ) + .await; + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + + let err = client + .get_object_inner("bucket", "probe", &opts) + .await + .expect_err("response header stalls must be bounded"); + + assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); + fixture.await.expect("fixture should join"); + } + + #[tokio::test] + async fn response_body_idle_stall_returns_timed_out() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let fixture = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + loop { + let read = stream.read(&mut buffer).await.expect("fixture should read request headers"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + stream + .write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\nRu") + .await + .expect("fixture should write the first body chunk"); + tokio::time::sleep(Duration::from_millis(200)).await; + }); + let client = client_for_endpoint( + &endpoint, + TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)), + ) + .await; + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + + let err = client + .get_object_inner("bucket", "probe", &opts) + .await + .expect_err("body stalls after partial progress must be bounded"); + + assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); + fixture.await.expect("fixture should join"); + } + + #[tokio::test] + async fn response_body_idle_timer_resets_on_progress() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let fixture = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + loop { + let read = stream.read(&mut buffer).await.expect("fixture should read request headers"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + stream + .write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\n") + .await + .expect("fixture should write response headers"); + for byte in b"RustFS!" { + stream.write_all(&[*byte]).await.expect("fixture should write body progress"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + }); + let client = client_for_endpoint( + &endpoint, + TransitionClientTimeouts::new(Duration::from_millis(10), Duration::from_secs(1), Duration::from_millis(100)), + ) + .await; + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + + let (_, _, mut reader) = client + .get_object_inner("bucket", "probe", &opts) + .await + .expect("continuous body progress must not be killed by the idle timer"); + let mut body = Vec::new(); + reader + .read_to_end(&mut body) + .await + .expect("bounded response should be readable"); + + assert_eq!(body, b"RustFS!"); + fixture.await.expect("fixture should join"); + } } #[derive(Default)] diff --git a/crates/s3-client/src/api_list.rs b/crates/s3-client/src/api_list.rs index dab74cf84..0bdebbf5c 100644 --- a/crates/s3-client/src/api_list.rs +++ b/crates/s3-client/src/api_list.rs @@ -27,7 +27,6 @@ use crate::{ transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body}, }; use http::{HeaderMap, StatusCode}; -use http_body_util::BodyExt; use hyper::body::Body; use hyper::body::Bytes; use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE; @@ -124,14 +123,9 @@ impl TransitionClient { } //let mut list_bucket_result = ListBucketV2Result::default(); - let mut body_vec = Vec::new(); - let mut body = resp.into_body(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; - if let Some(data) = frame.data_ref() { - body_vec.extend_from_slice(data); - } - } + let body_vec = self + .collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE) + .await?; let mut list_bucket_result = match quick_xml::de::from_str::(&String::from_utf8_lossy(&body_vec)) { Ok(result) => result, Err(err) => { @@ -214,7 +208,9 @@ impl TransitionClient { let resp_status = resp.status(); let headers = resp.headers().clone(); - let body = collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE).await?; + let body = self + .collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE) + .await?; if resp_status != StatusCode::OK { return Err(std::io::Error::other(http_resp_to_error_response( resp_status, @@ -428,6 +424,30 @@ fn decode_s3_name(name: &str, encoding_type: &str) -> Result Options { + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + } + } #[test] fn list_versions_xml_preserves_versions_and_delete_markers() { @@ -525,4 +545,56 @@ mod tests { assert_eq!(parsed.common_prefixes.len(), 1); assert_eq!(parsed.common_prefixes[0].prefix, "subdir/"); } + + #[tokio::test] + async fn list_objects_v2_body_stall_returns_timed_out() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let fixture = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("fixture should accept one list request"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + loop { + let read = stream.read(&mut buffer).await.expect("fixture should read request headers"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 512\r\nConnection: close\r\n\r\nwarm") + .await + .expect("fixture should write a partial list response"); + tokio::time::sleep(Duration::from_millis(200)).await; + }); + let client = TransitionClient::new_with_timeouts( + &endpoint, + timeout_test_options(), + "", + TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)), + ) + .await + .expect("fixture client should build"); + client + .bucket_loc_cache + .lock() + .expect("location cache should lock") + .set("bucket", "us-east-1"); + + let err = client + .list_objects_v2_query("bucket", "", "", false, false, "", "", 1, HeaderMap::new()) + .await + .expect_err("a stalled ListObjectsV2 body must be bounded"); + + assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); + fixture.await.expect("fixture should join"); + } } diff --git a/crates/s3-client/src/api_put_object_multipart.rs b/crates/s3-client/src/api_put_object_multipart.rs index d7d1ba7a1..8262a6598 100644 --- a/crates/s3-client/src/api_put_object_multipart.rs +++ b/crates/s3-client/src/api_put_object_multipart.rs @@ -18,7 +18,6 @@ #![allow(clippy::all)] use http::{HeaderMap, HeaderName, StatusCode}; -use http_body_util::BodyExt; use hyper::body::Bytes; use s3s::S3ErrorCode; use std::collections::HashMap; @@ -247,14 +246,9 @@ impl TransitionClient { // Parse the CreateMultipartUpload response for the UploadId. Returning a // default (empty) result here made every multipart transition fail at the // first UploadPart with "UploadID cannot be empty" (rustfs/rustfs#4811). - let mut body_vec = Vec::new(); - let mut body = resp.into_body(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|e| std::io::Error::other(e.to_string()))?; - if let Some(data) = frame.data_ref() { - body_vec.extend_from_slice(data); - } - } + let body_vec = self + .collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE) + .await?; let initiate_multipart_upload_result = quick_xml::de::from_str::(&String::from_utf8_lossy(&body_vec)) .map_err(|e| std::io::Error::other(format!("failed to parse CreateMultipartUpload response: {e}")))?; diff --git a/crates/s3-client/src/api_remove.rs b/crates/s3-client/src/api_remove.rs index 5e063e14f..4a552cfcd 100644 --- a/crates/s3-client/src/api_remove.rs +++ b/crates/s3-client/src/api_remove.rs @@ -19,7 +19,6 @@ #![allow(clippy::all)] use http::{HeaderMap, HeaderValue, Method, StatusCode}; -use http_body_util::BodyExt; use hyper::body::Body; use hyper::body::Bytes; use rustfs_utils::HashAlgorithm; @@ -351,14 +350,9 @@ impl TransitionClient { ) .await?; - let mut body_vec = Vec::new(); - let mut body = resp.into_body(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; - if let Some(data) = frame.data_ref() { - body_vec.extend_from_slice(data); - } - } + let body_vec = self + .collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE) + .await?; process_remove_multi_objects_response( ReaderImpl::Body(Bytes::from(body_vec)), bucket_name, diff --git a/crates/s3-client/src/api_stat.rs b/crates/s3-client/src/api_stat.rs index bac59c8ed..9386a209e 100644 --- a/crates/s3-client/src/api_stat.rs +++ b/crates/s3-client/src/api_stat.rs @@ -19,7 +19,6 @@ #![allow(clippy::all)] use http::{HeaderMap, HeaderValue, StatusCode}; -use http_body_util::BodyExt; use hyper::body::Body; use hyper::body::Bytes; use rustfs_utils::EMPTY_STRING_SHA256_HASH; @@ -119,14 +118,9 @@ impl TransitionClient { let resp_status = resp.status(); let h = resp.headers().clone(); - let mut body_vec = Vec::new(); - let mut body = resp.into_body(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; - if let Some(data) = frame.data_ref() { - body_vec.extend_from_slice(data); - } - } + let body_vec = self + .collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE) + .await?; let resperr = http_resp_to_error_response(resp_status, &h, body_vec, bucket_name, ""); warn!("bucket exists, resperr: {:?}", resperr); @@ -170,11 +164,13 @@ impl TransitionClient { let resp_status = resp.status(); let h = resp.headers().clone(); - let body_vec = collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE).await?; + let body_vec = self + .collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE) + .await?; parse_bucket_versioning_response(resp_status, &h, body_vec, bucket_name) } - Err(err) => Err(std::io::Error::other(err)), + Err(err) => Err(err), } } @@ -274,8 +270,14 @@ impl TransitionClient { #[cfg(test)] mod tests { use super::parse_bucket_versioning_response; + use crate::{ + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts}, + }; use http::{HeaderMap, StatusCode}; use s3s::dto::BucketVersioningStatus; + use std::time::Duration; + use tokio::{io::AsyncReadExt, net::TcpListener}; #[test] fn parses_bucket_versioning_statuses_mfa_delete_and_unversioned_state() { @@ -338,4 +340,63 @@ mod tests { assert_eq!(strict_err.kind(), std::io::ErrorKind::InvalidData); } } + + #[tokio::test] + async fn get_bucket_versioning_preserves_request_timeout_kind() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let fixture = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("fixture should accept one versioning request"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + loop { + let read = stream.read(&mut buffer).await.expect("fixture should read request headers"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + tokio::time::sleep(Duration::from_millis(200)).await; + }); + let client = TransitionClient::new_with_timeouts( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "", + TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)), + ) + .await + .expect("fixture client should build"); + client + .bucket_loc_cache + .lock() + .expect("location cache should lock") + .set("bucket", "us-east-1"); + + let err = client + .get_bucket_versioning("bucket") + .await + .expect_err("a stalled versioning request must time out"); + + assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); + fixture.await.expect("fixture should join"); + } } diff --git a/crates/s3-client/src/bucket_cache.rs b/crates/s3-client/src/bucket_cache.rs index c891d65a0..c3429da19 100644 --- a/crates/s3-client/src/bucket_cache.rs +++ b/crates/s3-client/src/bucket_cache.rs @@ -26,7 +26,6 @@ use crate::{ transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient}, }; use http::Request; -use http_body_util::BodyExt; use hyper::StatusCode; use hyper::body::Body; use hyper::body::Bytes; @@ -86,7 +85,7 @@ impl TransitionClient { let req = self.get_bucket_location_request(bucket_name)?; let mut resp = self.doit(req).await?; - location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?; + location = process_bucket_location_response(self, resp, bucket_name, &self.tier_type).await?; { if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() { bucket_loc_cache.set(bucket_name, &location); @@ -198,6 +197,7 @@ impl TransitionClient { } async fn process_bucket_location_response( + client: &TransitionClient, mut resp: http::Response, bucket_name: &str, tier_type: &str, @@ -237,14 +237,9 @@ async fn process_bucket_location_response( } //} - let mut body_vec = Vec::new(); - let mut body = resp.into_body(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; - if let Some(data) = frame.data_ref() { - body_vec.extend_from_slice(data); - } - } + let body_vec = client + .collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE) + .await?; let mut location = "".to_string(); if tier_type == "huaweicloud" { if let Ok(body_str) = String::from_utf8(body_vec) { diff --git a/crates/s3-client/src/transition_api.rs b/crates/s3-client/src/transition_api.rs index 6f45c76e2..bd3c62fb5 100644 --- a/crates/s3-client/src/transition_api.rs +++ b/crates/s3-client/src/transition_api.rs @@ -41,7 +41,7 @@ use http::{ request::{Builder, Request}, }; use http_body::Body; -use http_body_util::{BodyExt, LengthLimitError, Limited}; +use http_body_util::BodyExt; use hyper::body::Bytes; use hyper::body::Incoming; use hyper_rustls::{ConfigBuilderExt, HttpsConnector}; @@ -67,10 +67,12 @@ use s3s::dto::Owner; use s3s::dto::ReplicationStatus; use serde::{Deserialize, Serialize}; use sha2::Sha256; +use std::error::Error as StdError; use std::io::Cursor; use std::pin::Pin; use std::sync::atomic::{AtomicI32, Ordering}; use std::task::{Context, Poll}; +use std::time::Duration as StdDuration; use std::{ collections::HashMap, sync::{Arc, Mutex}, @@ -79,28 +81,108 @@ use time::Duration; use time::OffsetDateTime; use tokio::io::BufReader; use tokio::io::{AsyncRead, AsyncReadExt}; -use tracing::{debug, error, warn}; +use tracing::{debug, error, trace, warn}; use url::{Url, form_urlencoded}; use uuid::Uuid; const C_USER_AGENT: &str = "RustFS (linux; x86)"; pub const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024; +const EVENT_TIER_REMOTE_TRANSPORT: &str = "tier_remote_transport"; +const LOG_COMPONENT_S3_CLIENT: &str = "s3_client"; +const LOG_SUBSYSTEM_TIER: &str = "tier"; const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT]; +fn response_body_exceeds_limit_error() -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit") +} + +fn remote_tier_timeout_error(message: &'static str) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::TimedOut, message) +} + +fn source_chain_has_io_kind(error: &(dyn StdError + 'static), kind: std::io::ErrorKind) -> bool { + let mut current = Some(error); + while let Some(error) = current { + if error + .downcast_ref::() + .is_some_and(|io_error| io_error.kind() == kind) + { + return true; + } + current = error.source(); + } + false +} + +fn transition_transport_error(err: hyper_util::client::legacy::Error) -> std::io::Error { + if source_chain_has_io_kind(&err, std::io::ErrorKind::TimedOut) { + return remote_tier_timeout_error("remote tier connection timed out"); + } + std::io::Error::other(err) +} + +async fn next_response_body_data( + mut body: Pin<&mut B>, + idle_timeout: Option, +) -> Result, std::io::Error> +where + B: Body, + B::Error: Into>, +{ + let next_nonempty_data = async { + loop { + let Some(frame) = std::future::poll_fn(|cx| body.as_mut().poll_frame(cx)).await else { + return Ok(None); + }; + let frame = frame.map_err(std::io::Error::other)?; + let Ok(data) = frame.into_data() else { + continue; + }; + if !data.is_empty() { + return Ok(Some(data)); + } + } + }; + + if let Some(idle_timeout) = idle_timeout { + tokio::time::timeout(idle_timeout, next_nonempty_data) + .await + .map_err(|_| remote_tier_timeout_error("remote tier response body stalled"))? + } else { + next_nonempty_data.await + } +} + +async fn collect_response_body_inner( + body: B, + limit: Option, + idle_timeout: Option, +) -> Result, std::io::Error> +where + B: Body, + B::Error: Into>, +{ + let mut body_vec = Vec::new(); + let mut body = std::pin::pin!(body); + while let Some(data) = next_response_body_data(body.as_mut(), idle_timeout).await? { + let Some(new_len) = body_vec.len().checked_add(data.len()) else { + return Err(response_body_exceeds_limit_error()); + }; + if limit.is_some_and(|limit| new_len > limit) { + return Err(response_body_exceeds_limit_error()); + } + body_vec.extend_from_slice(&data); + } + Ok(body_vec) +} + pub async fn collect_response_body(body: B, limit: usize) -> Result, std::io::Error> where B: Body, - B::Error: Into>, + B::Error: Into>, { - let body = Limited::new(body, limit).collect().await.map_err(|err| { - if err.is::() { - std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit") - } else { - std::io::Error::other(err) - } - })?; - Ok(body.to_bytes().to_vec()) + collect_response_body_inner(body, Some(limit), None).await } const C_UNKNOWN: i32 = -1; @@ -196,6 +278,62 @@ pub struct TransitionClient { pub trailing_header_support: bool, pub max_retries: i64, pub tier_type: String, + pub timeouts: TransitionClientTimeouts, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TransitionClientTimeouts { + pub connect_timeout: StdDuration, + pub request_timeout: StdDuration, + pub response_body_idle_timeout: StdDuration, +} + +impl TransitionClientTimeouts { + pub const fn new( + connect_timeout: StdDuration, + request_timeout: StdDuration, + response_body_idle_timeout: StdDuration, + ) -> Self { + Self { + connect_timeout, + request_timeout, + response_body_idle_timeout, + } + } + + fn validate(self) -> Result { + if self.connect_timeout.is_zero() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "remote tier connect timeout must be greater than zero", + )); + } + if self.request_timeout.is_zero() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "remote tier request timeout must be greater than zero", + )); + } + if self.response_body_idle_timeout.is_zero() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "remote tier response body idle timeout must be greater than zero", + )); + } + Ok(self) + } +} + +impl Default for TransitionClientTimeouts { + fn default() -> Self { + Self { + connect_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS), + request_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS), + response_body_idle_timeout: StdDuration::from_secs( + rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS, + ), + } + } } #[derive(Debug, Default)] @@ -288,12 +426,28 @@ async fn build_tls_config() -> Result { impl TransitionClient { pub async fn new(endpoint: &str, opts: Options, tier_type: &str) -> Result { - let client = Self::private_new(endpoint, opts, tier_type).await?; - - Ok(client) + Self::private_new(endpoint, opts, tier_type, TransitionClientTimeouts::default()).await } - async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result { + /// Builds a transition client with explicit transport timeout budgets. + /// + /// [`Self::new`] keeps the historical constructor surface and uses the + /// production defaults from [`TransitionClientTimeouts::default`]. + pub async fn new_with_timeouts( + endpoint: &str, + opts: Options, + tier_type: &str, + timeouts: TransitionClientTimeouts, + ) -> Result { + Self::private_new(endpoint, opts, tier_type, timeouts).await + } + + async fn private_new( + endpoint: &str, + opts: Options, + tier_type: &str, + timeouts: TransitionClientTimeouts, + ) -> Result { if rustls::crypto::CryptoProvider::get_default().is_none() { // No default provider is set yet; try to install aws-lc-rs. // `install_default` can only fail if another thread races us and installs a provider @@ -306,15 +460,19 @@ impl TransitionClient { } let endpoint_url = get_endpoint_url(endpoint, opts.secure)?; + let timeouts = timeouts.validate()?; let tls = build_tls_config().await?; + let mut http = HttpConnector::new(); + http.enforce_http(false); + http.set_connect_timeout(Some(timeouts.connect_timeout)); let https = hyper_rustls::HttpsConnectorBuilder::new() .with_tls_config(tls) .https_or_http() .enable_http1() .enable_http2() - .build(); + .wrap_connector(http); let http_client = Client::builder(TokioExecutor::new()).build(https); let mut client = TransitionClient { @@ -337,6 +495,7 @@ impl TransitionClient { trailing_header_support: opts.trailing_headers, max_retries: opts.max_retries, tier_type: tier_type.to_string(), + timeouts, }; { @@ -501,29 +660,43 @@ impl TransitionClient { } pub async fn doit(&self, req: Request) -> Result, std::io::Error> { - let req_method; - let req_uri; - let resp; let http_client = self.http_client.clone(); - { - req_method = req.method().clone(); - req_uri = req.uri().clone(); - - debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string()); - resp = http_client.request(req); - } - let resp = resp.await; - debug!("http_client url: {} {}", req_method, req_uri); - if let Err(err) = resp { - error!("http_client call error: {:?}", err); - return Err(std::io::Error::other(err)); - } - + let req_method = req.method().clone(); + let resp = tokio::time::timeout(self.timeouts.request_timeout, http_client.request(req)).await; let resp = match resp { - Ok(r) => r, - Err(_) => return Err(std::io::Error::other("Unexpected error in response")), + Ok(Ok(resp)) => resp, + Ok(Err(err)) => { + let err = transition_transport_error(err); + error!( + event = EVENT_TIER_REMOTE_TRANSPORT, + component = LOG_COMPONENT_S3_CLIENT, + subsystem = LOG_SUBSYSTEM_TIER, + method = %req_method, + error_kind = ?err.kind(), + "remote tier request failed" + ); + return Err(err); + } + Err(_) => { + warn!( + event = EVENT_TIER_REMOTE_TRANSPORT, + component = LOG_COMPONENT_S3_CLIENT, + subsystem = LOG_SUBSYSTEM_TIER, + method = %req_method, + timeout_ms = self.timeouts.request_timeout.as_millis(), + "remote tier request timed out before response headers" + ); + return Err(remote_tier_timeout_error("remote tier request timed out before response headers")); + } }; - debug!(status = %resp.status(), "remote tier response received"); + trace!( + event = EVENT_TIER_REMOTE_TRANSPORT, + component = LOG_COMPONENT_S3_CLIENT, + subsystem = LOG_SUBSYSTEM_TIER, + method = %req_method, + status = %resp.status(), + "remote tier response received" + ); //let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); //debug!("http_resp_body: {}", String::from_utf8(b).unwrap()); @@ -537,7 +710,15 @@ impl TransitionClient { .and_then(|value| value.to_str().ok()) .unwrap_or_default() .to_string(); - warn!(status = %status, request_id, "remote tier request rejected"); + warn!( + event = EVENT_TIER_REMOTE_TRANSPORT, + component = LOG_COMPONENT_S3_CLIENT, + subsystem = LOG_SUBSYSTEM_TIER, + method = %req_method, + status = %status, + request_id, + "remote tier request rejected" + ); } Ok(resp) } @@ -581,7 +762,9 @@ impl TransitionClient { let resp_status = resp.status(); let h = resp.headers().clone(); - let body_vec = collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE).await?; + let body_vec = self + .collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE) + .await?; let parsed_error = http_resp_to_error_response(resp_status, &h, body_vec, &metadata.bucket_name, &metadata.object_name); let routing_region = parsed_error.region; @@ -635,6 +818,22 @@ impl TransitionClient { Err(std::io::Error::other("remote tier request did not produce a response")) } + pub async fn collect_response_body(&self, body: B, limit: usize) -> Result, std::io::Error> + where + B: Body, + B::Error: Into>, + { + collect_response_body_inner(body, Some(limit), Some(self.timeouts.response_body_idle_timeout)).await + } + + pub async fn collect_response_body_unbounded(&self, body: B) -> Result, std::io::Error> + where + B: Body, + B::Error: Into>, + { + collect_response_body_inner(body, None, Some(self.timeouts.response_body_idle_timeout)).await + } + async fn new_request( &self, method: &http::Method, @@ -1504,12 +1703,17 @@ pub struct CreateBucketConfiguration { mod tests { use super::{ MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body, - signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard, + collect_response_body_inner, signer_error_to_io_error, to_object_info_for_provider, validate_header_values, + with_rustls_init_guard, }; use crate::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion}; - use http::{HeaderMap, HeaderValue}; - use http_body_util::Full; + use futures::stream; + use http::{HeaderMap, HeaderValue, Request}; + use http_body::Frame; + use http_body_util::{Full, StreamBody}; use hyper::body::Bytes; + use std::time::Duration as StdDuration; + use tokio::net::TcpListener; use uuid::Uuid; #[tokio::test] @@ -1540,6 +1744,77 @@ mod tests { assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); } + #[tokio::test] + async fn empty_data_frames_do_not_reset_the_body_idle_timeout() { + let frames = stream::unfold((), |_| async { + tokio::time::sleep(StdDuration::from_millis(10)).await; + Some((Ok::<_, std::io::Error>(Frame::data(Bytes::new())), ())) + }); + let body = StreamBody::new(Box::pin(frames)); + + let err = tokio::time::timeout( + StdDuration::from_millis(200), + collect_response_body_inner(body, Some(1), Some(StdDuration::from_millis(50))), + ) + .await + .expect("the collector should enforce its own body idle timeout") + .expect_err("empty frames must not count as body progress"); + + assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); + } + + #[tokio::test] + async fn public_body_collector_accepts_non_unpin_bodies() { + let body = StreamBody::new(stream::once(async { Ok::<_, std::io::Error>(Frame::data(Bytes::from_static(b"ok"))) })); + + let collected = collect_response_body(body, 2) + .await + .expect("the public collector should pin non-Unpin bodies internally"); + + assert_eq!(collected, b"ok"); + } + + #[tokio::test] + async fn https_endpoints_reach_the_transport_connector() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let accepted = tokio::spawn(async move { + let (stream, _) = tokio::time::timeout(StdDuration::from_secs(1), listener.accept()) + .await + .expect("HTTPS connector should reach the TCP listener") + .expect("fixture should accept the HTTPS connection"); + drop(stream); + }); + let client = super::TransitionClient::new_with_timeouts( + &endpoint, + super::Options { + secure: true, + ..Default::default() + }, + "", + super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::from_secs(1)), + ) + .await + .expect("fixture client should build"); + let request = Request::builder() + .uri(format!("https://{endpoint}/")) + .body(s3s::Body::empty()) + .expect("fixture request should build"); + + client + .doit(request) + .await + .expect_err("the fixture closes before completing the TLS handshake"); + accepted.await.expect("fixture should join"); + } + #[test] fn rustls_guard_converts_panics_to_io_errors() { let err = with_rustls_init_guard(|| -> Result<(), std::io::Error> { panic!("missing provider") }) @@ -1573,6 +1848,18 @@ mod tests { assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set"); } + #[test] + fn transition_timeouts_reject_zero_budgets() { + for timeouts in [ + super::TransitionClientTimeouts::new(StdDuration::ZERO, StdDuration::from_secs(1), StdDuration::from_secs(1)), + super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::ZERO, StdDuration::from_secs(1)), + super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::ZERO), + ] { + let err = timeouts.validate().expect_err("zero timeout budgets must fail closed"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + } + } + #[test] fn validate_header_values_returns_header_name_for_non_utf8_values() { let mut headers = HeaderMap::new(); diff --git a/docs/operations/replication-outbound-transport.md b/docs/operations/replication-outbound-transport.md index 8ff7f285a..f3149e21c 100644 --- a/docs/operations/replication-outbound-transport.md +++ b/docs/operations/replication-outbound-transport.md @@ -29,6 +29,18 @@ Both knobs are read by the RustFS process that owns the replication target, at client build time; restart the server after changing them. +### Remote tier transport timeouts + +Remote tier S3-compatible clients use separate transport budgets. These settings do not change bucket or site replication clients. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS` | `10` | Maximum time to establish the remote tier TCP connection. | +| `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS` | `86400` | Maximum time for a remote tier request to reach response headers. The long default preserves large transition-upload headroom. | +| `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS` | `60` | Maximum time without a non-empty response-body chunk. Empty HTTP/2 frames do not count as progress. | + +All three values must be positive integers. Zero fails tier client initialization instead of silently disabling the boundary. An invalid integer is logged and falls back to the default; very large values are accepted and provide a correspondingly long effective budget. The values are read when the tier client is built; recreate or reload the tier configuration after changing them. + ## Before changing any of this Follow the SOP in `docs/postmortems/2026-09-03-replication-checksum-default-regression.md`: inventory the target-side rules the current default satisfies, run the outbound target matrix, and document any new knob here in the same PR. From af2e9df82105e21a2ddebe8fce23dd863695a99f Mon Sep 17 00:00:00 2001 From: RustFS Date: Sat, 5 Sep 2026 20:42:22 +0800 Subject: [PATCH 33/40] fix(lifecycle): correct expiration and transition evaluation (#7169) --- crates/lifecycle/src/core.rs | 504 +++++++++++++++++++++++--- crates/lifecycle/src/evaluator.rs | 100 ++++- docs/operations/tier-ilm-debugging.md | 10 + 3 files changed, 563 insertions(+), 51 deletions(-) diff --git a/crates/lifecycle/src/core.rs b/crates/lifecycle/src/core.rs index fc959b92a..bfa2b93c1 100644 --- a/crates/lifecycle/src/core.rs +++ b/crates/lifecycle/src/core.rs @@ -43,6 +43,10 @@ const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules"; const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer"; +const ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT: &str = "Expiration cannot specify both Days and Date"; +const ERR_LIFECYCLE_MULTIPLE_TRANSITIONS: &str = "Only one Transition action per lifecycle rule is supported"; +const ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS: &str = + "Only one NoncurrentVersionTransition action per lifecycle rule is supported"; const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str = "'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer"; const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str = @@ -361,6 +365,12 @@ impl Lifecycle for BucketLifecycleConfiguration { { return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS)); } + if expiration.days.is_some() && expiration.date.is_some() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT, + )); + } if let Some(expiration_date) = &expiration.date { let date = OffsetDateTime::from(expiration_date.clone()); if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 { @@ -394,11 +404,20 @@ impl Lifecycle for BucketLifecycleConfiguration { } } if let Some(transitions) = &r.transitions { + if transitions.len() > 1 { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, ERR_LIFECYCLE_MULTIPLE_TRANSITIONS)); + } for transition in transitions { TransitionOps::validate(transition)?; } } if let Some(noncurrent_transitions) = &r.noncurrent_version_transitions { + if noncurrent_transitions.len() > 1 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS, + )); + } for transition in noncurrent_transitions { NoncurrentVersionTransitionOps::validate(transition)?; } @@ -473,6 +492,8 @@ impl Lifecycle for BucketLifecycleConfiguration { } async fn eval(&self, obj: &ObjectOpts) -> Event { + // A single-object lookup cannot prove how many newer historical versions + // survive. Count-dependent actions wait for the complete-group evaluator. self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await } @@ -536,23 +557,8 @@ impl Lifecycle for BucketLifecycleConfiguration { return Event::default(); }; - if let Some(restore_expires) = obj.restore_expires - && restore_expires.unix_timestamp() != 0 - && now.unix_timestamp() > restore_expires.unix_timestamp() - { - let mut action = IlmAction::DeleteRestoredAction; - if !obj.is_latest { - action = IlmAction::DeleteRestoredVersionAction; - } - - events.push(Event { - action, - due: Some(now), - rule_id: "".into(), - noncurrent_days: 0, - newer_noncurrent_versions: 0, - storage_class: "".into(), - }); + if let Some(event) = obj.restored_copy_expiry(now) { + events.push(event); } if let Some(ref lc_rules) = self.filter_rules(obj).await { @@ -611,17 +617,12 @@ impl Lifecycle for BucketLifecycleConfiguration { continue; } - if !obj.is_latest - && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration - && let Some(retain_newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions - && newer_noncurrent_versions < usize::try_from(retain_newer_noncurrent_versions).unwrap_or(usize::MAX) - { - continue; - } - if !obj.is_latest && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration && let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days + && noncurrent_version_expiration + .newer_noncurrent_versions + .is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain)) { if let Some(successor_mod_time) = obj.successor_mod_time { let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days); @@ -651,7 +652,11 @@ impl Lifecycle for BucketLifecycleConfiguration { && let Some(noncurrent_version_transition) = rule .noncurrent_version_transitions .as_ref() + .filter(|transitions| transitions.len() == 1) .and_then(|transitions| transitions.first()) + && noncurrent_version_transition + .newer_noncurrent_versions + .is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain)) && let Some(storage_class) = noncurrent_version_transition.storage_class.as_ref() && !storage_class.as_str().is_empty() && !obj.delete_marker @@ -735,7 +740,11 @@ impl Lifecycle for BucketLifecycleConfiguration { } if obj.transition_status != TRANSITION_COMPLETE - && let Some(transition) = rule.transitions.as_ref().and_then(|transitions| transitions.first()) + && let Some(transition) = rule + .transitions + .as_ref() + .filter(|transitions| transitions.len() == 1) + .and_then(|transitions| transitions.first()) && let Some(storage_class) = transition.storage_class.as_ref() && !storage_class.as_str().is_empty() { @@ -758,18 +767,15 @@ impl Lifecycle for BucketLifecycleConfiguration { } if !events.is_empty() { - // Select the winning event using a strict total order (MinIO semantics): - // the earliest `due` wins, and ties break toward delete-type actions. A - // missing `due` is treated as UNIX_EPOCH. This replaces a hand-written - // `sort_by` comparator that was not a strict weak ordering (it could return - // `Ordering::Less` for both `(a, b)` and `(b, a)`), which panics on the - // repository toolchain and did not deterministically pick the earliest event. + // Eligible expiration takes precedence over transition, even when a + // failed transition has an earlier deadline. Within each action class, + // prefer the earliest deadline using a deterministic total order. let event = events .iter() .min_by_key(|event| { ( - event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(), ilm_action_priority_rank(&event.action), + event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(), ) }) .cloned() @@ -1042,6 +1048,27 @@ impl ObjectOpts { pub fn expired_object_deletemarker(&self) -> bool { self.delete_marker && self.is_latest && self.num_versions == 1 } + + pub(crate) fn restored_copy_expiry(&self, now: OffsetDateTime) -> Option { + let restore_expires = self.restore_expires?; + // Restore metadata alone does not prove that a durable remote copy exists. + if self.transition_status != TRANSITION_COMPLETE + || restore_expires.unix_timestamp() == 0 + || now.unix_timestamp() <= restore_expires.unix_timestamp() + { + return None; + } + let action = if self.is_latest { + IlmAction::DeleteRestoredAction + } else { + IlmAction::DeleteRestoredVersionAction + }; + expiration_action_has_valid_target(action, self.version_id, self.is_latest, self.delete_marker).then(|| Event { + action, + due: Some(now), + ..Default::default() + }) + } } /// Returns whether an expiry action has enough identity to target the object @@ -1064,11 +1091,8 @@ pub fn expiration_action_has_valid_target( } } -/// Total-order rank for lifecycle actions used to break `due` ties. -/// -/// Delete-type actions rank before every other action so that, when two events -/// share the same `due`, a delete wins (MinIO semantics). The concrete numeric -/// values only matter relative to each other. +/// Eligible logical expiration takes precedence over transition and restore-copy +/// cleanup. Deadlines break ties within an action class. fn ilm_action_priority_rank(action: &IlmAction) -> u8 { match action { IlmAction::DeleteAllVersionsAction @@ -4159,6 +4183,392 @@ mod tests { assert_eq!(event.action, IlmAction::NoneAction); } + mod adversarial_regressions { + use super::*; + use s3s::dto::NoncurrentVersionExpiration; + + fn run(test: impl std::future::Future) { + with_default_ilm_process_time(|| { + tokio::runtime::Builder::new_current_thread() + .build() + .expect("lifecycle regression runtime should build") + .block_on(test); + }); + } + + fn noncurrent_object() -> ObjectOpts { + ObjectOpts { + name: "logs/object".to_string(), + mod_time: Some(datetime!(2020-01-01 00:00:00 UTC)), + successor_mod_time: Some(datetime!(2020-01-02 00:00:00 UTC)), + version_id: Some(Uuid::from_u128(1)), + size: 1024 * 1024, + ..Default::default() + } + } + + #[test] + #[serial] + fn noncurrent_transition_retains_the_requested_newer_versions() { + run(async { + let mut rule = enabled_rule(None, None, Some("retain-two-hot-versions")); + rule.filter = Some(LifecycleRuleFilter::default()); + rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition { + noncurrent_days: Some(1), + newer_noncurrent_versions: Some(2), + storage_class: Some(TransitionStorageClass::from_static("WARM")), + }]); + let lc = Arc::new(BucketLifecycleConfiguration { + rules: vec![rule], + expiry_updated_at: None, + }); + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("valid noncurrent transition policy"); + let objects = (0..4) + .map(|index| ObjectOpts { + mod_time: Some(datetime!(2020-01-05 00:00:00 UTC) - Duration::days(index)), + successor_mod_time: (index > 0).then_some(datetime!(2020-01-06 00:00:00 UTC) - Duration::days(index)), + version_id: Some(Uuid::from_u128(u128::try_from(index + 1).expect("small version index"))), + is_latest: index == 0, + num_versions: 4, + ..noncurrent_object() + }) + .collect::>(); + let actions = crate::Evaluator::new(lc) + .eval(&objects) + .await + .expect("complete version chain should evaluate") + .into_iter() + .map(|event| event.action) + .collect::>(); + assert_eq!( + actions, + [ + IlmAction::NoneAction, + IlmAction::NoneAction, + IlmAction::NoneAction, + IlmAction::TransitionVersionAction + ], + "the two newest noncurrent versions must remain in their current storage class" + ); + }); + } + + #[test] + #[serial] + fn noncurrent_transition_checks_count_age_and_single_object_context() { + run(async { + let mut rule = enabled_rule(None, None, Some("retain-two")); + rule.filter = Some(LifecycleRuleFilter::default()); + rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition { + noncurrent_days: Some(3), + newer_noncurrent_versions: Some(2), + storage_class: Some(TransitionStorageClass::from_static("WARM")), + }]); + let mut lc = BucketLifecycleConfiguration { + rules: vec![rule], + expiry_updated_at: None, + }; + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("valid counted transition"); + let object = noncurrent_object(); + let now = datetime!(2020-01-10 00:00:00 UTC); + for (newer, expected) in [ + (0, IlmAction::NoneAction), + (1, IlmAction::NoneAction), + (2, IlmAction::TransitionVersionAction), + (3, IlmAction::TransitionVersionAction), + ] { + assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}"); + } + assert_eq!( + lc.eval_inner(&object, datetime!(2020-01-04 00:00:00 UTC), 2).await.action, + IlmAction::NoneAction, + "the retention count does not replace the age condition" + ); + assert_eq!( + lc.eval(&object).await.action, + IlmAction::NoneAction, + "a single-object lookup must not assume a complete version history" + ); + for retain in [None, Some(0), Some(-1), Some(i32::MAX)] { + lc.rules[0] + .noncurrent_version_transitions + .as_mut() + .expect("transition exists")[0] + .newer_noncurrent_versions = retain; + let expected = if matches!(retain, None | Some(0)) { + IlmAction::TransitionVersionAction + } else { + IlmAction::NoneAction + }; + assert_eq!(lc.eval_inner(&object, now, 2).await.action, expected, "retention: {retain:?}"); + } + }); + } + + #[test] + #[serial] + fn noncurrent_expiration_and_transition_have_independent_retention_counts() { + run(async { + let mut rule = enabled_rule(None, None, Some("independent-counts")); + rule.filter = Some(LifecycleRuleFilter::default()); + rule.noncurrent_version_expiration = Some(NoncurrentVersionExpiration { + noncurrent_days: Some(90), + newer_noncurrent_versions: Some(4), + }); + rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition { + noncurrent_days: Some(30), + newer_noncurrent_versions: Some(2), + storage_class: Some(TransitionStorageClass::from_static("WARM")), + }]); + let lc = BucketLifecycleConfiguration { + rules: vec![rule], + expiry_updated_at: None, + }; + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("valid independent retention limits"); + let object = noncurrent_object(); + let now = datetime!(2020-05-01 00:00:00 UTC); + for (newer, expected) in [ + (1, IlmAction::NoneAction), + (2, IlmAction::TransitionVersionAction), + (3, IlmAction::TransitionVersionAction), + (4, IlmAction::DeleteVersionAction), + ] { + assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}"); + } + }); + } + + #[test] + #[serial] + fn expiration_retention_does_not_skip_an_independent_transition() { + run(async { + let mut rule = enabled_rule(None, None, Some("transition-then-expire")); + rule.filter = Some(LifecycleRuleFilter::default()); + rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition { + noncurrent_days: Some(1), + newer_noncurrent_versions: None, + storage_class: Some(TransitionStorageClass::from_static("WARM")), + }]); + let mut lc = BucketLifecycleConfiguration { + rules: vec![rule], + expiry_updated_at: None, + }; + let object = noncurrent_object(); + let now = datetime!(2020-01-10 00:00:00 UTC); + let transition_only = lc.eval_inner(&object, now, 0).await; + assert_eq!(transition_only.action, IlmAction::TransitionVersionAction); + + lc.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration { + noncurrent_days: Some(90), + newer_noncurrent_versions: Some(2), + }); + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("valid combined policy"); + let combined = lc.eval_inner(&object, now, 0).await; + assert_eq!(combined.action, transition_only.action, "retention limits expiration, not transition"); + assert_eq!(combined.storage_class, transition_only.storage_class); + }); + } + + #[test] + #[serial] + fn current_transition_rejects_multiple_stages_in_any_order() { + run(async { + let mut rule = enabled_rule(None, None, Some("two-current-transitions")); + rule.transitions = Some(vec![ + Transition { + date: Some(datetime!(2020-03-01 00:00:00 UTC).into()), + days: None, + storage_class: Some(TransitionStorageClass::from_static("COLD")), + }, + Transition { + date: Some(datetime!(2020-01-03 00:00:00 UTC).into()), + days: None, + storage_class: Some(TransitionStorageClass::from_static("WARM")), + }, + ]); + let mut lc = BucketLifecycleConfiguration { + rules: vec![rule], + expiry_updated_at: None, + }; + let object = ObjectOpts { + is_latest: true, + ..noncurrent_object() + }; + let now = datetime!(2020-01-10 00:00:00 UTC); + for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] { + lc.rules[0].status = ExpirationStatus::from_static(status); + for _ in 0..2 { + let err = lc + .validate(&ObjectLockConfiguration::default()) + .await + .expect_err("multiple transition stages must be rejected"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_TRANSITIONS); + assert_eq!( + lc.eval_inner(&object, now, 0).await.action, + IlmAction::NoneAction, + "legacy multi-stage configurations must not silently execute their first stage" + ); + lc.rules[0] + .transitions + .as_mut() + .expect("transition array is present") + .reverse(); + } + } + lc.rules[0] + .transitions + .as_mut() + .expect("transition array is present") + .remove(0); + lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED); + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("one stage is supported"); + let event = lc.eval_inner(&object, now, 0).await; + assert_eq!(event.action, IlmAction::TransitionAction); + assert_eq!(event.storage_class, "WARM"); + }); + } + + #[test] + #[serial] + fn noncurrent_transition_rejects_multiple_stages_in_any_order() { + run(async { + let mut rule = enabled_rule(None, None, Some("two-noncurrent-transitions")); + rule.noncurrent_version_transitions = Some(vec![ + NoncurrentVersionTransition { + noncurrent_days: Some(30), + newer_noncurrent_versions: None, + storage_class: Some(TransitionStorageClass::from_static("COLD")), + }, + NoncurrentVersionTransition { + noncurrent_days: Some(1), + newer_noncurrent_versions: None, + storage_class: Some(TransitionStorageClass::from_static("WARM")), + }, + ]); + let mut lc = BucketLifecycleConfiguration { + rules: vec![rule], + expiry_updated_at: None, + }; + let object = noncurrent_object(); + let now = datetime!(2020-01-10 00:00:00 UTC); + for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] { + lc.rules[0].status = ExpirationStatus::from_static(status); + for _ in 0..2 { + let err = lc + .validate(&ObjectLockConfiguration::default()) + .await + .expect_err("multiple noncurrent transition stages must be rejected"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS); + assert_eq!( + lc.eval_inner(&object, now, 0).await.action, + IlmAction::NoneAction, + "legacy multi-stage configurations must not silently execute their first stage" + ); + lc.rules[0] + .noncurrent_version_transitions + .as_mut() + .expect("transition array is present") + .reverse(); + } + } + lc.rules[0] + .noncurrent_version_transitions + .as_mut() + .expect("transition array is present") + .remove(0); + lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED); + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("one stage is supported"); + let event = lc.eval_inner(&object, now, 0).await; + assert_eq!(event.action, IlmAction::TransitionVersionAction); + assert_eq!(event.storage_class, "WARM"); + }); + } + + #[test] + #[serial] + fn expiration_rejects_simultaneous_days_and_date() { + run(async { + let mut lc = BucketLifecycleConfiguration { + rules: vec![enabled_rule( + Some(LifecycleExpiration { + days: Some(1), + ..Default::default() + }), + None, + Some("ambiguous-expiry"), + )], + expiry_updated_at: None, + }; + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("a single Days expiration is valid"); + lc.rules[0].expiration.as_mut().expect("expiration is present").date = + Some(datetime!(2099-01-01 00:00:00 UTC).into()); + let err = lc + .validate(&ObjectLockConfiguration::default()) + .await + .expect_err("Days and Date are mutually exclusive; accepting both silently overrides Days"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!(err.to_string(), ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT); + }); + } + + #[test] + #[serial] + fn overdue_transition_does_not_starve_permanent_expiration() { + run(async { + let mut rule = enabled_rule( + Some(LifecycleExpiration { + days: Some(90), + ..Default::default() + }), + None, + Some("archive-then-delete"), + ); + rule.transitions = Some(vec![Transition { + days: Some(30), + date: None, + storage_class: Some(TransitionStorageClass::from_static("WARM")), + }]); + let lc = BucketLifecycleConfiguration { + rules: vec![rule], + expiry_updated_at: None, + }; + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("valid transition and expiration policy"); + let object = ObjectOpts { + is_latest: true, + version_id: None, + transition_status: TRANSITION_PENDING.to_string(), + ..noncurrent_object() + }; + let before_expiration = lc.eval_inner(&object, datetime!(2020-02-15 00:00:00 UTC), 0).await; + assert_eq!(before_expiration.action, IlmAction::TransitionAction); + let overdue = lc.eval_inner(&object, datetime!(2020-05-01 00:00:00 UTC), 0).await; + assert_eq!( + overdue.action, + IlmAction::DeleteAction, + "an unavailable tier must not prevent permanent expiration indefinitely" + ); + }); + } + } + /// Property-based tests for the rule evaluator (backlog#1148 ilm-14, /// follow-up to backlog#1030 / rustfs#4455). /// @@ -4169,7 +4579,7 @@ mod tests { /// /// * `eval_inner` never panics and is deterministic for a fixed input; /// * the winning event matches an independently recomputed candidate set: - /// earliest `due` wins, ties break toward delete-class actions (the + /// eligible expiration wins over transition, then earliest `due` wins (the /// `min_by_key` selection that replaced the rustfs#4455 comparator); /// * `expected_expiry_time` is monotonically non-decreasing in `days` and /// always lands on the processing boundary, both at production defaults @@ -4458,8 +4868,8 @@ mod tests { /// consider for a live current version under `selection`-shaped rules /// (expiration and first-transition only, no filters): expiration /// fires when `now >= due`, transition when `now > due` and the object - /// has not already transitioned. Selection semantics under test: - /// earliest due wins, ties prefer delete-class. + /// has not already transitioned. Eligible expiration wins over transition; + /// the earliest deadline wins within the selected action class. fn oracle_candidates(lc: &BucketLifecycleConfiguration, obj: &ObjectOpts, now: OffsetDateTime) -> Vec { let mod_time = obj.mod_time.expect("selection strategy always sets mod_time"); let mut candidates = Vec::new(); @@ -4548,8 +4958,8 @@ mod tests { /// Differential test of winner selection (the rustfs#4455 fix): /// for a live current version under randomized expiration and /// transition rules, `eval_inner`'s winner must carry the - /// minimum `(due, rank)` of the independently recomputed - /// candidate set — earliest due wins, ties prefer delete-class — + /// earliest expiration from the independently recomputed candidate + /// set, or the earliest transition when no expiration is eligible, /// and must be `NoneAction` exactly when that set is empty. #[test] #[serial] @@ -4578,7 +4988,13 @@ mod tests { // Oracle and evaluator must observe the same (pinned) time env. let (event, expected) = with_production_time_env(|| { - let expected = oracle_candidates(&lc, &obj, now).into_iter().min(); + let candidates = oracle_candidates(&lc, &obj, now); + let expected = candidates + .iter() + .filter(|(_, rank)| *rank == 0) + .min() + .copied() + .or_else(|| candidates.into_iter().min()); let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() diff --git a/crates/lifecycle/src/evaluator.rs b/crates/lifecycle/src/evaluator.rs index 80bd61de6..2da4ae76c 100644 --- a/crates/lifecycle/src/evaluator.rs +++ b/crates/lifecycle/src/evaluator.rs @@ -116,13 +116,10 @@ impl Evaluator { break 'top_loop; } } - IlmAction::DeleteAction - | IlmAction::DeleteRestoredAction - | IlmAction::DeleteVersionAction - | IlmAction::DeleteRestoredVersionAction - if self.is_object_locked(obj) => - { - event = Event::default(); + // Restore expiry removes only the temporary local copy; the + // retained logical version and its remote data remain intact. + IlmAction::DeleteAction | IlmAction::DeleteVersionAction if self.is_object_locked(obj) => { + event = obj.restored_copy_expiry(now).unwrap_or_default(); } _ => {} } @@ -206,6 +203,95 @@ mod tests { use super::*; use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType}; + + #[tokio::test] + async fn adversarial_restore_expiry_survives_legal_hold() { + let mut policy = (*latest_expiration_lifecycle()).clone(); + policy.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED); + let policy = Arc::new(policy); + policy + .validate(&lock_enabled_without_default_retention()) + .await + .expect("valid disabled lifecycle rule"); + let mut objects = [true, false].map(|is_latest| ObjectOpts { + is_latest, + num_versions: 2, + mod_time: Some( + OffsetDateTime::from_unix_timestamp(if is_latest { 1_200_000 } else { 1_000_000 }) + .expect("fixed version timestamp"), + ), + successor_mod_time: (!is_latest) + .then(|| OffsetDateTime::from_unix_timestamp(1_200_000).expect("fixed successor timestamp")), + transition_status: crate::TRANSITION_COMPLETE.to_string(), + restore_expires: Some(OffsetDateTime::from_unix_timestamp(2_000_000).expect("fixed expired restore timestamp")), + ..current_object_opts(ReplicationStatusType::Completed) + }); + let evaluator = Evaluator::new(policy).with_lock_retention(Some(lock_enabled_without_default_retention())); + let expected = [IlmAction::DeleteRestoredAction, IlmAction::DeleteRestoredVersionAction]; + let unlocked = evaluator + .eval(&objects) + .await + .expect("unlocked restored versions should evaluate"); + assert_eq!(unlocked.iter().map(|event| event.action).collect::>(), expected); + + for object in &mut objects { + object + .user_defined + .insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string()); + } + let locked = evaluator + .eval(&objects) + .await + .expect("locked restored versions should evaluate"); + assert_eq!( + locked.iter().map(|event| event.action).collect::>(), + expected, + "expiring a restored local copy preserves the retained logical version and remote object" + ); + + let mut expiring_policy = (*latest_expiration_lifecycle()).clone(); + expiring_policy.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: None, + }); + let expiring_evaluator = + Evaluator::new(Arc::new(expiring_policy)).with_lock_retention(Some(lock_enabled_without_default_retention())); + let locked = expiring_evaluator + .eval(&objects) + .await + .expect("locked expired versions should evaluate"); + assert_eq!( + locked.iter().map(|event| event.action).collect::>(), + expected, + "blocked logical expiration must still allow an eligible restore-copy cleanup" + ); + + for status in [ReplicationStatusType::Pending, ReplicationStatusType::Failed] { + for object in &mut objects { + object.replication_status = status.clone(); + } + for evaluator in [&evaluator, &expiring_evaluator] { + let events = evaluator.eval(&objects).await.expect("pending replication should evaluate"); + assert!(events.iter().all(|event| event.action == IlmAction::NoneAction)); + } + } + for object in &mut objects { + object.replication_status = ReplicationStatusType::Completed; + } + for transition_status in ["", crate::TRANSITION_PENDING, "unknown"] { + for object in &mut objects { + object.transition_status = transition_status.to_string(); + } + for evaluator in [&evaluator, &expiring_evaluator] { + let events = evaluator.eval(&objects).await.expect("incomplete transition should evaluate"); + assert!( + events.iter().all(|event| event.action == IlmAction::NoneAction), + "restore metadata cannot authorize cleanup without a completed transition" + ); + } + } + } + fn expired_marker_lifecycle() -> Arc { Arc::new(BucketLifecycleConfiguration { expiry_updated_at: None, diff --git a/docs/operations/tier-ilm-debugging.md b/docs/operations/tier-ilm-debugging.md index 3b4a5d3a6..d5da40af3 100644 --- a/docs/operations/tier-ilm-debugging.md +++ b/docs/operations/tier-ilm-debugging.md @@ -22,6 +22,16 @@ | `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` | | Dual-key internal metadata helpers (`insert_bytes` / `get_bytes`) | `crates/utils/src/http/metadata_compat.rs` | +## Lifecycle rule limits and evaluation + +Each lifecycle rule supports at most one `Transition` and one `NoncurrentVersionTransition`. A version can make one initial transition; chaining additional tiers after it reaches `complete` is not supported. Splitting stages across overlapping rules does not enable a transition chain. `PutBucketLifecycleConfiguration` rejects multiple entries in either transition array with `InvalidArgument`, including in disabled rules. Existing stored multi-entry arrays are not executed; replace each with a single intended destination. Independent expiration actions in the rule remain eligible. + +`Expiration.Days` and `Expiration.Date` are mutually exclusive. A request containing both is rejected instead of silently selecting the date. When expiration and transition are both eligible, expiration takes precedence; a failed earlier transition does not keep an expired object indefinitely. Deadlines select the earliest action within the same action class. + +Noncurrent expiration and transition have independent `NewerNoncurrentVersions` limits. A transition with a positive limit waits for a complete version-group evaluation to establish that enough newer noncurrent versions remain. Single-object evaluation, including the current manual transition and immediate-enqueue paths, conservatively defers these counted transitions to the lifecycle scanner. An unmet expiration retention limit does not suppress a separately eligible transition. + +An expired restored local copy can be cleaned up under Object Lock because the retained logical version and remote data remain intact. Cleanup requires a completed transition and still waits for pending or failed replication. The storage layer revalidates the source identity and restore metadata before removing the local copy; restore headers alone do not authorize cleanup. + ## Free-version recovery controls The dedicated free-version recovery loop is enabled by default and is independent of the data scanner and heal switches. Setting `RUSTFS_SCANNER_ENABLED=false` does not stop this repair loop. Set `RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED=false` before process startup to disable only the dedicated persisted-marker walk. That setting does not disable lifecycle workers or prevent another scanner path from discovering a free version, and it can leave remote cleanup markers pending for longer, so use it as a break-glass pressure control rather than a cleanup mechanism. From 33fd056000596ad92b16d865e325553d70db7e92 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 20:49:40 +0800 Subject: [PATCH 34/40] fix(ecstore): release heal disk snapshot before nested reads (#7189) * fix(ecstore): release heal disk snapshot before nested reads * fix(ecstore): remove duplicate local rename implementation Keep the canonical commit module after concurrent storage changes merged. The control-write and rollback changes are already present there. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(app): simplify absent SSE configuration matching * fix(tests): satisfy new clippy lints --------- Co-authored-by: houseme Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/set_disk/ops/heal.rs | 366 +++++++++++++++++++++++- 1 file changed, 363 insertions(+), 3 deletions(-) diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 3d4624b4e..c7285b9b8 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -2490,9 +2490,9 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { return Ok((result, err.map(|e| e.into()))); } - let disks = self.disks.read().await; - - let disks = disks.clone(); + // The inner heal and missing-object report read the registry again; + // release this snapshot guard before a topology writer can queue between reads. + let disks = self.get_disks_internal().await; let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false) .await .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; @@ -3419,6 +3419,366 @@ mod heal_result_report_tests { assert_eq!(unformatted, DiskError::UnformattedDisk); } + #[derive(Clone, Copy)] + enum InventoryWriterHealCase { + Existing, + Missing, + MissingVersion, + } + + async fn assert_heal_object_inventory_writer(case: InventoryWriterHealCase) { + use crate::set_disk::core::io_primitives::disk_call_counters; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "heal-inventory-writer-bucket"; + let object = match case { + InventoryWriterHealCase::Existing => "heal-inventory-writer-existing", + InventoryWriterHealCase::Missing => "heal-inventory-writer-missing", + InventoryWriterHealCase::MissingVersion => "heal-inventory-writer-missing-version", + }; + set.make_bucket( + bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("heal fixture bucket should be created"); + let body = vec![0x67; 64 * 1024]; + let stored_version = Uuid::new_v4(); + let stored_version_string = stored_version.to_string(); + let published = if matches!(case, InventoryWriterHealCase::Missing) { + None + } else { + let mut reader = PutObjReader::from_vec(body.clone()); + let info = set + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + no_lock: true, + versioned: true, + version_id: Some(stored_version_string.clone()), + ..Default::default() + }, + ) + .await + .expect("full-fanout PUT should seed the heal fixture"); + for disk in &disks { + let metadata = disk + .read_version("", bucket, object, &stored_version_string, &ReadOptions::default()) + .await + .expect("the seeded version must be present on every disk"); + assert_eq!(metadata.version_id, Some(stored_version)); + assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64")); + } + Some(info) + }; + let requested_version = match case { + InventoryWriterHealCase::Existing => stored_version_string.clone(), + InventoryWriterHealCase::Missing => String::new(), + InventoryWriterHealCase::MissingVersion => Uuid::new_v4().to_string(), + }; + let opts = HealOpts { + no_lock: true, + ..Default::default() + }; + let calls = disk_call_counters::observe(object); + let read_gate = set.disks.read().await; + // UFCS selects the trait's outer precheck, not the same-named inherent heal. + let heal = ::heal_object( + set.as_ref(), + bucket, + object, + &requested_version, + &opts, + ); + tokio::pin!(heal); + assert!(matches!( + futures::poll!(tokio::task::unconstrained(heal.as_mut())), + std::task::Poll::Pending + )); + // These tests use the current-thread runtime: full-wait metadata tasks + // have been spawned, but cannot run during the single unconstrained poll. + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0); + let writer = set.disks.write(); + tokio::pin!(writer); + assert!(matches!( + futures::poll!(tokio::task::unconstrained(writer.as_mut())), + std::task::Poll::Pending + )); + assert!(set.disks.try_read().is_err(), "the writer must already block new inventory readers"); + tokio::time::timeout(Duration::from_secs(5), async { + while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 { + tokio::task::yield_now().await; + } + }) + .await + .expect("the suspended trait heal must have started the real metadata fanout"); + for disk_index in 0..4 { + assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1); + } + drop(read_gate); + + let (_, outcome) = + tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(async { drop(writer.await) }, heal) }) + .await + .expect("trait heal must not deadlock its nested inventory read with the queued writer"); + let (result, error) = outcome.expect("heal should report the object's outcome"); + match case { + InventoryWriterHealCase::Existing => assert!(error.is_none(), "existing object heal failed: {error:?}"), + InventoryWriterHealCase::Missing => assert!(matches!(error, Some(Error::FileNotFound))), + InventoryWriterHealCase::MissingVersion => assert!(matches!(error, Some(Error::FileVersionNotFound))), + } + assert_eq!(result.bucket, bucket); + assert_eq!(result.object, object); + assert_eq!(result.version_id, requested_version); + assert_eq!(result.disk_count, 4); + assert_eq!(result.before.drives.len(), 4); + assert_eq!(result.after.drives.len(), 4); + for disk_index in 0..4 { + let endpoint = set.set_endpoints[disk_index].to_string(); + assert_eq!(result.before.drives[disk_index].endpoint, endpoint); + assert_eq!(result.after.drives[disk_index].endpoint, endpoint); + } + if let Some(published) = published { + tokio::time::timeout(Duration::from_secs(10), async { + let mut reader = set + .get_object_reader( + bucket, + object, + None, + Default::default(), + &ObjectOptions { + versioned: true, + version_id: Some(stored_version_string), + ..Default::default() + }, + ) + .await + .expect("the stored version must remain readable after heal"); + assert_eq!(reader.object_info.etag, published.etag); + assert_eq!(reader.object_info.version_id, Some(stored_version)); + let mut observed_body = Vec::new(); + reader + .stream + .read_to_end(&mut observed_body) + .await + .expect("stored body should stream"); + assert_eq!(observed_body, body); + }) + .await + .expect("GET must finish after the inventory writer and heal"); + } + } + + #[tokio::test] + async fn heal_object_inventory_writer_existing() { + assert_heal_object_inventory_writer(InventoryWriterHealCase::Existing).await; + } + + #[tokio::test] + async fn heal_object_inventory_writer_missing() { + assert_heal_object_inventory_writer(InventoryWriterHealCase::Missing).await; + } + + #[tokio::test] + async fn heal_object_inventory_writer_missing_version() { + assert_heal_object_inventory_writer(InventoryWriterHealCase::MissingVersion).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn heal_object_with_queued_disk_renewal() { + use crate::layout::endpoints::SetupType; + use crate::runtime::instance::InstanceContext; + use crate::set_disk::core::io_primitives::disk_call_counters; + use std::collections::HashMap; + use std::future::Future; + use std::task::Poll; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + // renew_disk still registers local disks on the ambient context. Match + // the default serial group used by its other setup/registry fixtures, + // and restore only this temporary endpoint, including on a failed join. + struct RenewDiskTestState { + ctx: Arc, + was_dist_erasure: bool, + map: Arc>>>, + endpoint: String, + previous_disk: Option>, + } + + impl Drop for RenewDiskTestState { + fn drop(&mut self) { + let ctx = self.ctx.clone(); + let was_dist_erasure = self.was_dist_erasure; + let map = self.map.clone(); + let endpoint = self.endpoint.clone(); + let previous_disk = self.previous_disk.take(); + let handle = tokio::runtime::Handle::current(); + std::thread::spawn(move || { + handle.block_on(async move { + let mut map = map.write().await; + match previous_disk { + Some(disk) => { + map.insert(endpoint, disk); + } + None => { + map.remove(&endpoint); + } + } + drop(map); + if was_dist_erasure { + ctx.update_erasure_type(SetupType::DistErasure).await; + } + }); + }) + .join() + .expect("renew fixture state restoration should finish"); + } + } + + let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let endpoint = set.set_endpoints[0].clone(); + let ctx = crate::runtime::global::current_ctx(); + let map = ctx.local_disk_map(); + let restore = RenewDiskTestState { + ctx: ctx.clone(), + was_dist_erasure: ctx.is_dist_erasure().await, + map: map.clone(), + endpoint: endpoint.to_string(), + previous_disk: map.read().await.get(&endpoint.to_string()).cloned(), + }; + // Only distributed erasure needs an override to avoid the ambient slot array. + if restore.was_dist_erasure { + ctx.update_erasure_type(SetupType::Erasure).await; + } + + let bucket = "heal-disk-renewal-bucket"; + let object = "heal-disk-renewal-object"; + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("renew fixture bucket should be created"); + let body = vec![0x73; 64 * 1024]; + let mut reader = PutObjReader::from_vec(body.clone()); + let published = set + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("full-fanout PUT should seed the renewal fixture"); + for disk in &disks { + let metadata = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("the seeded object must be present on every disk"); + assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64")); + } + + let opts = HealOpts { + no_lock: true, + ..Default::default() + }; + let calls = disk_call_counters::observe(object); + let read_gate = set.disks.read().await; + let heal = ::heal_object( + set.as_ref(), + bucket, + object, + "", + &opts, + ); + tokio::pin!(heal); + assert!(matches!(futures::poll!(tokio::task::unconstrained(heal.as_mut())), Poll::Pending)); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0); + + let renew = set.renew_disk(&endpoint); + tokio::pin!(renew); + tokio::time::timeout( + Duration::from_secs(5), + futures::future::poll_fn(|cx| { + assert!( + std::pin::pin!(tokio::task::unconstrained(renew.as_mut())) + .poll(cx) + .is_pending(), + "renewal must reach its inventory write before returning" + ); + if set.disks.try_read().is_err() { + Poll::Ready(()) + } else { + Poll::Pending + } + }), + ) + .await + .expect("real renewal must queue its topology writer behind the read gate"); + let registered = map + .read() + .await + .get(&endpoint.to_string()) + .cloned() + .flatten() + .expect("renewal must register the connected disk before its inventory write"); + assert!(!Arc::ptr_eq(®istered, &disks[0]), "renewal must construct a new disk handle"); + tokio::time::timeout(Duration::from_secs(5), async { + while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 { + tokio::task::yield_now().await; + } + }) + .await + .expect("the suspended trait heal must have started the real metadata fanout"); + for disk_index in 0..4 { + assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1); + } + drop(read_gate); + + let (_, outcome) = tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(renew, heal) }) + .await + .expect("trait heal and real disk renewal must finish without a nested inventory read deadlock"); + let (report, error) = outcome.expect("heal should report the existing object"); + assert!(error.is_none(), "existing object heal failed after renewal: {error:?}"); + assert_eq!(report.bucket, bucket); + assert_eq!(report.object, object); + assert_eq!(report.disk_count, 4); + let renewed = set.get_disks_internal().await[0] + .clone() + .expect("the renewed slot must remain online"); + assert!(Arc::ptr_eq(&renewed, ®istered), "the set must publish the newly connected handle"); + assert_eq!(renewed.endpoint(), endpoint); + let format = load_format_erasure(&renewed, false) + .await + .expect("renewed disk format should remain readable"); + assert_eq!(format.erasure.this, set.format.erasure.sets[0][0]); + tokio::time::timeout(Duration::from_secs(10), async { + let mut reader = set + .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default()) + .await + .expect("the object must remain readable after renewal and heal"); + assert_eq!(reader.object_info.etag, published.etag); + let mut observed_body = Vec::new(); + reader + .stream + .read_to_end(&mut observed_body) + .await + .expect("stored body should stream"); + assert_eq!(observed_body, body); + }) + .await + .expect("GET must finish after renewal and heal"); + } + // Regression for #955: an offline disk must contribute exactly one drive // record. Before the fix the offline branch fell through and pushed a second // (Corrupt) record for the same disk, so `before/after.drives` grew to From 55ad7508b9131f6d0a9e20785937a2fb993c3fef Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 21:24:00 +0800 Subject: [PATCH 35/40] fix(tier): persist coordinator intent before waking refresh (#7171) --- crates/ecstore/src/services/tier/tier.rs | 187 ++++++++++++++++++++--- 1 file changed, 168 insertions(+), 19 deletions(-) diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index af24423fa..5bb35f956 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -3541,7 +3541,7 @@ impl TierConfigMgr { // Get tier configuration and create new driver let tier_config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?; - let driver = new_warm_backend(tier_config, false).await?; + let driver = construct_warm_backend(tier_config).await?; self.replace_driver(tier_name, driver)?; Ok(self @@ -4486,6 +4486,11 @@ impl TierConfigMgr { let committed_coordinator_intent = committed_tier_mutation_intent(coordinator_intent.as_ref(), &committed_config_etag) .map_err(TierConfigUpdateError::Save)?; + // Persist Committed before notifying refresh; a Prepared disk record + // would restore the prepared block and invalidate our publish allowance. + let coordinator_commit = + commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag) + .await; if let Some(intent) = committed_coordinator_intent.as_ref() { TierConfigMgr::apply_committed_mutation_intent_block(&handle, intent) .await @@ -4496,9 +4501,9 @@ impl TierConfigMgr { .map_err(TierConfigUpdateError::Publish)?, ); } - commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag) - .await - .map_err(TierConfigUpdateError::Save)?; + // Config is already saved: retain the committed fence and wake recovery + // even when the coordinator commit failed or its outcome is unknown. + coordinator_commit.map_err(TierConfigUpdateError::Save)?; if coordinated_config_update { drop(update.take()); drop(config_lock.take()); @@ -10603,6 +10608,11 @@ mod tests { .expect_err("coordinator committed-state CAS failure must be observable"); assert!(matches!(err, TierConfigUpdateError::Save(_))); assert!(manager.read().await.tiers.contains_key("COLD-A")); + assert!(TierConfigMgr::has_committed_mutation_block(&manager).await); + let refresh = TierConfigMgr::mutation_refresh_notifier(&manager).await; + tokio::time::timeout(Duration::from_secs(1), refresh.notified()) + .await + .expect("failed coordinator commit must notify recovery after saving config"); let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await { Ok(_) => panic!("failed coordinator commit CAS must retain the local committed fence"), Err(err) => err, @@ -14329,6 +14339,12 @@ mod tests { after_commit: bool, } + #[derive(Debug, Default)] + struct CasCoordinatorCommitBarrier { + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, + } + #[derive(Debug)] struct CasConfigStore { objects: tokio::sync::Mutex, String)>>, @@ -14341,6 +14357,7 @@ mod tests { fail_delete_prefix: tokio::sync::Mutex>, delete_log: tokio::sync::Mutex>, list_barrier: tokio::sync::Mutex>>, + coordinator_commit_barrier: tokio::sync::Mutex>>, intent_list_calls: AtomicUsize, fail_reference_walk: AtomicBool, reference_walk_send_count: AtomicUsize, @@ -14363,6 +14380,7 @@ mod tests { fail_delete_prefix: tokio::sync::Mutex::new(None), delete_log: tokio::sync::Mutex::new(Vec::new()), list_barrier: tokio::sync::Mutex::new(None), + coordinator_commit_barrier: tokio::sync::Mutex::new(None), intent_list_calls: AtomicUsize::new(0), fail_reference_walk: AtomicBool::new(false), reference_walk_send_count: AtomicUsize::new(0), @@ -14554,6 +14572,19 @@ mod tests { } let mut payload = Vec::new(); tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?; + if object.starts_with(crate::services::tier::tier_mutation_intent::TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX) + && opts + .http_preconditions + .as_ref() + .and_then(HTTPPreconditions::if_match_value) + .is_some() + { + let barrier = self.coordinator_commit_barrier.lock().await.take(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } + } let race_rewrite = if opts .http_preconditions .as_ref() @@ -15651,14 +15682,7 @@ mod tests { ); } - #[tokio::test] - async fn force_remove_and_save_bypasses_lifecycle_only_reference() { - // rustfs/rustfs#6832: reproduces the admin RemoveTier path (not just the lower-level - // reference-proof function) for a tier with zero transitioned objects but a lifecycle - // rule still pointing at it — the exact shape of - // `test_manual_transition_async_tier_failure_reports_terminal_partial` in e2e_test, - // which force-removes a tier a lifecycle rule still references to simulate a - // decommissioned backend. + async fn assert_lifecycle_only_reference_obeys_force(clear: bool, force: bool) { let store = Arc::new(CasConfigStore::default()); let tier = build_rustfs_tier("COLD-A"); let mut persisted = empty_mgr(); @@ -15699,22 +15723,55 @@ mod tests { let manager = TierConfigMgr::new(); manager.write().await.tiers.insert("COLD-A".to_string(), tier); - TierConfigMgr::remove_and_save_with(&manager, store.clone(), "COLD-A", true) - .await - .expect("force remove must bypass a lifecycle-config-only reference"); + let mutation = if clear { + TierCandidateMutation::Clear(force) + } else { + TierCandidateMutation::Remove("COLD-A".to_string(), force) + }; + let result = TIER_DRIVER_TEST_FACTORY + .scope( + healthy_driver_factory(), + TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), mutation), + ) + .await; + if force { + result.expect("force mutation must bypass a lifecycle-config-only reference"); + } else { + let err = result.expect_err("non-force mutation must reject a lifecycle-only reference"); + let TierConfigUpdateError::Publish(err) = err else { + panic!("non-force mutation must fail during reference proof: {err:?}"); + }; + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(err.message.contains("move-current"), "{err}"); + } - assert!(!manager.read().await.tiers.contains_key("COLD-A")); - assert!( - !load_tier_config_for_update(store) + assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), !force); + assert_eq!( + load_tier_config_for_update(store) .await .expect("config should still reload") .0 .tiers .contains_key("COLD-A"), - "force removal must persist the empty candidate" + !force, + "persisted state must match the force mutation result" ); } + #[tokio::test] + async fn remove_with_config_lock_obeys_force_for_lifecycle_only_reference() { + for force in [false, true] { + assert_lifecycle_only_reference_obeys_force(false, force).await; + } + } + + #[tokio::test] + async fn clear_with_config_lock_obeys_force_for_lifecycle_only_reference() { + for force in [false, true] { + assert_lifecycle_only_reference_obeys_force(true, force).await; + } + } + #[tokio::test] async fn zero_reference_proof_blocks_clear_before_config_save() { let store = Arc::new(CasConfigStore::default()); @@ -17255,6 +17312,98 @@ mod tests { assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty()); } + async fn assert_coordinator_commit_refresh_succeeds(mutation: TierCandidateMutation) { + let adding = matches!(mutation, TierCandidateMutation::Add(..)); + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + if !adding { + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("existing tier fixture should persist"); + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let barrier = Arc::new(CasCoordinatorCommitBarrier::default()); + *store.coordinator_commit_barrier.lock().await = Some(barrier.clone()); + let update_manager = manager.clone(); + let update_store = store.clone(); + let update = tokio::spawn(async move { + TIER_DRIVER_TEST_FACTORY + .scope( + healthy_driver_factory(), + TIER_MUTATION_TEST_PEERS.scope( + Vec::new(), + TierConfigMgr::update_candidate_with_config_lock(&update_manager, update_store, mutation), + ), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), barrier.arrived.notified()) + .await + .expect("mutation should reach coordinator commit after saving config"); + assert_eq!( + load_tier_config_for_update(store.clone()) + .await + .expect("saved config should be readable before coordinator commit") + .0 + .tiers + .contains_key("COLD-A"), + adding + ); + assert_eq!( + TierConfigMgr::load_coordinator_mutation_intents(store.clone()) + .await + .expect("coordinator intent should remain readable")[0] + .state, + TierMutationIntentState::Prepared + ); + + let lock_requests = lock_unpoisoned(&store.lock_requests).len(); + // Also exercise an independently scheduled refresh while the durable + // coordinator record is still Prepared, before its commit notification. + TierConfigMgr::request_committed_mutation_refresh(&manager).await; + TIER_MUTATION_TEST_PEERS + .scope(Vec::new(), async { + let worker = TierConfigMgr::refresh_tier_config_handle_with(manager.clone(), store.clone()); + tokio::pin!(worker); + tokio::time::timeout(Duration::from_secs(5), async { + while lock_unpoisoned(&store.lock_requests).len() == lock_requests { + tokio::select! { + _ = &mut worker => panic!("refresh worker must remain available"), + _ = tokio::task::yield_now() => {} + } + } + }) + .await + .expect("refresh should reconcile the Prepared record before waiting for the config lock"); + barrier.release.notify_one(); + let result = tokio::time::timeout(Duration::from_secs(5), async { + tokio::select! { + _ = &mut worker => panic!("refresh worker must remain available"), + result = update => result.expect("tier mutation task should join"), + } + }) + .await + .expect("tier mutation should finish with refresh running"); + result.expect("saved tier mutation must publish successfully on the first attempt"); + }) + .await; + assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), adding); + } + + #[tokio::test] + async fn tier_add_succeeds_with_refresh_during_coordinator_commit() { + assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true)).await; + } + + #[tokio::test] + async fn tier_remove_succeeds_with_refresh_during_coordinator_commit() { + assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Remove("COLD-A".to_string(), true)).await; + } + async fn committed_refresh_fixture(fail_cleanup: bool) -> (Arc>, Arc, uuid::Uuid) { let manager = TierConfigMgr::new(); { From d915f9565e59939a5645fa0faebf5f1613bbba93 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 21:26:23 +0800 Subject: [PATCH 36/40] fix(ci): preserve reported functional suite failures (#7199) --- .github/workflows/rustfs-heal-test.yml | 3 - .github/workflows/rustfs-kms-test.yml | 2 - .github/workflows/rustfs-performance-test.yml | 3 - .github/workflows/rustfs-pool-expand-test.yml | 3 - .github/workflows/rustfs-replication-test.yml | 4 - .github/workflows/rustfs-s3-compat-test.yml | 2 - .github/workflows/rustfs-storage-test.yml | 2 - .github/workflows/rustfs-tier-test.yml | 3 - .github/workflows/rustfs-upgrade-test.yml | 2 - docs/testing/ci-gates.md | 6 + scripts/test_security_workflow.py | 121 ++++++++++++++++-- 11 files changed, 113 insertions(+), 38 deletions(-) diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 995cc9b23..666c83b4c 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -54,9 +54,6 @@ env: jobs: heal-test: runs-on: smoke-testing - # Requirement: a failing suite must not fail the workflow; failures - # are filed to rustfs/backlog and the chain continues. - continue-on-error: true timeout-minutes: 480 # Standalone manual run, or one link of the nightly functional chain # (storage -> heal -> pool). Pool expansion no longer re-runs heal. diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml index 642c4b5e7..8647c9268 100644 --- a/.github/workflows/rustfs-kms-test.yml +++ b/.github/workflows/rustfs-kms-test.yml @@ -49,7 +49,6 @@ env: jobs: kms-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -109,7 +108,6 @@ jobs: - name: Run KMS suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-kms.log run: | diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml index 6d960052c..ff3e2978d 100644 --- a/.github/workflows/rustfs-performance-test.yml +++ b/.github/workflows/rustfs-performance-test.yml @@ -84,9 +84,6 @@ env: jobs: performance-test: runs-on: pf-testing - # Requirement: a failing benchmark must not fail the workflow; - # failures are filed to rustfs/backlog. - continue-on-error: true timeout-minutes: 900 # Run on manual dispatch, or when the nightly build completed successfully. # Skipped when nightly failed. diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index d7002c446..3c9836b2d 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -76,9 +76,6 @@ jobs: pool-expansion-test: name: Pool expansion / decommission test runs-on: smoke-testing - # Requirement: a failing suite must not fail the workflow; failures - # are filed to rustfs/backlog and the chain continues. - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} env: diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml index b17864f72..74c57b6d5 100644 --- a/.github/workflows/rustfs-replication-test.yml +++ b/.github/workflows/rustfs-replication-test.yml @@ -62,9 +62,6 @@ env: jobs: replication-test: runs-on: smoke-testing - # A failed replication run must not break the chain or the workflow: the - # failure is reported to rustfs/backlog instead (see the issue step). - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -116,7 +113,6 @@ jobs: - name: Run replication suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-replication.log run: | diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml index d3fff002b..80856194c 100644 --- a/.github/workflows/rustfs-s3-compat-test.yml +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -37,7 +37,6 @@ env: jobs: s3-compat-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -88,7 +87,6 @@ jobs: - name: Run S3 compatibility suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-s3-compat.log run: | diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml index 1dceda80d..767f734dc 100644 --- a/.github/workflows/rustfs-storage-test.yml +++ b/.github/workflows/rustfs-storage-test.yml @@ -46,7 +46,6 @@ env: jobs: storage-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -97,7 +96,6 @@ jobs: - name: Run storage engine suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-storage.log run: | diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml index 4ac80e609..5d9a2c1d7 100644 --- a/.github/workflows/rustfs-tier-test.yml +++ b/.github/workflows/rustfs-tier-test.yml @@ -61,9 +61,6 @@ env: jobs: tier-test: runs-on: smoke-testing - # Requirement: a failing suite must not fail the workflow; failures - # are filed to rustfs/backlog and the chain continues. - continue-on-error: true timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml index 0c8c72cd0..0b4c19af1 100644 --- a/.github/workflows/rustfs-upgrade-test.yml +++ b/.github/workflows/rustfs-upgrade-test.yml @@ -79,7 +79,6 @@ env: jobs: upgrade-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -142,7 +141,6 @@ jobs: - name: Run upgrade compatibility suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-upgrade.log GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 93770b74d..99d589d64 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -91,6 +91,12 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched Manual `workflow_dispatch` runs are debugging evidence and do not open scheduled-failure issues. A manual performance run may explicitly allow a known regression; that override is not a passing baseline. +## Packaged functional acceptance + +`rustfs-functional-chain.yml` dispatches the packaged-build suites in `rustfs-*-test.yml` on the shared lab runners. A failing suite step or job must fail its workflow. Report collection, cleanup, and dispatch of the next suite can still run with `always()`; continuing diagnostics does not make the failed suite successful. + +Workflow status preserves errors that the test scripts report. It does not establish complete execution or a common package identity across the chain: inspect the current run's case results, package identity, and test-script revision as well. A script that returns zero after a failed tool invocation needs its own result check. + ## Release validation Post-merge and tag-driven; not a substitute for a PR gate. diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index ae82d3fb1..ea2d75487 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Run the security workflow's evidence and result steps without remote VMs.""" +"""Exercise functional workflow failures and security evidence without remote VMs.""" from __future__ import annotations @@ -18,16 +18,32 @@ WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml" CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |" +def named_steps(job: list[str]) -> dict[str, list[str]]: + starts = [i for i, line in enumerate(job) if line.startswith(" - name: ")] + return { + job[start].split(": ", 1)[1].strip('"'): job[start:end] + for start, end in zip(starts, starts[1:] + [len(job)]) + } + + +def shell_body(lines: list[str]) -> str: + start = lines.index(" run: |") + 1 + shell_lines = [] + for line in lines[start:]: + if line.strip() and not line.startswith(" "): + break + shell_lines.append(line[10:]) + if not shell_lines: + raise ValueError("missing literal shell body") + return "\n".join(shell_lines) + + class SecurityWorkflowTests(unittest.TestCase): def setUp(self) -> None: self.source = WORKFLOW.read_text() self.job = yaml_block(self.source.splitlines(), "security-test", 2) self.assertIsNotNone(self.job) - starts = [i for i, line in enumerate(self.job) if line.startswith(" - name: ")] - self.steps = { - self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end] - for start, end in zip(starts, starts[1:] + [len(self.job)]) - } + self.steps = named_steps(self.job) self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.directory = Path(self.temp.name) @@ -83,15 +99,8 @@ class SecurityWorkflowTests(unittest.TestCase): def run_step(self, name: str) -> subprocess.CompletedProcess[str]: lines = self.steps[name] - start = lines.index(" run: |") + 1 - shell_lines = [] - for line in lines[start:]: - if line.strip() and not line.startswith(" "): - break - shell_lines.append(line[10:]) - self.assertTrue(shell_lines, f"missing literal shell body: {name}") result = subprocess.run( - ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render("\n".join(shell_lines))], + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))], cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True, ) for line in lines: @@ -193,5 +202,89 @@ class SecurityWorkflowTests(unittest.TestCase): self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text()) +class FunctionalWorkflowTests(unittest.TestCase): + JOBS = { + "kms": "kms-test", "storage": "storage-test", "s3-compat": "s3-compat-test", + "upgrade": "upgrade-test", "replication": "replication-test", "heal": "heal-test", + "tier": "tier-test", "pool-expand": "pool-expansion-test", "performance": "performance-test", + } + DIRECT_TESTS = { + "kms": "Run KMS suite", "storage": "Run storage engine suite", + "s3-compat": "Run S3 compatibility suite", "upgrade": "Run upgrade compatibility suite", + "replication": "Run replication suite", + } + + def test_failure_and_always_step_wiring(self) -> None: + for suite, job_id in self.JOBS.items(): + with self.subTest(suite=suite): + source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text() + job = yaml_block(source.splitlines(), job_id, 2) + self.assertIsNotNone(job) + self.assertNotRegex("\n".join(job), r'''(?m)^ ["']?continue-on-error["']?\s*:''') + steps = named_steps(job) + if suite in self.DIRECT_TESTS: + test = steps[self.DIRECT_TESTS[suite]] + self.assertNotRegex("\n".join(test), r'''(?m)^ ["']?continue-on-error["']?\s*:''') + self.assertIn(" if: always()", steps["Generate report"]) + cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"] + condition = next(line.strip() for line in cleanup if line.startswith(" if:")) + self.assertIn(condition, ( + "if: always()", + "if: ${{ always() && inputs.cleanup_after != 'false' }}", + "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}", + )) + if suite != "performance": + handoff = steps["Chain complete"] if suite == "replication" else next( + value for name, value in steps.items() if name.startswith("Continue functional chain") + ) + self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff) + + def test_failed_suite_preserves_exit_and_cleanup_and_dispatch_execute(self) -> None: + for suite, test_name in self.DIRECT_TESTS.items(): + with self.subTest(suite=suite), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "auto-testing").mkdir() + script = root / f"auto-testing/rustfs-{suite}-test.sh" + script.write_text('#!/bin/sh\nprintf "partial suite diagnostics\\n"\nexit 17\n') + script.chmod(0o755) + fake_bin = root / "bin" + fake_bin.mkdir() + for command, marker in (("ssh", "cleanup"), ("gh", "dispatch")): + fake = fake_bin / command + fake.write_text(f'#!/bin/sh\nprintf "{marker}\\n" >> "$EXECUTED"\n') + fake.chmod(0o755) + env = { + **os.environ, "PATH": f"{fake_bin}{os.pathsep}{os.environ['PATH']}", + "EXECUTED": str(root / "executed"), "RUSTFS_NODES": "fixture-node", + "RUSTFS_SSH_USER": "fixture-user", "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", + "GH_TOKEN": "local-fixture", "GITHUB_EVENT_NAME": "repository_dispatch", "GITHUB_RUN_ID": "314159", + } + source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text() + steps = named_steps(yaml_block(source.splitlines(), self.JOBS[suite], 2)) + context = {"github.event_name": "repository_dispatch", "steps.test.outcome": "failure"} + for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", source): + if expression.startswith("inputs.") and re.fullmatch(r"inputs\.\w+", expression): + context[expression] = "" + def execute(name): + lines = steps[name] + rendered = re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: context[match[1]], shell_body(lines)) + return subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", rendered], + cwd=root, env={**env, "LOG_FILE": str(root / "suite.log")}, capture_output=True, text=True, + ) + failed = execute(test_name) + self.assertEqual(failed.returncode, 17, failed.stderr) + self.assertIn("partial suite diagnostics", failed.stdout) + cleanup = execute("Cleanup environment (after)") + self.assertEqual(cleanup.returncode, 0, cleanup.stderr) + handoff_name = "Chain complete" if suite == "replication" else next( + name for name in steps if name.startswith("Continue functional chain") + ) + handoff = execute(handoff_name) + self.assertEqual(handoff.returncode, 0, handoff.stderr) + markers = (root / "executed").read_text().splitlines() + self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"]) + + if __name__ == "__main__": unittest.main() From 447f3c704bc0c7125f7be2708d4d416ab122caaf Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 21:33:41 +0800 Subject: [PATCH 37/40] feat(heal): add explicit committed MRF snapshot reader (#7179) * chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * feat(heal): add explicit committed MRF snapshot reader Refs rustfs/backlog#2263 and rustfs/backlog#2240. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * docs(heal): register legacy MRF inspection cleanup State the compatibility removal condition on the source marker and in the architecture cleanup register. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/heal/src/heal/mrf_queue.rs | 4 + crates/heal/src/heal/mrf_queue/snapshot.rs | 681 +++++++++++++++++++ docs/architecture/compat-cleanup-register.md | 1 + 3 files changed, 686 insertions(+) create mode 100644 crates/heal/src/heal/mrf_queue/snapshot.rs diff --git a/crates/heal/src/heal/mrf_queue.rs b/crates/heal/src/heal/mrf_queue.rs index 026653d03..901064823 100644 --- a/crates/heal/src/heal/mrf_queue.rs +++ b/crates/heal/src/heal/mrf_queue.rs @@ -45,6 +45,10 @@ use uuid::Uuid; use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType}; +/// Read-only inspection of committed MRF checkpoints. The legacy consumer +/// remains unchanged until ownership-aware replay is deployed. +pub mod snapshot; + /// Journal location inside the metadata bucket, following the resume-state /// layout. pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin"; diff --git a/crates/heal/src/heal/mrf_queue/snapshot.rs b/crates/heal/src/heal/mrf_queue/snapshot.rs new file mode 100644 index 000000000..e8d51ed9e --- /dev/null +++ b/crates/heal/src/heal/mrf_queue/snapshot.rs @@ -0,0 +1,681 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Reader-first support for owner-local MRF checkpoints. +//! +//! Each of two slots has a payload and a commit manifest. The manifest binds +//! the writer identity, persistent sequence, length and whole-payload digest. +//! Replacing the inactive slot must leave the previous committed slot intact. +//! Production publication and reclamation are deliberately not enabled here. +//! An unreadable commit path cannot prove that only legacy data exists. This +//! explicit inspection API fails closed and never mutates recovery anchors. +//! It is not wired into the legacy consumer: that transition requires the +//! ownership-aware replay and producer handoff before writer activation. +//! One surviving committed replica supports process restart recovery only; +//! this reader does not establish a replication quorum or a power-loss policy. + +use super::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_journal}; +use crate::heal::RUSTFS_META_BUCKET; +use crate::heal::storage_api::owner::{EcstoreDiskAPI, EcstoreDiskError, EcstoreDiskStore}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use tokio::io::AsyncReadExt; +use uuid::Uuid; + +// Root-level control files avoid requiring a new directory before the first +// atomic commit. They remain inside the storage owner's metadata volume. +const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"]; +const MANIFEST_PATHS: [&str; 2] = [".heal-mrf-commit.0.bin", ".heal-mrf-commit.1.bin"]; +const MAGIC: &[u8; 8] = b"RFMRFC01"; +const MANIFEST_LEN: usize = 8 + 1 + 16 + 8 + 8 + 32 + 32; +const VERSION: u8 = 1; + +#[derive(Debug, thiserror::Error)] +pub enum SnapshotError { + #[error("MRF checkpoint has an invalid or incomplete commit record")] + Corrupt, + #[error("MRF checkpoint format is unsupported")] + Unsupported, + #[error("MRF checkpoint exceeds the configured byte limit")] + TooLarge, + #[error("MRF checkpoint replicas disagree at the same sequence")] + Conflict, + #[error("MRF checkpoint storage is unavailable")] + Disk(#[source] EcstoreDiskError), + #[error("MRF checkpoint body could not be read")] + Read(#[source] std::io::Error), +} + +#[derive(Debug, PartialEq, Eq)] +struct Manifest { + owner: Uuid, + sequence: u64, + payload_len: usize, + payload_digest: [u8; 32], +} + +impl Manifest { + fn decode(bytes: &[u8], limit: usize) -> Result { + if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC { + return Err(SnapshotError::Corrupt); + } + if bytes[8] != VERSION { + return Err(SnapshotError::Unsupported); + } + let signed = MANIFEST_LEN - 32; + let checksum: [u8; 32] = Sha256::digest(&bytes[..signed]).into(); + if checksum != bytes[signed..] { + return Err(SnapshotError::Corrupt); + } + let owner = Uuid::from_slice(&bytes[9..25]).map_err(|_| SnapshotError::Corrupt)?; + let sequence = u64::from_le_bytes(bytes[25..33].try_into().map_err(|_| SnapshotError::Corrupt)?); + let payload_len = u64::from_le_bytes(bytes[33..41].try_into().map_err(|_| SnapshotError::Corrupt)?); + let payload_len = usize::try_from(payload_len).map_err(|_| SnapshotError::TooLarge)?; + if owner.is_nil() || sequence == 0 || sequence == u64::MAX { + return Err(SnapshotError::Corrupt); + } + if payload_len > limit { + return Err(SnapshotError::TooLarge); + } + Ok(Self { + owner, + sequence, + payload_len, + payload_digest: bytes[41..73].try_into().map_err(|_| SnapshotError::Corrupt)?, + }) + } +} + +#[derive(Debug)] +pub struct CommittedSnapshot { + manifest: Manifest, + payload: Vec, +} + +impl CommittedSnapshot { + /// Persistent single-writer sequence, not a process UUID ordering. + pub fn sequence(&self) -> u64 { + self.manifest.sequence + } + + /// Identity recorded by the committed checkpoint's writer. + pub fn owner(&self) -> Uuid { + self.manifest.owner + } + + /// Complete, checksum-validated record bytes. Inspection does not consume + /// these records or acknowledge completion to any producer. + pub fn payload(&self) -> &[u8] { + &self.payload + } + + fn decode(manifest: &[u8], payload: Vec, limit: usize) -> Result { + let manifest = Manifest::decode(manifest, limit)?; + let checksum: [u8; 32] = Sha256::digest(&payload).into(); + if payload.len() != manifest.payload_len || checksum != manifest.payload_digest { + return Err(SnapshotError::Corrupt); + } + if decode_journal(&payload).1 != 0 { + return Err(SnapshotError::Corrupt); + } + Ok(Self { manifest, payload }) + } +} + +#[derive(Debug)] +pub enum RecoverySnapshot { + /// An intact legacy snapshot, without a comparable commit sequence. + Legacy(Vec), + /// A committed checkpoint requiring ownership-aware replay before use. + Committed(CommittedSnapshot), +} + +async fn read_bounded(disk: &EcstoreDiskStore, path: &str, limit: usize) -> Result>, SnapshotError> { + let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await { + Ok(reader) => reader, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) => return Ok(None), + Err(error) => return Err(SnapshotError::Disk(error)), + }; + let maximum = limit.checked_add(1).ok_or(SnapshotError::TooLarge)?; + let maximum = u64::try_from(maximum).map_err(|_| SnapshotError::TooLarge)?; + let mut bytes = Vec::new(); + reader + .take(maximum) + .read_to_end(&mut bytes) + .await + .map_err(SnapshotError::Read)?; + if bytes.len() > limit { + return Err(SnapshotError::TooLarge); + } + Ok(Some(bytes)) +} + +fn select_snapshot(selected: &mut Option, candidate: CommittedSnapshot) -> Result<(), SnapshotError> { + if let Some(current) = selected { + if current.manifest.sequence == candidate.manifest.sequence + && (current.manifest != candidate.manifest || current.payload != candidate.payload) + { + return Err(SnapshotError::Conflict); + } + if current.manifest.sequence >= candidate.manifest.sequence { + return Ok(()); + } + } + *selected = Some(candidate); + Ok(()) +} + +async fn read_committed(disks: &[EcstoreDiskStore], limit: usize) -> Result, SnapshotError> { + let mut selected = None; + let mut damaged = None; + let mut identities = HashMap::new(); + for disk in disks { + for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) { + let candidate = async { + let Some(manifest) = read_bounded(disk, manifest_path, MANIFEST_LEN).await? else { + return Ok(None); + }; + let header = Manifest::decode(&manifest, limit)?; + let payload = read_bounded(disk, payload_path, header.payload_len) + .await? + .ok_or(SnapshotError::Corrupt)?; + CommittedSnapshot::decode(&manifest, payload, limit).map(Some) + } + .await; + match candidate { + Ok(Some(candidate)) => { + let identity = ( + candidate.manifest.owner, + candidate.manifest.payload_len, + candidate.manifest.payload_digest, + ); + if identities + .insert(candidate.manifest.sequence, identity) + .is_some_and(|previous| previous != identity) + { + return Err(SnapshotError::Conflict); + } + select_snapshot(&mut selected, candidate)?; + } + Ok(None) => {} + // A future committed format may supersede all readable slots. + Err(SnapshotError::Unsupported) => return Err(SnapshotError::Unsupported), + Err(error) => damaged = Some(error), + } + } + } + match (selected, damaged) { + (Some(snapshot), _) => Ok(Some(snapshot)), + (None, Some(error)) => Err(error), + (None, None) => Ok(None), + } +} + +async fn read_legacy(disks: &[EcstoreDiskStore], path: &str, limit: usize) -> Result>, SnapshotError> { + let mut selected = None; + let mut incomplete: Option> = None; + for disk in disks { + match read_bounded(disk, path, limit).await { + Ok(Some(payload)) if decode_journal(&payload).1 == 0 => { + if selected.as_ref().is_some_and(|current| *current != payload) { + // Legacy snapshots have no sequence. There is no evidence + // that the first, longest or nonempty replica is newest. + return Err(SnapshotError::Conflict); + } + selected = Some(payload); + } + Ok(Some(payload)) => { + if let Some(previous) = &incomplete { + if previous.starts_with(&payload) { + continue; + } + if !payload.starts_with(previous) { + return Err(SnapshotError::Corrupt); + } + } + incomplete = Some(payload); + } + Ok(None) => {} + Err(error) => return Err(error), + } + } + if let Some(prefix) = incomplete + && !selected.as_ref().is_some_and(|payload| payload.starts_with(&prefix)) + { + // In particular, an empty O_TRUNC replica cannot supersede another + // replica containing intact records followed by a torn tail. + return Err(SnapshotError::Corrupt); + } + Ok(selected) +} + +/// Inspect local MRF checkpoints without replaying, acknowledging or deleting. +/// +/// `max_bytes` bounds each payload read. Every local replica is examined and +/// ambiguous identities, unavailable proof or unsupported formats return a +/// typed error. This API must not authorize a writer without the separate +/// ownership and mixed-version activation checks. +pub async fn inspect_local_recovery_snapshot(max_bytes: usize) -> Result, SnapshotError> { + read_recovery_snapshot(&super::journal_disks().await, max_bytes).await +} + +async fn read_recovery_snapshot(disks: &[EcstoreDiskStore], limit: usize) -> Result, SnapshotError> { + if let Some(snapshot) = read_committed(disks, limit).await? { + return Ok(Some(RecoverySnapshot::Committed(snapshot))); + } + // RUSTFS_COMPAT_TODO(backlog-2263): inspect retained legacy MRF journals. Remove after all supported upgrade and rollback readers understand committed snapshots and retained journals have migrated. + if let Some(payload) = read_legacy(disks, MRF_SCOPED_JOURNAL_PATH, limit).await? { + return Ok(Some(RecoverySnapshot::Legacy(payload))); + } + Ok(read_legacy(disks, MRF_JOURNAL_PATH, limit) + .await? + .map(RecoverySnapshot::Legacy)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::heal::mrf_queue::encode_intent; + use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes}; + use crate::heal::{DiskOption, Endpoint, new_disk}; + use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope}; + use std::sync::Arc; + use tempfile::TempDir; + + fn payload(object: &str) -> Vec { + let intent = MrfIntent { + bucket: Arc::from("bucket"), + object: Arc::from(object), + version_id: None, + kind: MrfKind::PartialWrite, + scope: None, + lease: None, + enqueued_at_ms: 1234, + attempts: 0, + }; + let mut bytes = Vec::new(); + assert!(encode_intent(&intent, &mut bytes), "fixture must encode a full record"); + bytes + } + + fn manifest(owner: Uuid, sequence: u64, payload: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(MANIFEST_LEN); + bytes.extend_from_slice(MAGIC); + bytes.push(VERSION); + bytes.extend_from_slice(owner.as_bytes()); + bytes.extend_from_slice(&sequence.to_le_bytes()); + bytes.extend_from_slice(&u64::try_from(payload.len()).expect("fixture length fits").to_le_bytes()); + bytes.extend_from_slice(&Sha256::digest(payload)); + bytes.extend_from_slice(&Sha256::digest(&bytes)); + bytes + } + + async fn disk(root: &TempDir, name: &str) -> EcstoreDiskStore { + let path = root.path().join(name); + std::fs::create_dir_all(&path).expect("create disk directory"); + let endpoint = Endpoint::try_from(path.to_string_lossy().as_ref()).expect("valid disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("open disk"); + let result = EcstoreDiskAPI::make_volume(disk.as_ref(), RUSTFS_META_BUCKET).await; + assert!( + matches!(result, Ok(()) | Err(EcstoreDiskError::VolumeExists)), + "metadata volume: {result:?}" + ); + disk + } + + // Exercise the existing storage owner's atomic CAS primitive. No production + // caller publishes this format until ownership-aware replay is available. + async fn install(disk: &EcstoreDiskStore, path: &str, bytes: &[u8]) { + let expected = EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await.ok(); + let result = EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + path, + expected, + Some(EcstoreDiskBytes::copy_from_slice(bytes)), + ) + .await + .expect("atomic snapshot slot write"); + assert_eq!(result, EcstoreConditionalFileUpdate::Updated); + } + + async fn commit(disk: &EcstoreDiskStore, slot: usize, owner: Uuid, sequence: u64, bytes: &[u8]) { + install(disk, PAYLOAD_PATHS[slot], bytes).await; + install(disk, MANIFEST_PATHS[slot], &manifest(owner, sequence, bytes)).await; + } + + #[test] + fn manifest_validates_identity_sequence_length_and_digest() { + let bytes = payload("object"); + let owner = Uuid::new_v4(); + assert!(CommittedSnapshot::decode(&manifest(owner, 1, &bytes), bytes.clone(), bytes.len()).is_ok()); + for (owner, sequence) in [(Uuid::nil(), 1), (owner, 0), (owner, u64::MAX)] { + assert!(matches!( + Manifest::decode(&manifest(owner, sequence, &bytes), bytes.len()), + Err(SnapshotError::Corrupt) + )); + } + assert!(matches!( + Manifest::decode(&manifest(owner, 1, &bytes), bytes.len() - 1), + Err(SnapshotError::TooLarge) + )); + let mut corrupt = manifest(owner, 1, &bytes); + corrupt[25] ^= 1; + assert!(matches!(Manifest::decode(&corrupt, bytes.len()), Err(SnapshotError::Corrupt))); + let mut unsupported = manifest(owner, 1, &bytes); + unsupported[8] = 2; + assert!(matches!(Manifest::decode(&unsupported, bytes.len()), Err(SnapshotError::Unsupported))); + } + + #[test] + fn whole_payload_integrity_is_required_even_with_a_valid_manifest() { + let bytes = payload("object"); + let owner = Uuid::new_v4(); + let header = manifest(owner, 1, &bytes); + assert!(matches!( + CommittedSnapshot::decode(&header, bytes[..bytes.len() - 1].to_vec(), bytes.len()), + Err(SnapshotError::Corrupt) + )); + let invalid = b"not an MRF record".to_vec(); + assert!(matches!( + CommittedSnapshot::decode(&manifest(owner, 2, &invalid), invalid, bytes.len()), + Err(SnapshotError::Corrupt) + )); + } + + #[tokio::test] + async fn newest_complete_replica_wins_in_both_disk_orders() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let owner = Uuid::new_v4(); + commit(&first, 0, owner, 1, &payload("old")).await; + commit(&second, 1, owner, 2, &payload("new")).await; + for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] { + let recovered = read_committed(&disks, 4096) + .await + .expect("read replicas") + .expect("committed snapshot"); + assert_eq!(recovered.manifest.sequence, 2); + assert_eq!(recovered.payload, payload("new")); + } + } + + #[tokio::test] + async fn divergent_commits_at_same_sequence_fail_closed() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let owner = Uuid::new_v4(); + commit(&first, 0, owner, 7, &payload("a")).await; + commit(&second, 1, owner, 7, &payload("b")).await; + assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict))); + } + + #[tokio::test] + async fn newer_slot_does_not_hide_a_conflicting_commit_history() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let owner = Uuid::new_v4(); + commit(&first, 0, owner, 8, &payload("newest")).await; + commit(&first, 1, owner, 7, &payload("a")).await; + commit(&second, 1, owner, 7, &payload("b")).await; + assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict))); + } + + #[tokio::test] + async fn uncommitted_or_torn_successor_preserves_previous_slot() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + commit(&disk, 0, owner, 1, &old).await; + install(&disk, PAYLOAD_PATHS[1], &next).await; + let recovered = read_committed(std::slice::from_ref(&disk), 4096) + .await + .expect("staged payload is not a commit") + .expect("old snapshot"); + assert_eq!(recovered.payload, old); + install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)[..20]).await; + let recovered = read_committed(std::slice::from_ref(&disk), 4096) + .await + .expect("torn manifest preserves old slot") + .expect("old snapshot"); + assert_eq!(recovered.manifest.sequence, 1); + install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)).await; + install(&disk, PAYLOAD_PATHS[1], b"torn").await; + let recovered = read_committed(&[disk], 4096) + .await + .expect("torn payload preserves old slot") + .expect("old snapshot"); + assert_eq!(recovered.manifest.sequence, 1); + } + + #[tokio::test] + async fn stale_manifest_cas_cannot_replace_committed_anchor() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let bytes = payload("object"); + commit(&disk, 0, owner, 1, &bytes).await; + let result = EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + MANIFEST_PATHS[0], + None, + Some(manifest(owner, 2, &bytes).into()), + ) + .await + .expect("CAS call"); + assert_eq!(result, EcstoreConditionalFileUpdate::Mismatch); + let recovered = read_committed(&[disk], 4096) + .await + .expect("read old anchor") + .expect("snapshot"); + assert_eq!(recovered.manifest.sequence, 1); + } + + #[tokio::test] + async fn legacy_import_requires_complete_consistent_replicas() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let bytes = payload("object"); + for (disk, data) in [(&first, &bytes[..bytes.len() - 1]), (&second, bytes.as_slice())] { + EcstoreDiskAPI::write_all( + disk.as_ref(), + RUSTFS_META_BUCKET, + MRF_SCOPED_JOURNAL_PATH, + EcstoreDiskBytes::copy_from_slice(data), + ) + .await + .expect("legacy fixture"); + } + let disks = [first.clone(), second]; + assert!( + matches!(read_recovery_snapshot(&disks, 4096).await.expect("intact legacy replica"), Some(RecoverySnapshot::Legacy(data)) if data == bytes) + ); + EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, payload("different").into()) + .await + .expect("divergent fixture"); + assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict))); + } + + #[tokio::test] + async fn committed_inspection_leaves_payload_and_manifest_unchanged() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let bytes = payload("object"); + commit(&disk, 0, owner, 3, &bytes).await; + assert!(matches!( + read_recovery_snapshot(std::slice::from_ref(&disk), 4096) + .await + .expect("new snapshot"), + Some(RecoverySnapshot::Committed(_)) + )); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]) + .await + .expect("manifest retained") + .as_ref(), + manifest(owner, 3, &bytes) + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]) + .await + .expect("payload retained") + .as_ref(), + bytes + ); + } + + #[tokio::test] + async fn legacy_inspection_rejects_complete_subsets_and_scope_ambiguity() { + let scoped = |set_index| { + let intent = MrfIntent { + bucket: Arc::from("bucket"), + object: Arc::from("a"), + version_id: None, + kind: MrfKind::PartialWrite, + scope: Some(MrfScope { + pool_index: 0, + set_index, + }), + lease: None, + enqueued_at_ms: 1234, + attempts: 0, + }; + let mut bytes = Vec::new(); + assert!(encode_intent(&intent, &mut bytes), "scoped fixture must encode"); + bytes + }; + let mut superset = payload("a"); + superset.extend_from_slice(&payload("b")); + for (case, first_bytes, second_bytes) in [ + ("complete-subset", payload("a"), superset), + ("different-set", scoped(1), scoped(2)), + ("unknown-scope", payload("a"), scoped(1)), + ] { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] { + assert_eq!(decode_journal(bytes).1, 0, "{case}: complete fixture"); + EcstoreDiskAPI::write_all( + disk.as_ref(), + RUSTFS_META_BUCKET, + MRF_SCOPED_JOURNAL_PATH, + EcstoreDiskBytes::copy_from_slice(bytes), + ) + .await + .expect("write legacy replica"); + } + for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] { + assert!( + matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict)), + "{case}: neither replica order proves a latest snapshot" + ); + } + for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] { + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("legacy evidence retained") + .as_ref(), + bytes.as_slice(), + "{case}: inspection must preserve both source replicas" + ); + } + } + } + + #[tokio::test] + async fn oversized_or_corrupt_scoped_snapshot_never_falls_back_to_legacy() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, vec![0; 1025].into()) + .await + .expect("oversized fixture"); + EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload("old").into()) + .await + .expect("legacy fixture"); + assert!(matches!( + read_recovery_snapshot(std::slice::from_ref(&disk), 1024).await, + Err(SnapshotError::TooLarge) + )); + assert!(matches!(read_recovery_snapshot(&[disk], 2048).await, Err(SnapshotError::Corrupt))); + } + + #[tokio::test] + async fn empty_legacy_replica_cannot_erase_records_in_a_torn_replica() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let mut incomplete = payload("durable-object"); + incomplete.extend_from_slice(b"torn"); + EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, Vec::new().into()) + .await + .expect("empty truncated replica"); + EcstoreDiskAPI::write_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, incomplete.clone().into()) + .await + .expect("records and torn tail"); + for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] { + assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Corrupt))); + } + assert_eq!( + EcstoreDiskAPI::read_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("recovery anchor preserved") + .as_ref(), + incomplete + ); + } + + #[tokio::test] + async fn unreadable_commit_record_never_implies_legacy_only() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let legacy = payload("old"); + EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, legacy.clone().into()) + .await + .expect("legacy fixture"); + // Opening a directory as a record either fails at open or at read, + // depending on the platform. Neither outcome proves absence. + std::fs::create_dir(root.path().join("disk").join(RUSTFS_META_BUCKET).join(MANIFEST_PATHS[0])) + .expect("unreadable manifest fixture"); + let recovered = read_recovery_snapshot(std::slice::from_ref(&disk), 4096).await; + assert!( + matches!(recovered, Err(SnapshotError::Disk(_) | SnapshotError::Read(_))), + "must preserve unavailable proof: {recovered:?}" + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH) + .await + .expect("legacy remains") + .as_ref(), + legacy + ); + } +} diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 924193b6e..52c50e042 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -11,6 +11,7 @@ ## Open Items +- `backlog-2263` legacy heal MRF inspection: retained per-record journals remain readable while committed-snapshot ownership and writer activation are staged. Remove legacy import only after all supported direct-upgrade and rollback readers understand committed snapshots and migration tooling confirms that no retained or restorable legacy journal requires it. This does not enable a new writer or change the automatic legacy consumer. - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation. - `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains. - `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on precedence-resolved MinIO PAX metadata; per-entry and cumulative extension limits; a physical-entry limit; cancellation-safe parsing and ownership of large streamed members; fused streams after errors; and compatibility with minio-go streams that omit the two-block terminator. Swift bulk extraction also uses the same fork. Keep the reviewed pin while the Snowball path is prototyped against tar-codec/tar-framing. Remove it only after a released API exposes the effective allowed vendor records, RustFS provides a cancellation-safe handoff for borrowed member payloads, footerless input is accepted solely when authenticated request framing proves EOF immediately after a complete member, the existing resource-limit, cancellation, error-fuse, and real minio-go fixtures pass against the replacement, and Swift no longer depends on the fork. From e2a921bc1608823c8efec955d7463ab8350a8a01 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 21:47:12 +0800 Subject: [PATCH 38/40] fix(storage): harden ODM and scanner publication (#7187) * fix(storage): harden ODM and scanner publication * fix(app): simplify absent SSE configuration matching * test(heal): settle PUT rename tails before disk-wipe fixtures * fix(ecstore): remove duplicate local rename implementation Keep the canonical commit module after concurrent storage changes merged. The control-write and rollback changes are already present there. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ci): satisfy new clippy lints * style(scanner): order merged test imports * fix(scanner): invalidate bucket work after namespace completion * fix(scanner): fence cached snapshots by scan execution --------- Co-authored-by: houseme Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/api/mod.rs | 6 + .../bucket/on_demand_migration/backfill.rs | 149 ++++- .../on_demand_migration/list_through.rs | 30 +- .../src/bucket/on_demand_migration/pull.rs | 67 ++- .../on_demand_migration/source_client.rs | 79 ++- crates/ecstore/src/object_api/types.rs | 3 + crates/ecstore/src/runtime/instance.rs | 112 ++++ .../src/set_disk/core/io_primitives.rs | 318 +++++++--- crates/ecstore/src/set_disk/ops/multipart.rs | 54 +- crates/ecstore/src/set_disk/ops/object.rs | 5 +- crates/ecstore/src/store/bucket.rs | 14 +- crates/ecstore/src/store/init.rs | 542 +++++++++++++++++- crates/ecstore/src/store/mod.rs | 15 +- .../heal_b5_versioned_regression_test.rs | 38 +- crates/heal/tests/storage_api.rs | 1 + crates/scanner/src/data_usage_define.rs | 12 +- crates/scanner/src/data_usage_define/tests.rs | 4 + crates/scanner/src/scanner.rs | 36 +- crates/scanner/src/scanner/activity.rs | 1 - crates/scanner/src/scanner/tests.rs | 173 +++++- crates/scanner/src/scanner_io.rs | 20 +- crates/scanner/src/scanner_io/cache.rs | 42 +- crates/scanner/src/scanner_io/io_cache.rs | 39 +- crates/scanner/src/scanner_io/io_cycle.rs | 10 +- crates/scanner/src/scanner_io/tests.rs | 237 +++++++- crates/scanner/src/storage_api.rs | 3 + .../architecture/scanner-usage-publication.md | 48 +- docs/operations/on-demand-migration.md | 18 +- rustfs/src/app/bucket_list_through.rs | 3 + rustfs/src/app/object/get.rs | 88 ++- rustfs/src/app/object/internal_put.rs | 19 +- .../src/app/object/on_demand_migration_put.rs | 169 ++++++ rustfs/src/app/object/put.rs | 13 +- rustfs/src/app/object/shared.rs | 71 ++- rustfs/src/storage/rpc/node_service.rs | 87 ++- 35 files changed, 2252 insertions(+), 274 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 9e6fd34be..fd66c897a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -562,6 +562,12 @@ pub mod set_disk { pub mod test_util { pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test; pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause}; + + /// Keep a namespace commit pending until the returned owner is dropped. + #[must_use] + pub fn hold_namespace_commit(store: &crate::store::ECStore) -> impl Send + Sync { + store.ctx.begin_namespace_commit() + } } } diff --git a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs index ddcbce8da..3a0ccfd69 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs @@ -25,8 +25,8 @@ //! [`BACKFILL_SAVE_INTERVAL`], and at every page end, with an `If-Match` //! compare-and-set so a concurrent cancel or takeover is never overwritten. //! - The `continuation_token` only advances once every pull queued from the -//! page before it has reported back, so a crash re-lists at most one page -//! (already-present keys are then skipped, never re-pulled). +//! page before it has succeeded. After a failure it stays at that page, +//! so crash recovery cannot skip failed pulls (existing keys are skipped). //! - The owner holds a lease of [`BACKFILL_LEASE`] renewed by every save. The //! recovery loop ([`run_backfill_recovery_loop`]) scans the buckets this //! node has an ODM state for every [`BACKFILL_RECOVERY_INTERVAL`] and takes @@ -367,9 +367,8 @@ pub struct LocalBackfillObject { pub source_etag: Option, } -/// Receiver of one queued pull's report; `None` when the pull was coalesced -/// into one already running. -pub type PullReport = Option>; +/// Shared report of a new or coalesced pull; absent only when not admitted. +pub type PullReport = Option; /// Everything the job needs from its bucket, so the loop can run against a /// mock in unit tests. Production: [`BucketBackfillContext`]. @@ -1191,9 +1190,11 @@ impl Job { } async fn main_loop(&mut self) -> Result<(), Stop> { + let mut cursor = self.checkpoint.continuation_token.clone(); + let failed_at_resume = self.checkpoint.failed; loop { self.check_cancel()?; - let page = self.list_page().await?; + let page = self.list_page(cursor.as_deref()).await?; for object in &page.objects { self.check_cancel()?; self.checkpoint.listed += 1; @@ -1205,10 +1206,13 @@ impl Job { self.drain_ready(); self.tick(false).await?; } - // Only advance the cursor once every pull of this page reported - // back, so a takeover re-lists at most this page. + // A persisted cursor certifies successful work, not just listing + // progress. Keep it at the first failed page for crash recovery. self.drain_all().await?; - self.checkpoint.continuation_token = page.next_continuation_token.clone(); + cursor = page.next_continuation_token; + if self.checkpoint.failed == failed_at_resume { + self.checkpoint.continuation_token = cursor.clone(); + } self.tick(true).await?; if !page.is_truncated { return Ok(()); @@ -1223,7 +1227,7 @@ impl Job { } } - async fn list_page(&mut self) -> Result { + async fn list_page(&mut self, cursor: Option<&str>) -> Result { let mut attempt = 0; loop { while !self.context.source_available() { @@ -1231,7 +1235,7 @@ impl Job { self.tick(false).await?; } let prefix = self.checkpoint.prefix.clone(); - let token = self.checkpoint.continuation_token.clone(); + let token = cursor.map(str::to_string); match self .context .list_page(prefix.as_deref(), token.as_deref(), BACKFILL_LIST_PAGE_SIZE) @@ -1305,9 +1309,10 @@ impl Job { } loop { match self.context.enqueue(key) { - (EnqueueOutcome::Enqueued, report) => { + (EnqueueOutcome::Enqueued | EnqueueOutcome::Coalesced, report) => { self.checkpoint.enqueued += 1; - if let Some(rx) = report { + let rx = report.ok_or(Stop::Unavailable)?; + { let key = key.to_string(); self.outstanding.push(Box::pin(async move { (key, rx.await) })); } @@ -1322,11 +1327,6 @@ impl Job { ); return Ok(()); } - (EnqueueOutcome::Coalesced, _) => { - // Someone else pulls it; its result is not ours to count. - self.checkpoint.enqueued += 1; - return Ok(()); - } (EnqueueOutcome::QueueFull, _) => { // Wait, never drop: one completion frees a slot. if self.outstanding.is_empty() { @@ -1640,6 +1640,7 @@ mod tests { queue_capacity: usize, pending: Mutex)>>, fail_keys: HashSet, + coalesced: bool, auto_complete: AtomicBool, cancel: CancellationToken, config_updated_at: Mutex>, @@ -1667,6 +1668,7 @@ mod tests { queue_capacity: usize::MAX, pending: Mutex::new(Vec::new()), fail_keys: HashSet::new(), + coalesced: false, auto_complete: AtomicBool::new(true), cancel: CancellationToken::new(), config_updated_at: Mutex::new(Some(ts(1_700_000_000))), @@ -1746,7 +1748,12 @@ mod tests { } else { self.pending.lock().push((key.to_string(), tx)); } - (EnqueueOutcome::Enqueued, Some(rx)) + let outcome = if self.coalesced { + EnqueueOutcome::Coalesced + } else { + EnqueueOutcome::Enqueued + }; + (outcome, Some(futures::FutureExt::shared(rx))) } fn cancel_token(&self) -> CancellationToken { @@ -1912,7 +1919,7 @@ mod tests { #[tokio::test] async fn failed_pulls_are_counted_hashed_and_finish_with_failures() { let bucket = "backfill-failed"; - let mut context = MockContext::new(5, 1000); + let mut context = MockContext::new(5, 2); Arc::get_mut(&mut context) .expect("unshared") .fail_keys @@ -1927,12 +1934,52 @@ mod tests { .checkpoint; assert_eq!(cp.state, BackfillState::CompletedWithFailures); assert_eq!((cp.pulled, cp.failed), (4, 1)); + assert_eq!(cp.continuation_token.as_deref(), Some("2"), "retain the first failed page for recovery"); assert_eq!(cp.failed_keys, vec![key_hash("k/00002")]); let last = cp.last_error.expect("last error"); assert_eq!(last.class, "local_write"); assert_eq!(last.key_hash.as_deref(), Some(key_hash("k/00002").as_str())); } + #[tokio::test] + async fn coalesced_pulls_block_the_checkpoint_and_report_failures() { + let bucket = "backfill-coalesced"; + let mut context = MockContext::new(1, 1); + { + let ctx = Arc::get_mut(&mut context).expect("unshared"); + ctx.coalesced = true; + ctx.auto_complete = AtomicBool::new(false); + ctx.fail_keys.insert("k/00000".to_string()); + } + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + runner.start(bucket, BackfillRequest::default()).await.expect("start"); + tokio::time::timeout(Duration::from_secs(10), async { + while context.pending.lock().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("job enqueued"); + assert!(runner.is_running_locally(bucket), "coalescing is not completion"); + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert!(cp.state.is_active()); + assert!(cp.continuation_token.is_none()); + context.complete_pending(); + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::CompletedWithFailures); + assert_eq!((cp.enqueued, cp.pulled, cp.failed), (1, 0, 1)); + assert_eq!(cp.failed_keys, vec![key_hash("k/00000")]); + } + #[tokio::test] async fn listing_failure_marks_the_job_failed_with_the_error_class() { let bucket = "backfill-list-error"; @@ -2145,6 +2192,68 @@ mod tests { assert_eq!(runner.recover_once().await.taken_over, 0, "a finished job is not recovered"); } + #[tokio::test] + async fn recovery_advances_past_historical_failures_but_pins_new_failures() { + let bucket = "backfill-takeover-failed"; + let mut context = MockContext::new(8, 2); + { + let ctx = Arc::get_mut(&mut context).expect("unshared"); + ctx.auto_complete = AtomicBool::new(false); + ctx.fail_keys.insert("k/00004".to_string()); + } + let (_dirs, store, runner) = runner_with("node-b", bucket, Arc::clone(&context)).await; + let crashed_at = OffsetDateTime::now_utc() - Duration::from_secs(300); + let mut crashed = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", crashed_at); + crashed.continuation_token = Some("2".to_string()); + crashed.failed = 1; + crashed.record_failure("local_write", Some("k/00002"), crashed_at); + write_checkpoint(&store, bucket, &crashed, None) + .await + .expect("seed failed page with an expired lease"); + + assert_eq!(runner.recover_once().await.taken_over, 1); + for (page_start, durable_token, failures) in [(2, "2", 1), (4, "4", 1), (6, "4", 2)] { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if context.pending.lock().len() == 2 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("resumed page enqueued before its reports complete"); + assert_eq!( + context.pending.lock().iter().map(|(key, _)| key.clone()).collect::>(), + vec![format!("k/{page_start:05}"), format!("k/{:05}", page_start + 1)] + ); + let cp = read_checkpoint(&store, bucket) + .await + .expect("read persisted page boundary") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.job_id, crashed.job_id); + assert_eq!(cp.owner.as_ref().map(|owner| owner.node.as_str()), Some("node-b")); + assert_eq!(cp.continuation_token.as_deref(), Some(durable_token)); + assert_eq!(cp.failed, failures); + context.complete_pending(); + } + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read completed checkpoint") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::CompletedWithFailures); + assert_eq!((cp.pulled, cp.failed), (5, 2)); + assert_eq!(cp.continuation_token.as_deref(), Some("4")); + assert_eq!(cp.failed_keys, vec![key_hash("k/00002"), key_hash("k/00004")]); + assert_eq!( + context.list_requests.lock().as_slice(), + &[Some("2".to_string()), Some("4".to_string()), Some("6".to_string())] + ); + } + #[tokio::test] async fn recovery_cancels_a_job_whose_config_changed_and_reclaims_own_node_jobs() { let bucket = "backfill-recovery-config"; diff --git a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs index a2ee6fea3..dd5a6a236 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs @@ -37,6 +37,9 @@ pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16; /// listing's own marker, so the decoder needs a positive signal before it /// treats an opaque token as a merged one. const LIST_THROUGH_TOKEN_TAG: &str = "odm-list"; +// Object keys cannot contain NUL (bucket::utils::is_valid_object_prefix), +// so this framing cannot collide with a local key used as an opaque marker. +const LIST_THROUGH_TOKEN_PREFIX: &str = "\0odm-list:"; /// Pages fetched per side per request: the first page, plus at most one refill /// when the first one was mostly consumed by the previous page. Two pages of @@ -91,8 +94,7 @@ pub struct MergePick { } /// The continuation-token envelope. Opaque to clients: it is serialized as -/// JSON and then base64-encoded by the same helper that encodes a plain local -/// marker, so the wire shape is `base64(json)`. +/// framed JSON and then base64-encoded by the same helper as a local marker. /// /// A `null` cursor with `done = false` means "list that side from the start"; /// `done = true` means the side is finished and must not be listed again. @@ -139,7 +141,7 @@ impl ListThroughToken { pub fn encode(&self) -> String { // The envelope is built here from owned strings, so serialization // cannot fail; the fallback keeps the signature infallible. - serde_json::to_string(self).unwrap_or_default() + format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default()) } } @@ -163,21 +165,18 @@ pub enum ListThroughTokenError { /// Classifies an already base64-decoded continuation token. /// -/// Only a JSON object carrying the envelope marker is read as a merged token; +/// Only a framed JSON object is read as a merged token; /// anything else is a local marker, so a bucket that turns `list_through` off /// keeps paginating with the tokens it handed out. A token that *is* an /// envelope but was tampered with (unknown version, unknown field, truncated /// JSON) is an error, never a silent fallback. pub fn decode_continuation_token(decoded: &str) -> Result { - if !decoded.starts_with('{') { - return Ok(ListThroughCursor::Local(decoded.to_string())); - } - let Ok(value) = serde_json::from_str::(decoded) else { - // Not JSON at all: an object key may legitimately start with '{'. + let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else { return Ok(ListThroughCursor::Local(decoded.to_string())); }; + let value = serde_json::from_str::(payload).map_err(|_| ListThroughTokenError::Malformed)?; if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) { - return Ok(ListThroughCursor::Local(decoded.to_string())); + return Err(ListThroughTokenError::Malformed); } match value.get("v").and_then(serde_json::Value::as_u64) { Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => { @@ -1052,9 +1051,9 @@ mod tests { assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed)); let truncated = &encoded[..encoded.len() - 3]; - assert_eq!(decode_continuation_token(truncated), Ok(ListThroughCursor::Local(truncated.to_string()))); + assert_eq!(decode_continuation_token(truncated), Err(ListThroughTokenError::Malformed)); - let no_version = "{\"t\":\"odm-list\"}"; + let no_version = "\0odm-list:{\"t\":\"odm-list\"}"; assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed)); } @@ -1311,6 +1310,13 @@ mod tests { #[test] fn a_plain_local_marker_stays_local() { + for marker in [ + r#"{"t":"odm-list","v":1}"#, + r#"{"t":"odm-list","v":2,"local_done":true}"#, + r#"{"t":"odm-list"}"#, + ] { + assert_eq!(decode_continuation_token(marker), Ok(ListThroughCursor::Local(marker.to_string()))); + } assert_eq!( decode_continuation_token("photos/2024/01.jpg"), Ok(ListThroughCursor::Local("photos/2024/01.jpg".to_string())) diff --git a/crates/ecstore/src/bucket/on_demand_migration/pull.rs b/crates/ecstore/src/bucket/on_demand_migration/pull.rs index 60f7145a2..df3dd6462 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/pull.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/pull.rs @@ -46,10 +46,10 @@ use super::stats::{PullFailureReason, PullPath}; use super::sys::{BucketOdmState, OnDemandMigrationSys, PullError, PullOutcome, PullSlot}; use async_trait::async_trait; use bytes::Bytes; -use futures::{Stream, StreamExt}; +use futures::{FutureExt, Stream, StreamExt, future::Shared}; use parking_lot::Mutex; use rand::RngExt; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::fmt; use std::io; use std::pin::Pin; @@ -133,6 +133,8 @@ pub enum QueuedPullOutcome { Failed(PullError), } +pub type QueuedPullReport = Shared>; + /// Result of [`PullQueue::enqueue`]. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum EnqueueOutcome { @@ -251,6 +253,7 @@ pub struct WriteBackRequest { pub preserve_etag: bool, /// `policy.emit_events`. pub emit_events: bool, + pub respect_delete_marker: bool, /// Source tags to copy (`policy.copy_tags`), `None` to skip. pub tags: Option>, } @@ -266,6 +269,7 @@ impl WriteBackRequest { pulled_at: OffsetDateTime::now_utc(), preserve_etag: config.policy.preserve_etag, emit_events: config.policy.emit_events, + respect_delete_marker: config.policy.respect_local_delete_marker, tags, } } @@ -830,7 +834,7 @@ pub struct PullQueue { bucket: String, tx: mpsc::Sender, /// Keys queued or running; the job removes its key when it ends. - pending: Mutex>, + pending: Mutex>, capacity: usize, cancel: CancellationToken, stats: Arc, @@ -869,7 +873,7 @@ impl PullQueue { let queue = Arc::new(Self { bucket: state.bucket().to_string(), tx, - pending: Mutex::new(HashSet::new()), + pending: Mutex::new(HashMap::new()), capacity, cancel: state.cancel_token(), stats: Arc::clone(state.stats()), @@ -903,29 +907,24 @@ impl PullQueue { self.enqueue_with_report(key, reason).0 } - /// [`Self::enqueue`] that also hands back the job's report channel when - /// a new job was queued (`Coalesced` pulls report to their first - /// requester only). - pub fn enqueue_with_report( - &self, - key: &str, - reason: PullReason, - ) -> (EnqueueOutcome, Option>) { + /// [`Self::enqueue`] with a shared report, including for coalesced pulls. + pub fn enqueue_with_report(&self, key: &str, reason: PullReason) -> (EnqueueOutcome, Option) { if self.cancel.is_cancelled() { return (EnqueueOutcome::Unavailable, None); } let mut pending = self.pending.lock(); - if pending.contains(key) { - return (EnqueueOutcome::Coalesced, None); + if let Some(report) = pending.get(key) { + return (EnqueueOutcome::Coalesced, Some(report.clone())); } let (report_tx, report_rx) = oneshot::channel(); + let report_rx = report_rx.shared(); match self.tx.try_send(PullJob { key: key.to_string(), reason, report: Some(report_tx), }) { Ok(()) => { - pending.insert(key.to_string()); + pending.insert(key.to_string(), report_rx.clone()); (EnqueueOutcome::Enqueued, Some(report_rx)) } Err(TrySendError::Full(_)) => { @@ -1072,7 +1071,7 @@ impl BucketOdmState { self: &Arc, key: &str, reason: PullReason, - ) -> (EnqueueOutcome, Option>) { + ) -> (EnqueueOutcome, Option) { match self.pull_queue() { Some(queue) => queue.enqueue_with_report(key, reason), None => (EnqueueOutcome::Unavailable, None), @@ -1399,13 +1398,21 @@ mod tests { assert_eq!(queue.capacity(), 1024); let mut outcomes = HashMap::new(); + let mut shared_report = None; for _ in 0..100 { - *outcomes.entry(queue.enqueue("a", PullReason::RangeGet)).or_insert(0) += 1; + let (outcome, report) = queue.enqueue_with_report("a", PullReason::RangeGet); + *outcomes.entry(outcome).or_insert(0) += 1; + shared_report = report; } assert_eq!(outcomes.get(&EnqueueOutcome::Enqueued), Some(&1)); assert_eq!(outcomes.get(&EnqueueOutcome::Coalesced), Some(&99)); assert_eq!(queue.pending_keys(), 1); + assert_eq!( + shared_report.expect("coalesced report").await, + Ok(QueuedPullOutcome::Stored { size: 1000 }) + ); + wait_until("first pull to finish", || queue.pending_keys() == 0).await; assert_eq!(source.head_calls.load(Ordering::SeqCst), 1); assert_eq!(source.get_calls.load(Ordering::SeqCst), 1); @@ -1438,6 +1445,23 @@ mod tests { assert_eq!(queue.enqueue("a", PullReason::RangeGet), EnqueueOutcome::Unavailable); } + #[tokio::test] + async fn coalesced_enqueues_share_failure_reports() { + let sys = OnDemandMigrationSys::new(); + let state = enabled_state(&sys, &config()).await; + let source = MockSource::with_object("missing", 1000, BodyKind::Bytes(body_bytes(1000))); + let queue = PullQueue::start(Arc::clone(&state), source, Arc::new(MockWriteBack::default())); + let (first, first_report) = queue.enqueue_with_report("absent", PullReason::RangeGet); + let (second, second_report) = queue.enqueue_with_report("absent", PullReason::Backfill); + assert_eq!(first, EnqueueOutcome::Enqueued); + assert_eq!(second, EnqueueOutcome::Coalesced); + let (first, second) = tokio::join!(first_report.expect("leader report"), second_report.expect("coalesced report")); + assert_eq!(first, second); + assert!(matches!(first, Ok(QueuedPullOutcome::Failed(_)))); + sys.remove(BUCKET); + queue.wait_until_stopped().await; + } + #[tokio::test] async fn queue_full_is_reported_and_cancel_drains_without_leaking_tasks() { let sys = OnDemandMigrationSys::new(); @@ -1467,7 +1491,8 @@ mod tests { wait_until("dispatcher to wait for a slot", || state.stats().queue_depth() == 1).await; assert_eq!(queue.enqueue("c", PullReason::LargeObject), EnqueueOutcome::Enqueued); assert_eq!(queue.enqueue("d", PullReason::LargeObject), EnqueueOutcome::QueueFull); - assert_eq!(queue.enqueue("c", PullReason::LargeObject), EnqueueOutcome::Coalesced); + let (coalesced, canceled_report) = queue.enqueue_with_report("c", PullReason::LargeObject); + assert_eq!(coalesced, EnqueueOutcome::Coalesced); assert_eq!(queue.pending_keys(), 3); assert_eq!(failures(&state).get("queue_full"), Some(&1)); assert!(!queue.is_stopped()); @@ -1477,6 +1502,12 @@ mod tests { .await .expect("dispatcher and in-flight job must exit after cancel"); assert!(queue.is_stopped()); + assert!( + tokio::time::timeout(Duration::from_secs(5), canceled_report.expect("coalesced cancellation report")) + .await + .expect("cancellation closes the report") + .is_err() + ); assert_eq!(queue.pending_keys(), 0); assert_eq!(state.inflight_keys(), 0); assert_eq!(state.stats().inflight_pulls(), 0); diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs index f06301036..a97cf5a0a 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs @@ -153,8 +153,8 @@ pub struct SourceClientSpec { /// Wire requests one logical source call may cost. The pull pipeline and /// the backfill job own the retry budget (`pull.rs` `PULL_MAX_RETRIES`, /// `backfill.rs` `LIST_MAX_RETRIES`) and the breaker counts logical calls, - /// so ODM declares [`RemoteS3RetryPolicy::Disabled`] and keeps one counted - /// failure equal to one request against a struggling source. + /// so ODM declares [`RemoteS3RetryPolicy::Disabled`]. An ambiguous HEAD + /// 404 additionally probes the bucket before declaring a key absent. pub retry: RemoteS3RetryPolicy, /// Bytes per second the pull pipeline may consume from this source; /// `None` means unlimited. Enforced by the consumer, not by this client. @@ -262,7 +262,7 @@ const THROTTLE_CODES: &[&str] = &[ "TooManyRequests", "RequestThrottled", ]; -const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "NotFound", "NoSuchBucket", "NoSuchVersion"]; +const NOT_FOUND_CODES: &[&str] = &["NoSuchKey"]; const ACCESS_DENIED_CODES: &[&str] = &[ "AccessDenied", "InvalidAccessKeyId", @@ -285,7 +285,6 @@ fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceEr } } match status { - 404 => SourceError::NotFound, 401 | 403 => SourceError::AccessDenied, 429 | 503 => SourceError::Throttled, 500..=599 => SourceError::ServerError(status), @@ -631,8 +630,7 @@ impl SourceClient { } /// `config` must come from [`SourceClientSpec::endpoint_spec`], which is - /// where the retry policy that keeps one logical call equal to one wire - /// request is declared. + /// where the policy disabling SDK-level retries is declared. fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self { let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build()); Self { @@ -754,15 +752,16 @@ impl SourceClient { #[async_trait::async_trait] impl SourceBackend for S3SourceBackend { async fn head(&self, key: &str) -> Result { - let output = self - .client - .head_object() - .bucket(&self.bucket) - .key(key) - .send() - .await - .map_err(classify_sdk_error)?; - source_head_from_head_output(output) + match self.client.head_object().bucket(&self.bucket).key(key).send().await { + Ok(output) => source_head_from_head_output(output), + Err(err) if err.raw_response().is_some_and(|response| response.status().as_u16() == 404) => { + // HEAD has no error body: a missing bucket must not poison + // the per-key negative cache as though only the key was absent. + self.probe().await?; + Err(SourceError::NotFound) + } + Err(err) => Err(classify_sdk_error(err)), + } } /// Streams the object; `range` is passed through as an HTTP `Range` @@ -809,8 +808,8 @@ impl SourceBackend for S3SourceBackend { .contents .unwrap_or_default() .into_iter() - .filter_map(s3_source_object) - .collect(); + .map(s3_source_object) + .collect::, _>>()?; let common_prefixes = output .common_prefixes .unwrap_or_default() @@ -849,14 +848,20 @@ impl SourceBackend for S3SourceBackend { } } -fn s3_source_object(object: SdkObject) -> Option { - let key = object.key?; +fn s3_source_object(object: SdkObject) -> Result { + let key = object + .key + .ok_or_else(|| SourceError::Other("source listing object has no key".to_string()))?; + let size = object + .size + .and_then(|size| u64::try_from(size).ok()) + .ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?; let etag = normalize_etag(object.e_tag); let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag); - Some(SourceObject { + Ok(SourceObject { key, etag, - size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0), + size, last_modified: system_time(object.last_modified), storage_class: object.storage_class.map(|class| class.as_str().to_string()), is_multipart_etag, @@ -1489,7 +1494,10 @@ mod tests { #[tokio::test] async fn source_error_classification_covers_every_class() { let cases: Vec<(Scripted, &str, bool)> = vec![ - (status(404, ""), "not_found", false), + (status(404, ""), "other", false), + (status(404, "NoSuchKey"), "not_found", false), + (status(404, "NoSuchBucket"), "other", false), + (status(404, "NoSuchVersion"), "other", false), (status(403, ACCESS_DENIED_BODY), "access_denied", false), (status(401, ""), "access_denied", false), (status(429, ""), "throttled", true), @@ -1512,14 +1520,35 @@ mod tests { } } - // HEAD carries no error body, so the classification must work from the - // status alone as well. - let (client, _) = scripted_client(&spec(None), vec![status(404, "")]).await; + let (client, requests) = scripted_client(&spec(None), vec![status(404, ""), status(200, "")]).await; assert!(matches!(client.head_object("missing").await, Err(SourceError::NotFound))); + assert_eq!(recorded(&requests).len(), 2, "ambiguous HEAD 404 must check the bucket"); + let (client, _) = scripted_client(&spec(None), vec![status(404, ""), status(404, "")]).await; + assert!(matches!(client.head_object("missing").await, Err(SourceError::Other(_)))); + let (client, _) = scripted_client(&spec(None), vec![status(404, ""), status(403, "")]).await; + assert!(matches!(client.head_object("missing").await, Err(SourceError::AccessDenied))); let (client, _) = scripted_client(&spec(None), vec![status(403, "")]).await; assert!(matches!(client.head_object("secret").await, Err(SourceError::AccessDenied))); } + #[test] + fn source_listing_rejects_missing_and_negative_sizes() { + for size in [None, Some(-1)] { + let object = SdkObject::builder().key("key").set_size(size).build(); + assert!(matches!(s3_source_object(object), Err(SourceError::Other(_)))); + } + assert!(matches!( + s3_source_object(SdkObject::builder().size(0).build()), + Err(SourceError::Other(_)) + )); + assert_eq!( + s3_source_object(SdkObject::builder().key("empty").size(0).build()) + .expect("empty object") + .size, + 0 + ); + } + #[tokio::test] async fn source_client_debug_redacts_credentials() { let (client, _) = scripted_client(&spec(Some("data/")), Vec::new()).await; diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 701596cbf..81e61ca82 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -956,6 +956,9 @@ pub struct ObjectOptions { pub preserve_etag: Option, pub metadata_chg: bool, pub http_preconditions: Option, + /// Internal create-only writes may also preserve an acknowledged deletion. + /// Evaluated with `http_preconditions` under the namespace commit lock. + pub preserve_delete_marker: bool, pub delete_replication: Option, pub delete_replication_config_snapshot: Option>, diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index ad65354ed..fd4007698 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -78,6 +78,21 @@ pub(crate) struct ScannerPublicationLeaseEntry { pub(crate) _operation_guard: OwnedRwLockReadGuard<()>, } +pub(crate) struct NamespaceCommitGuard { + ctx: Arc, + counted: bool, +} + +impl Drop for NamespaceCommitGuard { + fn drop(&mut self) { + if self.counted { + // Publish the new generation before a zero-pending publication probe. + self.ctx.advance_namespace_commit_generation(); + self.ctx.namespace_commits.fetch_sub(1, Ordering::AcqRel); + } + } +} + /// Runtime state owned by a single `ECStore` instance. /// /// This is intentionally minimal in the first migration slice; subsequent @@ -209,9 +224,13 @@ pub struct InstanceContext { /// Last storage-owned movement snapshot observed under the operation /// gate. SetDisks cache writers fail closed until ECStore refreshes it. scanner_publication_state: AtomicU8, + namespace_commits: AtomicU64, + namespace_commit_generation: AtomicU64, /// Resolves object-encryption material at the application boundary. object_encryption_resolver: OnceLock>, tier_delete_journal_recovery_stores: std::sync::Mutex>, + #[cfg(test)] + suppress_tier_delete_journal_recovery: bool, transition_transaction_recovery_stores: std::sync::Mutex>, tier_delete_journal_recovery_wakeup: tokio::sync::Notify, } @@ -256,8 +275,12 @@ impl InstanceContext { data_movement_generation_exhausted: AtomicBool::new(false), data_movement_generation_notify: Arc::new(Notify::new()), scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN), + namespace_commits: AtomicU64::new(0), + namespace_commit_generation: AtomicU64::new(0), object_encryption_resolver: OnceLock::new(), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), + #[cfg(test)] + suppress_tier_delete_journal_recovery: false, transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(), } @@ -385,6 +408,36 @@ impl InstanceContext { && self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED } + pub(crate) fn begin_namespace_commit(self: &Arc) -> Arc { + let counted = self + .namespace_commits + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1)) + .is_ok(); + if counted { + self.advance_namespace_commit_generation(); + } else { + self.namespace_commit_generation.store(u64::MAX, Ordering::Release); + } + Arc::new(NamespaceCommitGuard { + ctx: Arc::clone(self), + counted, + }) + } + + fn advance_namespace_commit_generation(&self) { + let _ = self + .namespace_commit_generation + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| Some(generation.saturating_add(1))); + } + + pub(crate) fn namespace_commit_generation(&self) -> u64 { + self.namespace_commit_generation.load(Ordering::Acquire) + } + + pub(crate) fn namespace_commits_pending(&self) -> bool { + self.namespace_commits.load(Ordering::Acquire) != 0 || self.namespace_commit_generation() == u64::MAX + } + pub(crate) fn set_scanner_publication_state(&self, blocked: bool) { self.scanner_publication_state.store( if blocked { @@ -640,12 +693,21 @@ impl InstanceContext { } pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool { + #[cfg(test)] + if self.suppress_tier_delete_journal_recovery { + return false; + } self.tier_delete_journal_recovery_stores .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .insert(store_id) } + #[cfg(test)] + pub(crate) fn suppress_tier_delete_journal_recovery_for_test(&mut self) { + self.suppress_tier_delete_journal_recovery = true; + } + pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool { self.transition_transaction_recovery_stores .lock() @@ -756,6 +818,50 @@ pub fn bootstrap_ctx() -> Arc { mod tests { use super::*; + #[test] + fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() { + let first = Arc::new(InstanceContext::new()); + let other = Arc::new(InstanceContext::new()); + first.set_scanner_publication_state(false); + other.set_scanner_publication_state(false); + assert!(first.scanner_publication_state_allowed()); + let one = first.begin_namespace_commit(); + let shared_owner = Arc::clone(&one); + let two = first.begin_namespace_commit(); + assert!(first.namespace_commits_pending()); + assert!(first.scanner_publication_state_allowed(), "pending writes must not block scan admission"); + assert_eq!(first.namespace_commit_generation(), 2); + assert!(!other.namespace_commits_pending()); + assert_eq!(other.namespace_commit_generation(), 0); + assert!(other.scanner_publication_state_allowed()); + drop(one); + assert_eq!(first.namespace_commit_generation(), 2); + drop(shared_owner); + assert!(first.namespace_commits_pending()); + assert_eq!(first.namespace_commit_generation(), 3); + drop(two); + assert!(!first.namespace_commits_pending()); + assert_eq!(first.namespace_commit_generation(), 4); + assert!(first.scanner_publication_state_allowed()); + } + + #[test] + fn namespace_commit_counter_exhaustion_keeps_publication_blocked() { + for (count, generation) in [(0, u64::MAX - 1), (u64::MAX, 0)] { + let ctx = Arc::new(InstanceContext::new()); + ctx.set_scanner_publication_state(false); + ctx.namespace_commits.store(count, Ordering::Release); + ctx.namespace_commit_generation.store(generation, Ordering::Release); + let guard = ctx.begin_namespace_commit(); + assert!(ctx.namespace_commits_pending()); + assert_eq!(ctx.namespace_commit_generation(), u64::MAX); + drop(guard); + assert!(ctx.namespace_commits_pending()); + assert_eq!(ctx.namespace_commit_generation(), u64::MAX); + assert_eq!(ctx.namespace_commits.load(Ordering::Acquire), count); + } + } + // The SetupType inputs must derive the exact (is_erasure, // is_dist_erasure, is_erasure_sd) triples that the original three // process-global erasure bools produced via update_erasure_type(). @@ -1073,6 +1179,12 @@ mod tests { assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a)); assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b)); assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a)); + + let mut manual_ctx = InstanceContext::new(); + manual_ctx.suppress_tier_delete_journal_recovery_for_test(); + assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_a)); + assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_b)); + assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_b)); } #[test] diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 18b04c781..14da6fc97 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3558,6 +3558,11 @@ impl RenameRollbackReceipt { } } +struct RenameRollbackOwnership { + receipt: Option, + namespace_commit_guard: Option>, +} + async fn inspect_incomplete_rename_rollback( disks: &[Option], bucket: &str, @@ -3604,8 +3609,12 @@ async fn rollback_failed_rename( dispatch_states: &[RenameDispatchState], rollback_dirs: &[Option], dst: (&str, &str), - receipt: Option, + ownership: RenameRollbackOwnership, ) { + let RenameRollbackOwnership { + receipt, + namespace_commit_guard, + } = ownership; let owned_disks = disks.to_vec(); let owned_errs = errs.to_vec(); let owned_dispatch_states = dispatch_states.to_vec(); @@ -3651,7 +3660,9 @@ async fn rollback_failed_rename( let fi = std::mem::take(&mut file_infos[disk_index]); let bucket = bucket.to_string(); let object = object.to_string(); + let disk_namespace_commit_guard = namespace_commit_guard.clone(); let task = tokio::spawn(async move { + let _namespace_commit_guard = disk_namespace_commit_guard; #[allow(clippy::let_unit_value)] let _task_guard = SetDisks::rename_fanout_task_guard(&object); SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await; @@ -3672,6 +3683,9 @@ async fn rollback_failed_rename( }); tasks.push(async move { (disk_index, task.await) }); } + #[cfg(test)] + rollback_fault_injection::after_undo_dispatch(object); + let _namespace_commit_guard = namespace_commit_guard; for (disk_index, result) in join_all(tasks).await { outcomes[disk_index].outcome = rename_rollback_task_outcome(result); } @@ -3778,6 +3792,7 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> { write_quorum: usize, scanner_publication_lease_tokens: Option<&'a HashMap>, scanner_publication_commit_scope: Option, + namespace_commit_guard: Option>, rollback_receipt: Option, } @@ -3790,6 +3805,7 @@ impl<'a> RenameDataFenceOptions<'a> { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope: None, + namespace_commit_guard: None, rollback_receipt: None, } } @@ -3806,6 +3822,14 @@ impl<'a> RenameDataFenceOptions<'a> { self.scanner_publication_commit_scope = scanner_publication_commit_scope; self } + + pub(in crate::set_disk) fn with_namespace_commit_guard( + mut self, + namespace_commit_guard: Option>, + ) -> Self { + self.namespace_commit_guard = namespace_commit_guard; + self + } } #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] @@ -4164,6 +4188,7 @@ impl SetDisks { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope: _scanner_publication_commit_scope, + namespace_commit_guard, rollback_receipt, } = fence_options; if let Some(file_info) = disks @@ -4210,7 +4235,9 @@ impl SetDisks { let dst_object = fanout_dst_object.clone(); let file_info = file_info.clone(); let successful_rename_completion_rank = successful_rename_completion_rank.clone(); + let namespace_commit_guard = namespace_commit_guard.clone(); tasks.spawn(async move { + let _namespace_commit_guard = namespace_commit_guard; let mut dispatch_state = RenameDispatchState::NotDispatched; let result = std::panic::AssertUnwindSafe(async { #[allow(clippy::let_unit_value)] @@ -4372,7 +4399,10 @@ impl SetDisks { &dispatch_states, &data_dirs, (&fanout_dst_bucket, &fanout_dst_object), - rollback_receipt, + RenameRollbackOwnership { + receipt: rollback_receipt, + namespace_commit_guard, + }, ) .await; if let Some(commit_tx) = commit_tx.take() { @@ -4528,6 +4558,7 @@ impl SetDisks { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope, + namespace_commit_guard, rollback_receipt, } = fence_options; if let Some(file_info) = disks @@ -4561,6 +4592,7 @@ impl SetDisks { let fanout_dst_bucket = dst_bucket.clone(); let fanout_dst_object = dst_object.clone(); let fanout_publication_scope = scanner_publication_commit_scope.clone(); + let fanout_namespace_commit_guard = namespace_commit_guard.clone(); // Keep one coordinator task so a cancelled caller cannot drop partially // completed disk mutations. Per-disk futures stay ordered in `join_all`, // preserving slot-indexed quorum and convergence accounting without a @@ -4569,6 +4601,7 @@ impl SetDisks { // Keep the storage-owned movement permit attached to the actual // fan-out owner, even if the caller future is cancelled. let _fanout_publication_scope = fanout_publication_scope; + let _namespace_commit_guard = fanout_namespace_commit_guard; let successful_rename_completion_rank = rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0))); let futures = fanout_disks @@ -4790,7 +4823,10 @@ impl SetDisks { &dispatch_states, &data_dirs, (&dst_bucket, &dst_object), - rollback_receipt, + RenameRollbackOwnership { + receipt: rollback_receipt, + namespace_commit_guard, + }, ) .await; return Err(ret_err); @@ -6503,9 +6539,9 @@ impl SetDisks { match oi { Ok(oi) => { // Ordinary writes may proceed past a top-level delete marker; - // data movement must not replace an acknowledged deletion. + // data movement and guarded internal writes must preserve it. if oi.delete_marker { - return opts.data_movement.then_some(StorageError::PreconditionFailed); + return (opts.data_movement || opts.preserve_delete_marker).then_some(StorageError::PreconditionFailed); } let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned); let if_match = http_preconditions.if_match_value().map(str::to_owned); @@ -6754,6 +6790,7 @@ pub(in crate::set_disk) mod rollback_fault_injection { VolumeNotFoundAfterRename, PanicAfterRename, CoordinatorPanic, + RollbackCoordinatorPanic, } fn registry() -> &'static Mutex> { @@ -6816,6 +6853,17 @@ pub(in crate::set_disk) mod rollback_fault_injection { panic!("injected rename coordinator panic"); } } + + pub(super) fn after_undo_dispatch(object: &str) { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + if matches!(fault, Some((_, Fault::RollbackCoordinatorPanic))) { + panic!("injected rollback coordinator panic"); + } + } } /// Test-only per-disk call counters for the metadata fan-out (backlog#1325, @@ -6977,7 +7025,7 @@ pub(crate) mod rename_fanout_barrier { use tokio::sync::Notify; pub use super::rename_fanout_barrier_phase::{ - CLEANUP as PHASE_CLEANUP, READ_VERSION as PHASE_READ_VERSION, RENAME as PHASE_RENAME, + CLEANUP as PHASE_CLEANUP, READ_VERSION as PHASE_READ_VERSION, RENAME as PHASE_RENAME, ROLLBACK as PHASE_ROLLBACK, }; /// One armed barrier: the fan-out task matching `(disk_index, phase)` pauses. @@ -10814,79 +10862,177 @@ mod tests { #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() { - for cancel_caller in [false, true] { - let bucket = "rename-rollback-barrier"; - let object = if cancel_caller { - "rollback-barrier-cancelled" - } else { - "rollback-barrier-object" - }; - let (dirs, disks) = call_counter_local_disks(bucket, 4).await; - prepare_rename_source_dirs(&dirs, &disks, "source").await; - let mut old = metadata_test_fileinfo(object); - old.mod_time = Some(OffsetDateTime::now_utc()); - old.data = Some(Bytes::from_static(b"old-inline-body")); - old.set_inline_data(); - old.metadata.insert("etag".to_string(), "old-etag".to_string()); - for disk in disks.iter().flatten() { - disk.write_metadata(bucket, bucket, object, old.clone()) - .await - .expect("old metadata should be staged"); - } - let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); - let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); - let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); - let receipt = RenameRollbackReceipt::default(); - let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( - &disks, - (RUSTFS_META_TMP_BUCKET, "source"), - rename_commit_fileinfos(object, 4, "new-etag"), - (bucket, object), - false, - RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), - )); - tokio::time::timeout(BARRIER_PAUSE_GUARD, async { - tokio::select! { - () = barrier.wait_until_paused() => {} - _ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"), + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for (allow_early_ack, cancel_caller, object) in [ + (false, false, "rollback-barrier-object"), + (false, true, "rollback-barrier-cancelled"), + (true, false, "rollback-barrier-early-object"), + (true, true, "rollback-barrier-early-cancelled"), + ] { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let bucket = "rename-rollback-barrier"; + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let mut old = metadata_test_fileinfo(object); + old.mod_time = Some(OffsetDateTime::now_utc()); + old.data = Some(Bytes::from_static(b"old-inline-body")); + old.set_inline_data(); + old.metadata.insert("etag".to_string(), "old-etag".to_string()); + for disk in disks.iter().flatten() { + disk.write_metadata(bucket, bucket, object, old.clone()) + .await + .expect("old metadata should be staged"); } - }) - .await - .expect("undo must reach its disk barrier"); - assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success"); - if cancel_caller { - drop(rename); + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); + let receipt = RenameRollbackReceipt::default(); + let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(object, 4, "new-etag"), + (bucket, object), + allow_early_ack, + RenameDataFenceOptions::new(3, None) + .with_rollback_receipt(receipt.clone()) + .with_namespace_commit_guard(Some(ctx.begin_namespace_commit())), + )); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + tokio::select! { + () = barrier.wait_until_paused() => {} + _ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"), + } + }) + .await + .expect("undo must reach its disk barrier"); + assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success"); + assert!(ctx.namespace_commits_pending()); + assert_eq!(ctx.namespace_commit_generation(), 1); + if cancel_caller { + drop(rename); + assert!(ctx.namespace_commits_pending(), "caller cancellation must not retire pending undo work"); + assert_eq!(ctx.namespace_commit_generation(), 1); + barrier.release(); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + while receipt.0.get().is_none() || ctx.namespace_commits_pending() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled caller must not cancel rollback accounting"); + } else { + barrier.release(); + assert!(rename.await.is_err()); + } + assert!( + !ctx.namespace_commits_pending(), + "the completed rollback must release its namespace ownership" + ); + assert_eq!(ctx.namespace_commit_generation(), 2); + assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt"); + for dir in dirs.iter().skip(1) { + let reopened = reopen_local_disk(dir).await; + let restored = reopened + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("old version must remain readable after caller cancellation"); + assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice())); + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_children_keep_namespace_ownership_after_coordinator_panic() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for (allow_early_ack, object) in [ + (false, "rollback-coordinator-panic"), + (true, "rollback-coordinator-panic-early"), + ] { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let bucket = "rename-rollback-coordinator-panic"; + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let mut old = metadata_test_fileinfo(object); + old.mod_time = Some(OffsetDateTime::now_utc()); + old.data = Some(Bytes::from_static(b"old-inline-body")); + old.set_inline_data(); + old.metadata.insert("etag".to_string(), "old-etag".to_string()); + for disk in disks.iter().flatten() { + disk.write_metadata(bucket, bucket, object, old.clone()) + .await + .expect("old metadata should be staged"); + } + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _rollback_fault = + rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::RollbackCoordinatorPanic); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); + let receipt = RenameRollbackReceipt::default(); + let result = tokio::time::timeout( + BARRIER_PAUSE_GUARD, + SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(object, 4, "new-etag"), + (bucket, object), + allow_early_ack, + RenameDataFenceOptions::new(3, None) + .with_rollback_receipt(receipt.clone()) + .with_namespace_commit_guard(Some(ctx.begin_namespace_commit())), + ), + ) + .await + .expect("coordinator failure must return without waiting for detached undo tasks"); + assert!(result.is_err()); + tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused()) + .await + .expect("detached undo must reach its disk barrier"); + assert!( + receipt.is_incomplete(), + "coordinator failure must preserve indeterminate recovery evidence" + ); + assert!(ctx.namespace_commits_pending(), "the paused child must retain namespace ownership"); + assert_eq!(ctx.namespace_commit_generation(), 1); barrier.release(); tokio::time::timeout(BARRIER_PAUSE_GUARD, async { - while receipt.0.get().is_none() { + while ctx.namespace_commits_pending() { tokio::task::yield_now().await; } }) .await - .expect("cancelled caller must not cancel rollback accounting"); - } else { - barrier.release(); - assert!(rename.await.is_err()); + .expect("completed undo children must release their namespace ownership"); + assert_eq!(ctx.namespace_commit_generation(), 2); + for dir in &dirs { + let reopened = reopen_local_disk(dir).await; + let restored = reopened + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("old version must remain readable after rollback coordinator failure"); + assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice())); + } } - assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt"); - for dir in dirs.iter().skip(1) { - let reopened = reopen_local_disk(dir).await; - let restored = reopened - .read_version( - "", - bucket, - object, - "", - &ReadOptions { - read_data: true, - ..Default::default() - }, - ) - .await - .expect("old version must remain readable after caller cancellation"); - assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice())); - } - } + }) + .await; } #[tokio::test] @@ -11001,9 +11147,35 @@ mod tests { let mut file_infos = rename_commit_fileinfos(object, DISKS, "fresh-rollback-etag"); file_infos[3] = FileInfo::default(); - SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 4) + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + ctx.set_scanner_publication_state(false); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_ROLLBACK); + let rename = SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + file_infos, + (bucket, object), + false, + RenameDataFenceOptions::new(4, None).with_namespace_commit_guard(Some(ctx.begin_namespace_commit())), + ); + let control = async { + barrier.wait_until_paused().await; + assert!(ctx.namespace_commits_pending(), "rollback must retain namespace publication ownership"); + assert!(ctx.scanner_publication_state_allowed(), "rollback must not disable namespace walks"); + assert_eq!(ctx.namespace_commit_generation(), 1); + barrier.release(); + }; + let (result, ()) = tokio::time::timeout(BARRIER_PAUSE_GUARD, async { tokio::join!(rename, control) }) .await - .expect_err("three successful disks must fail a strict write quorum of four"); + .expect("rename rollback must reach its barrier and finish after release"); + assert_eq!( + result.err(), + Some(DiskError::ErasureWriteQuorum), + "three successful disks must fail a strict write quorum of four" + ); + assert!(!ctx.namespace_commits_pending()); + assert!(ctx.scanner_publication_state_allowed()); + assert_eq!(ctx.namespace_commit_generation(), 2); for (idx, dir) in dirs.iter().enumerate() { let reopened = reopen_local_disk(dir).await; diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index aaf6204b5..4aefdb04c 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -4050,6 +4050,7 @@ mod tests { let _ = drain_global_dirty_scopes(); let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let rename_tasks = rename_fanout_barrier::observe_tasks(object); let complete_store = Arc::clone(&set_disks); let mut complete = tokio::spawn(async move { let mut opts = ObjectOptions::default(); @@ -4061,16 +4062,6 @@ mod tests { tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused()) .await .expect("multipart completion should pause one tail disk during rename"); - assert!( - tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(), - "multipart completion must not publish success while a tail rename is still paused" - ); - - let initial = drain_global_dirty_scopes().into_iter().collect::>(); - assert!( - initial.is_empty(), - "capacity must not be marked as committed before the full multipart rename finishes" - ); let abort_store = Arc::clone(&set_disks); let abort = tokio::spawn(async move { @@ -4079,21 +4070,46 @@ mod tests { .await }); signaling.wait_for_attempts(2).await; - assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard"); - let retained_staging = futures::future::join_all( - disk_stores - .iter() - .map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)), - ) + // A paused rename does not establish that the other disks reached quorum. + let retained_staging = tokio::time::timeout(Duration::from_secs(30), async { + loop { + let mut retained = 0; + for result in futures::future::join_all( + disk_stores + .iter() + .map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)), + ) + .await + { + match result { + Ok(_) => retained += 1, + Err(DiskError::FileNotFound) => {} + Err(error) => panic!("staged rename source lookup failed: {error}"), + } + } + if retained <= 1 && rename_tasks.running() == 1 { + break retained; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) .await - .into_iter() - .filter(|result| result.is_ok()) - .count(); + .expect("unpaused multipart renames should finish before the tail is released"); assert_eq!( retained_staging, 1, "only the paused tail disk should still retain the multipart rename source" ); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(), + "multipart completion must not publish success while a tail rename is still paused" + ); + let initial = drain_global_dirty_scopes().into_iter().collect::>(); + assert!( + initial.is_empty(), + "capacity must not be marked as committed before the full multipart rename finishes" + ); + assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard"); signaling.set_target(rustfs_lock::ObjectKey::new(bucket, object)); let object_attempt = signaling.attempts.load(Ordering::Acquire) + 1; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index c76daf707..3e09515d2 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -4459,7 +4459,10 @@ impl SetDisks { commit_scanner_publication_lease_tokens.as_ref(), ) .with_publication_scope(commit_scanner_publication_scope.clone()) - .with_rollback_receipt(commit_rollback_receipt.clone()), + .with_rollback_receipt(commit_rollback_receipt.clone()) + .with_namespace_commit_guard( + (!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()), + ), ) .await; if let Some(scope) = commit_scanner_publication_scope.as_ref() { diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 036585d0e..210118cda 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -1059,6 +1059,7 @@ mod tests { use crate::storage_api_contracts::{ bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp}, list::ListOperations as _, + namespace::NamespaceLocking as _, object::{ObjectIO as _, ObjectOperations as _}, }; use crate::store::{ECStore, init_local_disks_with_instance_ctx}; @@ -1486,10 +1487,19 @@ mod tests { .put_object(bucket, object, &mut reader, &ObjectOptions::default()) .await .expect("object should be written"); + let lock = ecstore.pools[0].disk_set[0] + .new_ns_lock(bucket, object) + .await + .expect("fixture namespace lock should be created"); + drop( + lock.get_write_lock(Duration::from_secs(30)) + .await + .expect("fixture rename tail should finish before checking its generation"), + ); assert_eq!( ecstore.scanner_namespace_mutation_generation(), - generation_before_put.saturating_add(1), - "successful object creation should advance scanner namespace activity" + generation_before_put.saturating_add(3), + "successful object creation must observe the logical mutation and both fanout boundaries" ); ecstore .get_object_info(bucket, object, &ObjectOptions::default()) diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index d0b426b68..63d2b5fdd 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -787,6 +787,12 @@ impl ECStore { pub fn single_pool(&self) -> bool { self.pools.len() == 1 } + + /// The set-local create-only check is atomic only when every object + /// mutation uses that same, enabled namespace lock domain. + pub fn supports_atomic_create_only_write_back(&self) -> bool { + !self.ctx.lock_manager().is_disabled() && self.pools.len() == 1 && self.pools[0].disk_set.len() == 1 + } } #[cfg(test)] @@ -2127,7 +2133,7 @@ mod tests { .iter() .map(|&drives_per_set| (1, drives_per_set)) .collect::>(); - build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown).await + build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown, None).await } async fn build_isolated_test_store_with_layout( @@ -2135,6 +2141,7 @@ mod tests { cmd_line: &str, pool_layouts: &[(usize, usize)], shutdown: CancellationToken, + instance_ctx: Option>, ) -> ( Arc, Arc, @@ -2167,7 +2174,7 @@ mod tests { let endpoint_pools = EndpointServerPools(pools); crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test(); - let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let instance_ctx = instance_ctx.unwrap_or_else(|| Arc::new(crate::runtime::instance::InstanceContext::new())); crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) .await .expect("register local disks into the fresh context"); @@ -2535,6 +2542,348 @@ mod tests { shutdown.cancel(); } + #[cfg(feature = "test-util")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn early_ack_put_tails_block_scanner_publication_until_all_renames_finish() { + use crate::storage_api_contracts::namespace::NamespaceLocking as _; + + let temp_dir = tempfile::tempdir().expect("create scanner PUT tail store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-put-tails", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = format!("scanner-put-tails-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create scanner PUT tail bucket"); + let set = &store.pools[0].disk_set[0]; + let objects = [("scanner-tail-a", vec![0xA1; 273]), ("scanner-tail-b", vec![0xB2; 379])]; + + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let (active, blocked, movement_generation) = store.scanner_data_movement_activity().await; + assert!(!active && !blocked); + assert!(ctx.scanner_publication_state_allowed(), "the set admission cache should start allowed"); + let (old_lease, _) = store + .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL) + .await + .expect("publication lease should be admitted before either PUT starts"); + + let barriers: Vec<_> = objects + .iter() + .map(|(object, _)| { + crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME) + }) + .collect(); + let trackers: Vec<_> = objects + .iter() + .map(|(object, _)| crate::set_disk::rename_fanout_barrier::observe_tasks(object)) + .collect(); + let puts: Vec<_> = objects + .iter() + .map(|(object, body)| { + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let object = *object; + let body = body.clone(); + tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(body); + put_store + .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default()) + .await + }) + }) + .collect(); + let committed = tokio::time::timeout(Duration::from_secs(30), async { + for barrier in &barriers { + barrier.wait_until_paused().await; + } + let mut committed = Vec::with_capacity(puts.len()); + for put in puts { + committed.push( + put.await + .expect("early-ACK PUT task should join while its tail is paused") + .expect("root PUT should return after quorum without waiting for its tail"), + ); + } + committed + }) + .await + .expect("both root PUTs must quorum-ACK while their tail disks remain paused"); + + assert!(trackers.iter().all(|tracker| tracker.running() >= 1)); + assert!(ctx.namespace_commits_pending()); + assert!( + ctx.scanner_publication_state_allowed(), + "pending PUT tails must not disable scanner namespace walks" + ); + let (active, blocked, observed_movement_generation) = store.scanner_data_movement_activity().await; + assert!(!active, "ordinary PUT tails are not decommission or rebalance work"); + assert!(!blocked, "ordinary PUT tails must not block the movement-only scan baseline"); + assert_eq!(observed_movement_generation, movement_generation); + assert!(store.scanner_data_usage_publication_blocked().await); + assert!(store.scanner_data_usage_publication_admission_guard().await.is_some()); + assert!(set.scanner_data_usage_publication_admission_guard().await.is_some()); + for error in [ + store + .acquire_scanner_publication_lease( + movement_generation, + crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL, + ) + .await + .expect_err("a new remote publication lease must reject pending PUT tails"), + store + .validate_scanner_publication_lease(old_lease, movement_generation) + .await + .expect_err("an existing remote lease must not bypass pending PUT tails"), + store + .acquire_scanner_publication_lease_guard(old_lease) + .await + .expect_err("target-side publication admission must reject pending PUT tails"), + ] { + assert!( + error.to_string().contains("blocked"), + "publication must fail because of active tails: {error}" + ); + } + store.release_scanner_publication_lease(old_lease).await; + + for (index, barrier) in barriers.iter().enumerate() { + let commit_generation = ctx.namespace_commit_generation(); + let namespace_generation = store.scanner_namespace_mutation_generation(); + barrier.release(); + tokio::time::timeout(Duration::from_secs(30), async { + while trackers[index].running() != 0 || ctx.namespace_commit_generation() <= commit_generation { + tokio::task::yield_now().await; + } + if index + 1 == barriers.len() { + while ctx.namespace_commits_pending() { + tokio::task::yield_now().await; + } + } + }) + .await + .expect("released tail must drain and publish its terminal namespace generation"); + assert!(store.scanner_namespace_mutation_generation() > namespace_generation); + let pending = index + 1 < barriers.len(); + assert_eq!(ctx.namespace_commits_pending(), pending); + assert_eq!(store.scanner_data_usage_publication_blocked().await, pending); + assert!(!store.scanner_data_movement_activity().await.1); + assert!(store.scanner_data_usage_publication_admission_guard().await.is_some()); + assert!(set.scanner_data_usage_publication_admission_guard().await.is_some()); + } + + let (lease, generation) = store + .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL) + .await + .expect("remote publication lease should resume after both tails drain"); + store + .validate_scanner_publication_lease(lease, generation) + .await + .expect("a resumed remote publication lease should validate"); + drop( + store + .acquire_scanner_publication_lease_guard(lease) + .await + .expect("target-side publication admission should resume after both tails drain"), + ); + assert!(store.release_scanner_publication_lease(lease).await); + + let disks = set.disk_inventory().await; + assert_eq!(disks.len(), 4); + for ((object, body), committed) in objects.iter().zip(&committed) { + let logical_size = i64::try_from(body.len()).expect("fixture payload size should fit i64"); + let etag = committed.etag.as_ref().expect("root PUT should return a committed ETag"); + for (disk_index, disk) in disks.iter().enumerate() { + let file_info = disk + .as_ref() + .expect("every fixture disk should remain online") + .read_version( + "", + &bucket, + object, + "", + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .unwrap_or_else(|err| panic!("disk {disk_index} should publish {object} after its tail finishes: {err}")); + assert_eq!(file_info.size, logical_size); + assert_eq!(file_info.metadata.get(http::header::ETAG.as_str()), Some(etag)); + assert!( + file_info.inline_data(), + "small fixture payloads should have an inline shard on every disk" + ); + let inline_data = file_info.data.as_ref().expect("every disk should retain its inline shard"); + let erasure = crate::erasure::coding::Erasure::try_new_with_options( + file_info.erasure.data_blocks, + file_info.erasure.parity_blocks, + file_info.erasure.block_size, + file_info.uses_legacy_checksum, + ) + .expect("persisted erasure geometry should be valid"); + let shard_size = + usize::try_from(erasure.shard_file_size(logical_size)).expect("fixture shard size should fit usize"); + crate::erasure::coding::bitrot_verify( + Cursor::new(inline_data.clone()), + inline_data.len(), + shard_size, + rustfs_utils::HashAlgorithm::HighwayHash256S, + erasure.shard_size(), + ) + .await + .unwrap_or_else(|err| panic!("disk {disk_index} should retain a complete valid shard for {object}: {err}")); + } + let mut reader = store + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("fully drained PUT should be readable"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("PUT body should drain"); + assert_eq!(&actual, body); + } + + let generation_before_internal_put = ctx.namespace_commit_generation(); + let internal_object = "scanner-tail-regression/internal-metadata"; + let internal_body = b"scanner metadata must not invalidate its own publication"; + let mut internal_reader = PutObjReader::from_vec(internal_body.to_vec()); + store + .put_object(RUSTFS_META_BUCKET, internal_object, &mut internal_reader, &ObjectOptions::default()) + .await + .expect("internal metadata PUT should commit without scanner self-invalidation"); + let internal_lock = set + .new_ns_lock(RUSTFS_META_BUCKET, internal_object) + .await + .expect("internal metadata tail lock should be available"); + drop( + internal_lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("internal metadata tail should drain"), + ); + assert_eq!(ctx.namespace_commit_generation(), generation_before_internal_put); + assert!(!ctx.namespace_commits_pending()); + assert!(store.scanner_data_usage_publication_admission_guard().await.is_some()); + assert!(set.scanner_data_usage_publication_admission_guard().await.is_some()); + let mut internal_reader = store + .get_object_reader(RUSTFS_META_BUCKET, internal_object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("internal metadata should remain readable"); + let mut actual = Vec::new(); + internal_reader + .stream + .read_to_end(&mut actual) + .await + .expect("internal metadata body should drain"); + assert_eq!(actual, internal_body); + }) + .await; + shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn cancelled_early_ack_put_keeps_scanner_publication_blocked_until_tail_finishes() { + let temp_dir = tempfile::tempdir().expect("create cancelled scanner PUT tail store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-cancelled-put-tail", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = format!("scanner-cancelled-put-tail-{}", Uuid::new_v4()); + let object = "scanner-cancelled-tail"; + let body = vec![0xC3; 273]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create cancelled scanner PUT tail bucket"); + + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let tracker = crate::set_disk::rename_fanout_barrier::observe_tasks(object); + let tail = + crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME); + let quorum = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + object, + crate::set_disk::PutObjectCommitPause::AfterRenameQuorum, + ); + let handoff = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + object, + crate::set_disk::PutObjectCommitPause::AfterRenameHandoff, + ); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put_body = body.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(put_body); + put_store + .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default()) + .await + }); + tokio::time::timeout(Duration::from_secs(30), tail.wait_until_paused()) + .await + .expect("cancelled PUT should pause one disk before rename"); + quorum.wait_until_paused().await; + put.abort(); + assert!( + put.await + .expect_err("caller should be cancelled after rename quorum") + .is_cancelled() + ); + quorum.release(); + handoff.wait_until_paused().await; + assert!(tracker.running() >= 1); + assert!(ctx.namespace_commits_pending()); + assert!(!store.scanner_data_movement_activity().await.1); + assert!(store.scanner_data_usage_publication_blocked().await); + assert!(store.scanner_data_usage_publication_admission_guard().await.is_some()); + assert!( + store.pools[0].disk_set[0] + .scanner_data_usage_publication_admission_guard() + .await + .is_some() + ); + let generation = store.scanner_namespace_mutation_generation(); + + handoff.release(); + tail.release(); + tokio::time::timeout(Duration::from_secs(30), async { + while tracker.running() != 0 || ctx.namespace_commits_pending() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled request's detached fanout must release scanner admission after finishing"); + assert!(store.scanner_namespace_mutation_generation() > generation); + assert!(!store.scanner_data_usage_publication_blocked().await); + assert!(store.scanner_data_usage_publication_admission_guard().await.is_some()); + for (disk_index, disk) in store.pools[0].disk_set[0].disk_inventory().await.iter().enumerate() { + let file_info = disk + .as_ref() + .expect("cancelled PUT fixture disk should remain online") + .read_version("", &bucket, object, "", &crate::disk::ReadOptions::default()) + .await + .unwrap_or_else(|err| panic!("cancelled PUT must still publish on disk {disk_index}: {err}")); + assert_eq!(file_info.size, i64::try_from(body.len()).expect("fixture body size should fit i64")); + } + let mut reader = store + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("a cancelled caller must not discard its quorum-committed object"); + let mut actual = Vec::new(); + reader + .stream + .read_to_end(&mut actual) + .await + .expect("cancelled PUT body should drain"); + assert_eq!(actual, body); + }) + .await; + shutdown.cancel(); + } + #[cfg(feature = "test-util")] #[test] #[serial_test::serial(storage_class_env)] @@ -2986,8 +3335,9 @@ mod tests { ) -> crate::core::pools::DecommissionTestFaultDecision { let target_bucket = bucket.to_string(); let target_object = object.to_string(); - Arc::new(move |stage, bucket, object, _attempt, succeeded| { + Arc::new(move |stage, bucket, object, attempt, succeeded| { if !succeeded + || attempt >= crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS || stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT || bucket != target_bucket || object != target_object @@ -2997,6 +3347,7 @@ mod tests { // Entry retries reset the local attempt; real copy errors can skip // successful attempts. Only injected faults spend this global budget. + // A real failure may consume an attempt, so preserve the final chance. faults .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| { (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1)) @@ -5018,6 +5369,7 @@ mod tests { "decommission-delete-fence", &[(2, 4), (1, 4)], CancellationToken::new(), + None, )) .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; @@ -5149,7 +5501,15 @@ mod tests { #[test] fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() { - for attempts in [[1, 2, 3], [1, 1, 2], [1, 3, 3]] { + let cases: &[&[(usize, bool, bool)]] = &[ + &[(1, true, true), (2, true, true), (3, true, false)], + &[(1, true, true), (1, true, true), (2, true, false)], + &[(1, true, true), (3, true, false), (3, true, false)], + &[(1, true, true), (2, false, false), (1, true, true), (2, true, false)], + &[(1, true, true), (2, false, false), (3, true, false)], + &[(3, true, false), (4, true, false)], + ]; + for case in cases { let faults = Arc::new(AtomicUsize::new(0)); let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults)); @@ -5163,14 +5523,16 @@ mod tests { } assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults"); - for (index, attempt) in attempts.into_iter().enumerate() { + let mut expected_faults = 0; + for &(attempt, succeeded, expected) in *case { assert_eq!( - hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, true), - index < 2, - "attempts={attempts:?}, index={index}" + hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, succeeded), + expected, + "fault plan {case:?} at attempt {attempt}" ); + expected_faults += usize::from(expected); + assert_eq!(faults.load(Ordering::SeqCst), expected_faults); } - assert_eq!(faults.load(Ordering::SeqCst), 2, "attempts={attempts:?}"); } } @@ -5306,6 +5668,15 @@ mod tests { changed_result.expect("SourceChanged entry retry must converge"); other_result.expect("other bucket entry must continue through ordinary copy retries"); + assert_eq!( + store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .expect("decommission progress should remain available") + .items_decommission_failed, + 0, + "entry completion must not hide an exhausted copy failure" + ); assert!(!rx.is_cancelled(), "entry-level SourceChanged must not cancel the shared worker token"); assert_eq!(mutation_calls.load(Ordering::SeqCst), 2, "entry must be re-listed after SourceChanged"); assert_eq!(ordinary_faults.load(Ordering::SeqCst), 2, "ordinary copy must consume the retry budget"); @@ -5934,6 +6305,7 @@ mod tests { "reverse-decommission-fixed-target", &[(1, 4), (1, 4)], CancellationToken::new(), + None, )) .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; @@ -6355,6 +6727,7 @@ mod tests { "multi-set-decommission-source-cleanup", &[(2, 4)], CancellationToken::new(), + None, )) .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; @@ -8870,18 +9243,17 @@ mod tests { const MANIFEST_COUNT: usize = 10; let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir"); - let (ctx, store, _shutdown) = - without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-manifest-pass", &[4])).await; + let mut instance_ctx = crate::runtime::instance::InstanceContext::new(); + instance_ctx.suppress_tier_delete_journal_recovery_for_test(); + let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "tier-delete-fast-manifest-pass", + &[(1, 4)], + CancellationToken::new(), + Some(Arc::new(instance_ctx)), + )) + .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; - let bucket = "tier-delete-fast-manifest-pass-bucket"; - store - .make_bucket(bucket, &MakeBucketOptions::default()) - .await - .expect("fast manifest pass bucket should be created"); - let incarnation = store - .bucket_incarnation_id(bucket) - .await - .expect("fast manifest pass bucket incarnation should resolve"); let tier_name = "FAST-MANIFEST-PASS"; let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) @@ -8889,9 +9261,19 @@ mod tests { .expect("fast manifest pass tier lease should resolve") .backend_identity(); for index in 0..MANIFEST_COUNT { + // Pagination must not depend on same-bucket lock wait deadlines. + let bucket = format!("tier-delete-fast-manifest-pass-{index}"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("fast manifest pass bucket should be created"); + let incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("fast manifest pass bucket incarnation should resolve"); install_aborting_dispatch_fixture( store.clone(), - bucket, + &bucket, incarnation, &format!("manifest-page-{index:06}/"), tier_name, @@ -8922,12 +9304,78 @@ mod tests { "one production pass must cross the default eight-manifest page limit" ); assert_eq!(stats.manifests.scanned, MANIFEST_COUNT); - assert_eq!(stats.manifests.deleted, MANIFEST_COUNT); - assert_eq!(stats.manifests.failed, 0); + assert_eq!(stats.manifests.deleted, MANIFEST_COUNT, "full recovery result: {stats:?}"); + assert_eq!(stats.manifests.failed, 0, "full recovery result: {stats:?}"); assert_eq!(manifest_marker, None); assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); assert_eq!(tier_delete_journal_count(store).await, 0); assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier"); + shutdown.cancel(); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn tier_delete_manual_pass_retains_manifest_owned_by_startup_recovery() { + let temp_dir = tempfile::tempdir().expect("create automatic recovery ownership store dir"); + let (ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-auto-owner", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "tier-delete-auto-owner-bucket"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("automatic recovery bucket should be created"); + let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation"); + let tier_name = "AUTO-OWNER"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("automatic recovery tier lease") + .backend_identity(); + + // The automatic worker must not observe a partially installed fixture. + let lifecycle_guard = store + .acquire_bucket_lifecycle_write_lock(bucket) + .await + .expect("fixture lifecycle lock"); + let (manifest_name, entries) = + install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "auto-owner/", tier_name, identity, 1).await; + let journal_name = tier_delete_journal_object_name(&entries[0]); + let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name); + drop(lifecycle_guard); + ctx.wake_tier_delete_journal_recovery(); + tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused()) + .await + .expect("startup recovery should own the manifest before a manual pass"); + assert!(tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name)); + + let stats = recover_tier_delete_dispatch_manifests(store.clone(), 8, None) + .await + .expect("manual recovery scan"); + assert_eq!(stats.scanned, 1, "{stats:?}"); + assert_eq!(stats.retained, 1, "{stats:?}"); + assert_eq!(stats.deleted, 0, "{stats:?}"); + assert_eq!(stats.failed, 0, "{stats:?}"); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1); + assert_eq!(tier_delete_journal_count(store.clone()).await, 1); + + hook.release_delete(); + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound)); + if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("automatic recovery should converge without a manual retry"); + assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); + assert_eq!(tier_delete_journal_count(store).await, 0); + assert_eq!(backend.remove_count().await, 0, "rollback must not delete from the remote tier"); + shutdown.cancel(); } #[cfg(feature = "test-util")] @@ -10302,8 +10750,17 @@ mod tests { const JOURNAL_COUNT: usize = 40; let temp_dir = tempfile::tempdir().expect("create rollback retry store dir"); - let (ctx, store, _shutdown) = - without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-rollback-retry", &[4])).await; + // Manual retries must own progress between fault removal and the next attempt. + let mut instance_ctx = crate::runtime::instance::InstanceContext::new(); + instance_ctx.suppress_tier_delete_journal_recovery_for_test(); + let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "dispatch-rollback-retry", + &[(1, 4)], + CancellationToken::new(), + Some(Arc::new(instance_ctx)), + )) + .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; let bucket = "dispatch-rollback-retry-bucket"; store @@ -10377,6 +10834,7 @@ mod tests { assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier"); + shutdown.cancel(); } #[cfg(feature = "test-util")] @@ -13204,6 +13662,7 @@ mod tests { "partial-set-prefix-delete", &[(2, 4)], CancellationToken::new(), + None, )) .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; @@ -16576,6 +17035,7 @@ mod tests { "prepared-directory-recovery", &[(2, 4)], shutdown, + None, )) .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; @@ -17228,6 +17688,38 @@ mod tests { .expect("test thread should complete"); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn odm_write_back_requires_one_set_and_enabled_namespace_locking() { + for (layout, locking, supported) in [ + (&[(1, 4)][..], true, true), + (&[(1, 4), (1, 4)][..], true, false), + (&[(2, 4)][..], true, false), + (&[(1, 4)][..], false, false), + ] { + temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some(if locking { "true" } else { "false" }))], async { + let dir = tempfile::tempdir().expect("isolated topology"); + let shutdown = CancellationToken::new(); + let (_ctx, store, _) = without_storage_class_env(build_isolated_test_store_with_layout( + dir.path(), + "odm-topology", + layout, + shutdown.clone(), + None, + )) + .await; + assert_eq!( + store.supports_atomic_create_only_write_back(), + supported, + "layout={layout:?}, locking={locking}" + ); + shutdown.cancel(); + }) + .await; + } + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 8a4579e1b..b2f2965f6 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -848,7 +848,7 @@ impl ECStore { } pub fn scanner_namespace_mutation_generation(&self) -> u64 { - list_objects::scanner_namespace_mutation_generation() + list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation()) } pub async fn scanner_data_movement_active(&self) -> bool { @@ -857,7 +857,7 @@ impl ECStore { } /// Return the storage-owned movement state and generation as one - /// authenticated activity snapshot. The read lock is acquired before + /// authenticated activity snapshot. The read lock is acquired before /// the state locks (cancelers, pool metadata, then rebalance metadata), /// matching the transition writer order and preventing a terminal state /// from being reported with the preceding generation. @@ -886,11 +886,12 @@ impl ECStore { /// Returns whether scanner metadata may still be hidden by a local /// data-movement state. Terminal failed/canceled decommission entries /// remain suspended until an operator clears or retries them, so they are - /// a publication barrier even after the worker has stopped. + /// a publication barrier even after the worker has stopped. Active PUT + /// rename fanouts also defer publication, including post-ACK tails. pub async fn scanner_data_usage_publication_blocked(&self) -> bool { let operation_gate = self.ctx.data_movement_operation_gate(); let _operation_guard = operation_gate.read_owned().await; - self.scanner_data_usage_publication_snapshot_blocked().await + self.scanner_data_usage_publication_snapshot_blocked().await || self.ctx.namespace_commits_pending() } pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus { @@ -1070,7 +1071,7 @@ impl ECStore { { return Err(Error::other("scanner publication lease generation is stale")); } - if self.scanner_data_movement_snapshot_locked().await.1 { + if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() { return Err(Error::other("scanner publication lease is blocked by data movement")); } @@ -1109,7 +1110,7 @@ impl ECStore { { return Err(Error::other("scanner publication lease generation is stale")); } - if self.scanner_data_movement_snapshot_locked().await.1 { + if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() { return Err(Error::other("scanner publication lease is blocked by data movement")); } if !self.ctx.scanner_publication_lease_is_active(token).await { @@ -1129,7 +1130,7 @@ impl ECStore { if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() { return Err(Error::other("scanner publication lease generation is exhausted")); } - if self.scanner_data_movement_snapshot_locked().await.1 { + if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() { return Err(Error::other("scanner publication lease is blocked by data movement")); } let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else { diff --git a/crates/heal/tests/heal_b5_versioned_regression_test.rs b/crates/heal/tests/heal_b5_versioned_regression_test.rs index 81cfdc79a..7b291f3db 100644 --- a/crates/heal/tests/heal_b5_versioned_regression_test.rs +++ b/crates/heal/tests/heal_b5_versioned_regression_test.rs @@ -44,7 +44,9 @@ use walkdir::WalkDir; mod storage_api; -use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _, ObjectOperations as _}; +use storage_api::integration::{ + BucketOperations, ECStore, MakeBucketOptions, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _, +}; /// 256 KiB + change: large enough to be stored as non-inline erasure shards /// (so each data version materializes as an on-disk `part.*` file we can assert @@ -106,6 +108,7 @@ async fn put_versioned(ecstore: &Arc, bucket: &str, object: &str, data: .put_object(bucket, object, &mut reader, &opts) .await .expect("versioned put_object failed"); + wait_for_put_tail(ecstore, bucket, object).await; info.version_id .map(|u| u.to_string()) .expect("versioned put must return a version id") @@ -117,6 +120,7 @@ async fn put_unversioned(ecstore: &Arc, bucket: &str, object: &str, dat .put_object(bucket, object, &mut reader, &ObjectOptions::default()) .await .expect("unversioned put_object failed"); + wait_for_put_tail(ecstore, bucket, object).await; } /// Create a delete-marker as the latest version (versioned:true, no version_id) @@ -160,20 +164,16 @@ fn xl_meta_path(obj_dir: &Path) -> PathBuf { obj_dir.join("xl.meta") } -async fn wait_for_two_version_copies(disks: &[PathBuf], bucket: &str, object: &str) { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - if disks.iter().all(|disk| { - let object_dir = object_dir(disk, bucket, object); - xl_meta_path(&object_dir).exists() && count_part_files(&object_dir) >= 2 - }) { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("PUT rename tails must converge before wiping the versioned fixture"); +async fn wait_for_put_tail(ecstore: &Arc, bucket: &str, object: &str) { + // Shards and xl.meta can exist before the detached PUT owner finishes. + let lock = ecstore + .new_ns_lock(bucket, object) + .await + .expect("fixture namespace lock should be created"); + let _settled = lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("PUT rename tail must finish before inspecting or wiping the fixture"); } fn recreate_heal_opts() -> HealOpts { @@ -305,7 +305,13 @@ mod serial_tests { let data_v2 = versioned_test_data(20); let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await; // OLD, non-latest let v2 = put_versioned(&ecstore, bucket, object, &data_v2).await; // latest - wait_for_two_version_copies(&disk_paths, bucket, object).await; + assert!( + disk_paths.iter().all(|disk| { + let dir = object_dir(disk, bucket, object); + xl_meta_path(&dir).exists() && count_part_files(&dir) >= 2 + }), + "both versions must exist on every disk before wiping the fixture" + ); // ── Pre-wipe: prove the fixture actually has 2 versions on disk[0] ── let obj_dir0 = object_dir(&disk_paths[0], bucket, object); diff --git a/crates/heal/tests/storage_api.rs b/crates/heal/tests/storage_api.rs index d224f4dfc..341834f57 100644 --- a/crates/heal/tests/storage_api.rs +++ b/crates/heal/tests/storage_api.rs @@ -23,6 +23,7 @@ pub(crate) mod integration { pub(crate) use rustfs_ecstore::api::storage::ECStore; pub(crate) use rustfs_storage_api::BucketOperations; pub(crate) use rustfs_storage_api::MakeBucketOptions; + pub(crate) use rustfs_storage_api::NamespaceLocking; pub(crate) use rustfs_storage_api::ObjectIO; pub(crate) use rustfs_storage_api::ObjectOperations; } diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 35229ba6b..dbc803839 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -196,7 +196,7 @@ pub(crate) async fn read_config_revision(store: Arc, path } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct DataUsageCacheRevisions { main: DataUsageCacheRevision, backup: Option, @@ -503,6 +503,10 @@ pub struct DataUsageCacheInfo { pub lkg_leader_epoch: Option, #[serde(default)] pub lkg_scan_plan_digest: Option, + /// Activity-sensitive identity for same-cycle set snapshot reuse. The + /// structural plan remains reusable across ordinary bucket writes. + #[serde(default)] + pub scan_execution_digest: Option, } impl Serialize for DataUsageCacheInfo { @@ -519,7 +523,8 @@ impl Serialize for DataUsageCacheInfo { + usize::from(self.lkg_next_cycle.is_some()) + usize::from(self.lkg_last_update.is_some()) + usize::from(self.lkg_leader_epoch.is_some()) - + usize::from(self.lkg_scan_plan_digest.is_some()); + + usize::from(self.lkg_scan_plan_digest.is_some()) + + usize::from(self.scan_execution_digest.is_some()); let mut state = serializer.serialize_map(Some(field_count))?; state.serialize_entry("name", &self.name)?; state.serialize_entry("next_cycle", &self.next_cycle)?; @@ -558,6 +563,9 @@ impl Serialize for DataUsageCacheInfo { if let Some(scan_plan_digest) = self.lkg_scan_plan_digest { state.serialize_entry("lkg_scan_plan_digest", &scan_plan_digest)?; } + if let Some(scan_execution_digest) = self.scan_execution_digest { + state.serialize_entry("scan_execution_digest", &scan_execution_digest)?; + } state.end() } } diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index 624ba13a3..19f4f778c 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -1067,6 +1067,7 @@ fn test_data_usage_cache_info_deserialize_defaults_scan_resume_after() { assert!(decoded.source.is_none()); assert!(!decoded.snapshot_complete); assert!(decoded.scan_plan_digest.is_none()); + assert!(decoded.scan_execution_digest.is_none()); assert_eq!(decoded.cache_key_format, 0); } @@ -1109,6 +1110,7 @@ fn test_data_usage_cache_info_unmarshal_old_msgpack_defaults_scan_resume_after() assert!(decoded.source.is_none()); assert!(!decoded.snapshot_complete); assert!(decoded.scan_plan_digest.is_none()); + assert!(decoded.scan_execution_digest.is_none()); assert_eq!(decoded.cache_key_format, 0); } @@ -1145,6 +1147,7 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() { source: Some(DataUsageCacheSource::new(1, 2)), snapshot_complete: true, scan_plan_digest: Some(TEST_PLAN_DIGEST), + scan_execution_digest: Some(DataUsageScanPlanDigest([42; 32])), cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, ..Default::default() }, @@ -1164,6 +1167,7 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() { assert_eq!(current.info.source, Some(DataUsageCacheSource::new(1, 2))); assert!(current.info.snapshot_complete); assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST)); + assert_eq!(current.info.scan_execution_digest, Some(DataUsageScanPlanDigest([42; 32]))); assert_eq!(current.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT); assert_eq!(current.find("bucket").map(|entry| entry.objects), Some(3)); diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 0f97924a4..df20e6178 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1616,7 +1616,7 @@ where // Refresh the storage-owned movement snapshot before reading background // heal state. A missing heal object yields an in-memory default; do not // let that default influence a cycle while publication is blocked. - if storeapi.scanner_data_usage_publication_blocked().await { + if storeapi.scanner_data_movement_pause_status().await.paused { mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); } @@ -1816,6 +1816,19 @@ where let publication_defer_reason = publication_defer_reason .or(remote_lease_defer_reason) .or(remote_lease_fence_defer_reason); + // A PUT tail can finish between the walk and lease acquisition without + // changing the movement epoch accepted by those leases. Re-prove the + // namespace baseline only after every peer has granted publication. + let post_lease_activity_defer_reason = if publication_defer_reason.is_none() + && remote_publication_leases.is_some() + && let Ok(result) = &scan_result + && result.status == ScannerCycleStatus::Complete + { + scanner_post_lease_activity_defer_reason(result.activity_digest(), probe_scanner_activity(storeapi.as_ref(), true).await) + } else { + None + }; + let publication_defer_reason = publication_defer_reason.or(post_lease_activity_defer_reason); // Include reasons discovered while acquiring or validating remote leases. let publication_deferred = publication_defer_reason.is_some(); let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled(); @@ -3240,6 +3253,21 @@ where } } +fn scanner_post_lease_activity_defer_reason( + expected_digest: Option<[u8; 32]>, + activity: Result, +) -> Option { + match activity { + Ok(snapshot) + if scanner_activity_allows_usage_publication(&snapshot) + && expected_digest == Some(scanner_activity_snapshot_digest(&snapshot)) => + { + None + } + Ok(_) | Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable), + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ScannerCyclePreCommitOutcome { RecoverCacheCycle(u64), @@ -3428,13 +3456,11 @@ use cycle_state::*; use leadership::*; use usage_store::*; -#[cfg(test)] -pub(crate) use activity::scanner_activity_snapshot_digest; pub use activity::scanner_topology_digest; pub(crate) use activity::{ ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication, - scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest, - scanner_dirty_usage_acknowledgements, + scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, + scanner_activity_structural_digest, scanner_dirty_usage_acknowledgements, }; pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance}; pub use backlog::{ diff --git a/crates/scanner/src/scanner/activity.rs b/crates/scanner/src/scanner/activity.rs index ffcbc5313..58c79f3ed 100644 --- a/crates/scanner/src/scanner/activity.rs +++ b/crates/scanner/src/scanner/activity.rs @@ -902,7 +902,6 @@ where observation } -#[cfg(test)] pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] { let mut hasher = Sha256::new(); hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes()); diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index b9b4cd299..17174246f 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -15,7 +15,8 @@ use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info}; use super::*; use crate::EcstoreResult; -use crate::storage_api::scan::BucketOperations as _; +use crate::storage_api::owner::ecstore_hold_namespace_commit; +use crate::storage_api::scan::{BucketOperations as _, ObjectIO as _}; use crate::{ DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_CACHE_KEY_FORMAT, DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntry, DataUsageScanPlanDigest, Endpoint, EndpointServerPools, @@ -1165,6 +1166,116 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() { global_metrics().set_cycle(None).await; } +#[tokio::test] +#[serial] +async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledging_usage() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let bucket = format!("scanner-coordinator-pending-{}", Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &crate::storage_api::scan::MakeBucketOptions::default()) + .await + .expect("fixture bucket should be created"); + let mut reader = PutObjReader::from_vec(b"first".to_vec()); + store.pools[0].disk_set[0] + .put_object( + &bucket, + "object", + &mut reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("fixture object should finish its rename fanout"); + crate::scanner_io::record_dirty_usage_bucket(&bucket); + let dirty_before = crate::scanner_io::dirty_usage_buckets_for_tests(); + let baseline = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("fixture usage baseline should be readable"); + let pending = ecstore_hold_namespace_commit(store.as_ref()); + let ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default()); + let mut cycle_info = CurrentCycle { + next: 1, + ..Default::default() + }; + let mut revision = DataUsageCacheRevision::Missing; + let outcome = tokio::time::timeout( + Duration::from_secs(30), + run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&budget)), + ) + .await + .expect("the coordinator must finish its namespace walk while a PUT is pending"); + assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal"); + assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)); + assert_eq!(cycle_info.next, 1, "a rejected publication must not advance the cycle"); + assert_eq!(revision, DataUsageCacheRevision::Missing); + assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before); + assert_eq!( + read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("the prior authoritative usage must remain readable"), + baseline, + "the pending candidate must not replace the authoritative baseline" + ); + + let committed_body = b"committed-after-walk"; + let mut reader = PutObjReader::from_vec(committed_body.to_vec()); + store.pools[0].disk_set[0] + .put_object( + &bucket, + "object", + &mut reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("the pending tail must change the physical object before it drains"); + assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before); + drop(pending); + let retry_budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default()); + let outcome = tokio::time::timeout( + Duration::from_secs(30), + run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&retry_budget)), + ) + .await + .expect("the same cycle must converge after the pending PUT drains"); + assert_eq!( + retry_budget.progress().0, + 1, + "the same-cycle retry must not reuse the pre-tail bucket cache" + ); + assert!(matches!( + outcome, + ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance + )); + assert_eq!(cycle_info.next, 2); + assert!(!crate::scanner_io::dirty_usage_buckets_for_tests().contains_key(&bucket)); + let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("the converged usage should be persisted"); + let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("the persisted usage should decode"); + assert_eq!(usage.usage_snapshot_converged, Some(true)); + assert_eq!(usage.scanner_cycle, Some(1)); + assert_eq!(usage.objects_total_count, 1); + assert_eq!( + usage.objects_total_size, + u64::try_from(committed_body.len()).expect("fixture body length") + ); + let bucket_usage = usage + .buckets_usage + .get(&bucket) + .expect("the scanned bucket should be published"); + assert_eq!(bucket_usage.objects_count, 1); + assert_eq!(bucket_usage.size, u64::try_from(committed_body.len()).expect("fixture body length")); + global_metrics().set_cycle(None).await; + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + #[tokio::test] #[serial] async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() { @@ -8485,6 +8596,66 @@ fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_gen } } +#[test] +fn post_lease_activity_proof_rejects_a_put_tail_that_finished_before_lease_acquisition() { + let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let expected_digest = Some(scanner_activity_snapshot_digest(&before)); + assert_eq!(scanner_post_lease_activity_defer_reason(expected_digest, Ok(before.clone())), None); + + let mut after = before.clone(); + after + .get_mut("node-2") + .expect("writer should be present") + .namespace_generation += 1; + assert_eq!( + before["node-2"].movement_generation, after["node-2"].movement_generation, + "the existing movement-only lease remains valid after a PUT tail drains" + ); + assert!(scanner_activity_allows_usage_publication(&after)); + let reason = scanner_post_lease_activity_defer_reason(expected_digest, Ok(after)); + assert_eq!(reason, Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)); + + let result = ScannerCycleResult::new(ScannerCycleStatus::Complete, None).with_remote_dirty_usage_acknowledgements(vec![ + ScannerDirtyUsageAcknowledgement { + host: "node-2".to_string(), + instance_id: "epoch-a".to_string(), + generation: 5, + }, + ]); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result( + result, + DataUsagePersistOutcome::Deferred(reason.expect("changed namespace should defer publication")), + ); + assert_eq!( + outcome, + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable) + ); + assert!( + acknowledgements.is_empty(), + "a rejected publication must not acknowledge the peer's dirty usage" + ); +} + +#[test] +fn post_lease_activity_proof_requires_a_complete_matching_baseline() { + let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let digest = scanner_activity_snapshot_digest(&before); + let mut blocked = before.clone(); + blocked.get_mut("node-2").expect("peer should be present").publication_blocked = true; + let blocked_digest = scanner_activity_snapshot_digest(&blocked); + for (expected, observed) in [ + (None, Ok(before)), + (Some(digest), Err("peer is unavailable".to_string())), + (Some(digest), Ok(BTreeMap::new())), + (Some(blocked_digest), Ok(blocked)), + ] { + assert_eq!( + scanner_post_lease_activity_defer_reason(expected, observed), + Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) + ); + } +} + #[test] fn scanner_activity_snapshot_digest_fences_storage_topology() { let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 0d5e82eeb..f57df0eec 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT; +use crate::data_usage_define::{DATA_USAGE_CACHE_KEY_FORMAT, DataUsageCacheRevisions}; use crate::scanner_budget::ScannerCycleBudget; use crate::scanner_folder::{ScannerItem, scan_data_folder}; use crate::sleeper::SCANNER_SLEEPER; @@ -271,6 +271,8 @@ pub struct ScannerBucketScanPlan { all_buckets: Arc>, scope: ScannerBucketScanScope, digest: DataUsageScanPlanDigest, + // Cache work must invalidate on namespace completion even when its scoped baseline remains reusable. + execution_digest: DataUsageScanPlanDigest, leader_epoch: u64, tier_registry_generation: u64, /// Epoch captured once for the whole scanner cycle. `None` is retained @@ -456,9 +458,12 @@ async fn scanner_cycle_activity_status( where S: ScannerStorage, { + // Read the pending-commit barrier before sampling its completion generation. + // A tail that drains during this await must invalidate the earlier baseline. + let publication_blocked = store.scanner_data_usage_publication_blocked().await; match crate::scanner::probe_scanner_activity(store, distributed).await { Ok(after) => { - let status = if after == *before { + let status = if !publication_blocked && after == *before { ScannerCycleActivityStatus::Unchanged } else { ScannerCycleActivityStatus::Changed @@ -760,6 +765,7 @@ fn scanner_activity_preflight( pub(crate) struct ScannerCycleResult { pub(crate) status: ScannerCycleStatus, publication_epoch: Option, + activity_digest: Option<[u8; 32]>, observational_snapshot_published: bool, dirty_usage_clear: Option, remote_dirty_usage_acknowledgements: Vec, @@ -774,6 +780,7 @@ impl ScannerCycleResult { Self { status, publication_epoch: None, + activity_digest: None, observational_snapshot_published: false, dirty_usage_clear, remote_dirty_usage_acknowledgements: Vec::new(), @@ -793,6 +800,15 @@ impl ScannerCycleResult { self.publication_epoch } + fn with_activity_digest(mut self, activity_digest: [u8; 32]) -> Self { + self.activity_digest = Some(activity_digest); + self + } + + pub(crate) fn activity_digest(&self) -> Option<[u8; 32]> { + self.activity_digest + } + pub(crate) fn with_observational_snapshot_published(mut self, published: bool) -> Self { self.observational_snapshot_published = published; self diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 7ff684c99..ba5203f20 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -604,10 +604,12 @@ pub(super) async fn persist_and_publish_cache_snapshot( store: Arc, updates: &mpsc::Sender, mut cache_snapshot: DataUsageCache, + initial_revisions: Option<&DataUsageCacheRevisions>, cache_cycle_floor: &AtomicU64, expected_publication_epoch: u64, ) -> Option { let source = cache_snapshot.info.source?; + let execution_digest = cache_snapshot.info.scan_execution_digest?; let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await { Ok(guard) => guard, Err(err) => { @@ -672,20 +674,36 @@ pub(super) async fn persist_and_publish_cache_snapshot( ); return None; } - if matches!( - current_cache_root_entry_with_generation( - &persisted, - DATA_USAGE_ROOT, - source, - cache_snapshot.info.next_cycle, - cache_snapshot.info.leader_epoch, - scan_plan_digest, - cache_snapshot.info.tier_registry_generation, - ), - Ok(Some(_)) - ) { + if persisted.info.scan_execution_digest == Some(execution_digest) + && matches!( + current_cache_root_entry_with_generation( + &persisted, + DATA_USAGE_ROOT, + source, + cache_snapshot.info.next_cycle, + cache_snapshot.info.leader_epoch, + scan_plan_digest, + cache_snapshot.info.tier_registry_generation, + ), + Ok(Some(_)) + ) + { cache_snapshot = persisted; } else { + // A later execution may have completed while this scan was walking. + // Only replace the cache revision from which this scan started. + if initial_revisions != Some(&revisions) { + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + state = "scan_baseline_revision_changed", + cache_name = DATA_USAGE_CACHE_NAME, + "Scanner skipped set snapshot without an unchanged baseline revision" + ); + return None; + } if guard.is_lock_lost() { error!( target: "rustfs::scanner::io", diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index a8686a1e4..b4f04481b 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -118,6 +118,7 @@ impl ScannerIOCache for SetDisks { all_buckets, scope, digest: scan_plan_digest, + execution_digest, leader_epoch, tier_registry_generation, publication_epoch, @@ -137,20 +138,24 @@ impl ScannerIOCache for SetDisks { .ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?, }; let mut old_cache = DataUsageCache::default(); - if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await { - warn!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_CACHE_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - pool = self.pool_index, - set = self.set_index, - cache_name = DATA_USAGE_CACHE_NAME, - state = "old_cache_load_failed", - error = %e, - "Scanner old data usage cache load failed; rebuilding from bucket caches" - ); - } + let initial_revisions = match old_cache.load_with_revisions(self.clone(), DATA_USAGE_CACHE_NAME).await { + Ok(revisions) => Some(revisions), + Err(e) => { + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + cache_name = DATA_USAGE_CACHE_NAME, + state = "old_cache_load_failed", + error = %e, + "Scanner old data usage cache load failed; rebuilding from bucket caches" + ); + None + } + }; let scoped_scan = prepare_scoped_set_scan( &old_cache, &buckets, @@ -195,6 +200,7 @@ impl ScannerIOCache for SetDisks { }; cache.info.last_update = Some(now); cache.info.snapshot_complete = true; + cache.info.scan_execution_digest = Some(execution_digest); cache.info.lkg_snapshot_complete = false; cache.info.lkg_next_cycle = None; cache.info.lkg_last_update = None; @@ -208,6 +214,7 @@ impl ScannerIOCache for SetDisks { self, &updates, cache, + initial_revisions.as_ref(), cache_cycle_floor.as_ref(), expected_publication_epoch, ) @@ -637,7 +644,7 @@ impl ScannerIOCache for SetDisks { let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]); let bucket_scan_plan_digest = - scanner_bucket_cache_digest(scan_plan_digest, dirty_usage_buckets_clone.get(&bucket.name).copied()); + scanner_bucket_cache_digest(execution_digest, dirty_usage_buckets_clone.get(&bucket.name).copied()); if let Some(server_epoch) = remote_server_epoch { let request_sequence = remote_session_sequence; @@ -1360,6 +1367,7 @@ impl ScannerIOCache for SetDisks { cache.info.next_cycle = want_cycle; cache.info.last_update.get_or_insert_with(SystemTime::now); cache.info.snapshot_complete = true; + cache.info.scan_execution_digest = Some(execution_digest); cache.info.lkg_snapshot_complete = false; cache.info.lkg_next_cycle = None; cache.info.lkg_last_update = None; @@ -1371,6 +1379,7 @@ impl ScannerIOCache for SetDisks { self.clone(), &updates, cache_snapshot, + initial_revisions.as_ref(), cache_cycle_floor.as_ref(), expected_publication_epoch, ) diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index 2947d638d..ad0afd579 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -180,7 +180,7 @@ where // canceled decommission remains suspended after its worker exits, so // starting a scan in that state could build a snapshot that cannot be // routed to the authoritative metadata object. - if store.scanner_data_usage_publication_blocked().await { + if store.scanner_data_movement_pause_status().await.paused { debug!( target: "rustfs::scanner::io", event = EVENT_SCANNER_SET_STATE, @@ -260,8 +260,13 @@ where } } bucket_plan_complete &= buckets_by_source.keys().copied().collect::>() == *expected_sources; + let activity_digest = crate::scanner::scanner_activity_snapshot_digest(&activity_before); let scan_plan_digest = scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before)); + let mut execution_hasher = Sha256::new(); + execution_hasher.update(scan_plan_digest.0); + execution_hasher.update(activity_digest); + let execution_digest = DataUsageScanPlanDigest(execution_hasher.finalize().into()); let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list)); let scan_scope = resolve_scanner_bucket_scan_scope( store, @@ -326,6 +331,7 @@ where }; return Ok(ScannerCycleResult::new(status, dirty_usage_clear) .with_publication_epoch(publication_epoch) + .with_activity_digest(activity_digest) .with_observational_snapshot_published(observational_snapshot_published) .with_remote_publication_lease_targets(remote_publication_lease_targets) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)); @@ -410,6 +416,7 @@ where all_buckets: Arc::clone(&all_buckets), scope: scan_scope.clone(), digest: scan_plan_digest, + execution_digest, leader_epoch, tier_registry_generation, publication_epoch, @@ -598,6 +605,7 @@ where }; Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear) .with_publication_epoch(publication_epoch) + .with_activity_digest(activity_digest) .with_observational_snapshot_published(observational_snapshot_published) .with_remote_publication_lease_targets(remote_publication_lease_targets) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements) diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index ec2c1ad65..89f81ff23 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -20,6 +20,7 @@ use crate::scanner_folder::ScannerItem; use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot; use crate::storage_api::owner::{ EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats, + ecstore_hold_namespace_commit, }; use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _}; use crate::{ @@ -343,6 +344,16 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() { .put_object(&bucket, object, &mut reader, &ScannerObjectOptions::default()) .await .expect("object should be written to its selected pool"); + + // Quorum ACK can precede tail publication on the disk chosen to scan. + let lock = store.pools[pool_index].disk_set[0] + .new_ns_lock(&bucket, object) + .await + .expect("fixture namespace lock should be created"); + let _settled = lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("fixture rename tail should finish before the usage scan"); } let ctx = CancellationToken::new(); @@ -362,7 +373,7 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() { .buckets_usage .get(&bucket) .expect("combined bucket usage should be present"); - assert_eq!(bucket_usage.objects_count, 2); + assert_eq!(bucket_usage.objects_count, 2, "{usage:?}"); assert_eq!(bucket_usage.size, 11); assert_eq!(usage.objects_total_count, 2); assert_eq!(usage.objects_total_size, 11); @@ -372,6 +383,102 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() { ); } +#[tokio::test] +#[serial] +async fn pending_put_commit_keeps_scanner_walk_live_without_authoritative_usage() { + let (_temp_dir, store) = setup_two_pool_scanner_store().await; + let bucket = format!("scanner-pending-put-{}", Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created across both pools"); + for (pool_index, (object, body)) in [("pool-a", b"first".as_slice()), ("pool-b", b"second".as_slice())] + .into_iter() + .enumerate() + { + let mut reader = ScannerPutObjReader::from_vec(body.to_vec()); + store.pools[pool_index].disk_set[0] + .put_object( + &bucket, + object, + &mut reader, + &ScannerObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("fixture objects must finish their rename fanouts before scanning"); + } + + let mut pending = Some(ecstore_hold_namespace_commit(store.as_ref())); + let mut previous_activity_digest = None; + let mut structural_plan_digest = None; + for (cycle, converged) in [(1, false), (2, true)] { + if converged { + drop(pending.take()); + } + assert_eq!(store.scanner_data_usage_publication_blocked().await, !converged); + assert!(!store.scanner_data_movement_pause_status().await.paused); + let activity = crate::scanner::probe_scanner_activity(store.as_ref(), false) + .await + .expect("the fixture activity should be observable"); + let activity_digest = crate::scanner::scanner_activity_snapshot_digest(&activity); + if let Some(previous) = previous_activity_digest.replace(activity_digest) { + assert_ne!(previous, activity_digest, "draining a namespace commit must change the publication proof"); + } + let ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default()); + let (updates, mut receiver) = mpsc::channel(1); + let result = tokio::time::timeout( + Duration::from_secs(30), + ScannerIOCycle::nsscanner_with_status( + store.as_ref(), + ctx, + Arc::clone(&budget), + updates, + cycle, + 1, + HealScanMode::Normal, + ), + ) + .await + .expect("namespace scanning must finish while a PUT commit is pending") + .expect("namespace scanning must remain available during a pending PUT commit"); + assert_eq!(result.activity_digest(), Some(activity_digest)); + if !converged { + assert_eq!(budget.progress().0, 2, "the pending commit must not suppress actual object traversal"); + } + assert_eq!( + result.status, + if converged { + ScannerCycleStatus::Complete + } else { + ScannerCycleStatus::Superseded + } + ); + let usage = receiver + .recv() + .await + .expect("the completed walk should produce a usage candidate"); + assert_eq!(usage.usage_snapshot_converged, Some(converged)); + assert_eq!(usage.scanner_cycle, Some(cycle)); + assert_eq!(usage.objects_total_count, 2); + assert_eq!(usage.objects_total_size, 11); + assert_eq!(usage.usage_snapshot_set_states.len(), 2); + for state in &usage.usage_snapshot_set_states { + let digest = state + .scan_plan_digest + .expect("each set must retain its structural cache identity"); + assert_eq!(*structural_plan_digest.get_or_insert(digest), digest); + } + let bucket_usage = usage.buckets_usage.get(&bucket).expect("the walked bucket must be present"); + assert_eq!(bucket_usage.objects_count, 2); + assert_eq!(bucket_usage.size, 11); + assert!(receiver.recv().await.is_none(), "each walk must emit exactly one terminal candidate"); + } +} + #[tokio::test] #[serial] async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() { @@ -387,6 +494,16 @@ async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() { .put_object(&bucket, "pool-b", &mut reader, &ScannerObjectOptions::default()) .await .expect("object should be written only to the second pool"); + { + let lock = store.pools[1].disk_set[0] + .new_ns_lock(&bucket, "pool-b") + .await + .expect("fixture namespace lock should be created"); + let _settled = lock + .get_write_lock(Duration::from_secs(30)) + .await + .expect("fixture rename tail should finish before the usage scan"); + } store.pools[0] .delete_bucket(&bucket, &DeleteBucketOptions::default()) .await @@ -797,6 +914,124 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa cache } +#[tokio::test] +#[serial] +async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers() { + let (_temp_dir, store) = setup_two_pool_scanner_store().await; + let set = Arc::clone(&store.pools[0].disk_set[0]); + let epoch = scanner_publication_epoch(Arc::clone(&set)).await.expect("idle set admission"); + let mut legacy = complete_set_usage_cache(&[("photos", 5)], DataUsageScanPlanDigest([1; 32])); + legacy.info.source = Some(DataUsageCacheSource::new(0, 0)); + legacy + .save(Arc::clone(&set), DATA_USAGE_CACHE_NAME) + .await + .expect("seed legacy set cache"); + let mut persisted = DataUsageCache::default(); + let initial = persisted + .load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME) + .await + .expect("capture the shared starting revision"); + let mut fresh = legacy.clone(); + fresh.info.scan_execution_digest = Some(DataUsageScanPlanDigest([2; 32])); + fresh.replace( + "photos", + DATA_USAGE_ROOT, + DataUsageEntry { + size: 20, + objects: 1, + ..Default::default() + }, + ); + let cycle_floor = AtomicU64::new(fresh.info.next_cycle); + let (tx, mut rx) = mpsc::channel(1); + assert!( + persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, fresh.clone(), Some(&initial), &cycle_floor, epoch) + .await + .is_some(), + "a legacy cache without execution identity must be refreshed" + ); + let published = rx.try_recv().expect("fresh snapshot should be forwarded"); + assert_eq!(published.find("photos").expect("published bucket").size, 20); + assert_eq!(published.info.scan_execution_digest, fresh.info.scan_execution_digest); + let current = persisted + .load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME) + .await + .expect("capture the current revision for the unidentified execution"); + + let mut stale = legacy.clone(); + stale.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32])); + for (candidate, revisions) in [(stale, &initial), (legacy, ¤t)] { + assert!( + persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, candidate, Some(revisions), &cycle_floor, epoch) + .await + .is_none(), + "a stale or unidentified execution must not replace the newer snapshot" + ); + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + } + fresh.info.scan_execution_digest = Some(DataUsageScanPlanDigest([4; 32])); + assert!( + persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, fresh.clone(), None, &cycle_floor, epoch) + .await + .is_none(), + "an unreadable starting revision must not authorize an overwrite" + ); + + fresh.info.scan_execution_digest = published.info.scan_execution_digest; + fresh.replace("photos", DATA_USAGE_ROOT, DataUsageEntry::default()); + assert!( + persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, fresh, Some(&initial), &cycle_floor, epoch) + .await + .is_some(), + "an overlapping identical execution must reuse the completed snapshot" + ); + assert_eq!( + rx.try_recv() + .expect("reused snapshot") + .find("photos") + .expect("reused bucket") + .size, + 20 + ); + persisted + .load(Arc::clone(&set), DATA_USAGE_CACHE_NAME) + .await + .expect("read the final durable set cache"); + assert_eq!(persisted.find("photos").expect("durable bucket").size, 20); + assert_eq!(persisted.info.scan_execution_digest, published.info.scan_execution_digest); + + let ctx = CancellationToken::new(); + let empty_execution = DataUsageScanPlanDigest([5; 32]); + set.nsscanner_cache( + ctx.clone(), + ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()), + ScannerBucketScanPlan { + buckets: Vec::new(), + all_buckets: Arc::new(Vec::new()), + scope: ScannerBucketScanScope::default(), + digest: DataUsageScanPlanDigest([6; 32]), + execution_digest: empty_execution, + leader_epoch: 11, + tier_registry_generation: 13, + publication_epoch: Some(epoch), + dirty_usage_buckets: Arc::new(HashMap::new()), + bucket_failures: ScannerBucketFailureState::default(), + pending_maintenance_work: Arc::new(AtomicBool::new(false)), + cache_cycle_floor: Arc::new(AtomicU64::new(8)), + }, + tx, + 8, + HealScanMode::Normal, + ) + .await + .expect("empty set scope should replace its prior nonempty cache"); + let empty = rx.try_recv().expect("empty set snapshot should be published"); + assert_eq!(empty.info.scan_execution_digest, Some(empty_execution)); + assert!(empty.info.snapshot_complete); + let root = empty.checked_flatten(DATA_USAGE_ROOT).expect("complete empty root"); + assert_eq!((root.size, root.objects), (0, 0)); +} + fn complete_usage_baseline( source: DataUsageCacheSource, scan_plan_digest: DataUsageScanPlanDigest, diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index 7b2cda493..a1981df30 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -127,6 +127,9 @@ pub(crate) use rustfs_lifecycle::{ use rustfs_storage_api as storage_contracts; pub(crate) mod owner { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::set_disk::test_util::hold_namespace_commit as ecstore_hold_namespace_commit; + pub(crate) use super::storage_contracts::{ HTTPPreconditions, HTTPRangeSpec, NS_SCANNER_PROTOCOL_VERSION, ObjectIO, ObjectOperations, ObjectToDelete, }; diff --git a/docs/architecture/scanner-usage-publication.md b/docs/architecture/scanner-usage-publication.md index 4b4791b1f..0665fc473 100644 --- a/docs/architecture/scanner-usage-publication.md +++ b/docs/architecture/scanner-usage-publication.md @@ -23,6 +23,30 @@ therefore has three identities: If any identity changes before commit, the result is a candidate for retry or observation, not an authoritative baseline. +Ordinary PUT rename fanouts also track instance-scoped in-flight work. A quorum +ACK does not release it: the actual disk tasks retain ownership until their +rename work ends, including when the request caller is cancelled. Scan admission +remains movement-only so sustained PUTs do not stop namespace walks and +scanner-driven lifecycle discovery. The post-walk local publication check and +remote publication leases reject pending fanouts. Begin/end namespace generations +invalidate scans and cached plans across the fanout; after acquiring remote +leases, the coordinator rechecks the full activity digest before publishing an +authoritative aggregate. This catches a tail that finishes between the scan's +last probe and lease acquisition. + +This adds no namespace or movement lock. An already-verified older snapshot may +still precede a newly started write. Sustained or stalled PUT tails can delay +authoritative usage publication, which resumes through the existing retry +schedule rather than a new immediate-wakeup protocol. Intermediate per-set and +prefix cache readers retain their existing approximate-cache semantics. A +prolonged pending tail with no generation changes can also delay cycle advancement +and fresh rescans of already-current caches; this is not a guarantee of lifecycle +progress under indefinitely stalled storage I/O. + +This PUT-tail protection requires every writer node to be upgraded. It does not +prove that a failed tail replica has healed, and it does not extend the same +in-flight tracking to multipart or other namespace mutation paths. + ## Fences The protocol uses separate fences because they exclude different stale inputs. @@ -33,7 +57,7 @@ They must not be collapsed unless the replacement proves the same exclusions. | Scanner leadership claim | scanner | competing scanner leaders and stale cycle writers | | Storage publication epoch | ECStore | usage computed across rebalance, decommission, or other data-movement generations | | Publication lease | scanner peers through ECStore-facing activity probes | remote dirty-usage or maintenance state that has not acknowledged the candidate | -| CAS revision | backing config object store | lost updates to `.usage.v2.json`, `.usage.json`, or cycle-state objects | +| CAS revision | backing config object store | lost updates to usage snapshots, scanner caches, or cycle-state objects | | Per-set freshness | scanner aggregation | a merged usage snapshot that combines stale and current set results | | Tier registry generation | scanner tier accounting | bytes classified against a different warm-tier registry | | Usage floor identity | scanner publication and ECStore quota fallback | empty or legacy values becoming plausible authoritative quota input | @@ -42,6 +66,28 @@ A reader that cannot prove the required fence for its surface must fail closed or use the documented observed path below. It must not synthesize an empty usage snapshot for a missing or corrupt authoritative object. +## Cache Execution Identity + +The structural scan-plan digest can remain stable across ordinary bucket writes +so a scoped scan can retain unaffected baseline buckets. It is not sufficient +proof for reusing a completed result within the same cycle. Bucket work uses an +execution digest combining the structural plan and the full activity snapshot, +with the bucket's dirty generation included in its cache identity. Completed set +caches carry the same execution digest separately from their structural plan. +The persisted set-root fast path requires equal execution identities as well as +the existing source, cycle, leader, tier, and cache-structure checks. + +A set scan also captures its starting cache revisions. When the persisted +execution differs, replacement requires those revisions to remain unchanged; +otherwise a slow scan could overwrite a newer completed result. The existing +cache lock, conditional save, and movement admission still fence the commit. + +The optional `scan_execution_digest` field is appended to the map-encoded cache +metadata. Legacy caches remain readable but cannot satisfy same-cycle set-root +reuse without this identity. Older readers can ignore the added map key, but +older writers do not enforce its fence; readability is not a mixed-version +publication-safety guarantee. + ## Persisted Objects The persisted objects are part of the compatibility contract. Removing one diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md index eb7cde57f..8155c6f11 100644 --- a/docs/operations/on-demand-migration.md +++ b/docs/operations/on-demand-migration.md @@ -89,9 +89,11 @@ Setting `"enabled": false` in the config has the same read-path effect as deleti The status endpoint reports **the node that answered the request**. Counters, queue depth and breaker state are per-node runtime state, so in a distributed deployment query every node; the saved configuration and `updated_at` are cluster-wide. -### Backfill (ships with ODM-12) +### Backfill -Read-through only migrates what clients touch. The background backfill job walks the source listing and pulls the remainder, with a persisted checkpoint (`.rustfs.sys/buckets//on-demand-migration-backfill.json`), a single-owner lease, resume after restart, and `POST .../{bucket}/backfill?op=start|cancel` plus `GET .../{bucket}/backfill` admin routes. That slice (rustfs/backlog#2159) is not part of the build this page was written against: the shape above is the agreed design, and the exact request/response bodies must be re-checked against `docs/architecture/admin-route-action-snapshot.md` once it lands. +Backfill waits for the result of every pull, including a pull already queued by an online request. A failed or cancelled shared pull is counted as a failure, never as successful migration. The persisted continuation cursor stays at the first failed page; a takeover replays from there and skips objects already present locally. `completed_with_failures` is not a cutover-ready state. + +Read-through only migrates what clients touch. The background backfill job walks the source listing and pulls the remainder, with a persisted checkpoint (`.rustfs.sys/buckets//on-demand-migration-backfill.json`), a single-owner lease, resume after restart, and `POST .../{bucket}/backfill?op=start|cancel` plus `GET .../{bucket}/backfill` admin routes. See `docs/architecture/admin-route-action-snapshot.md` for the route contract. ## Configuration reference @@ -129,7 +131,7 @@ The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unkno | `policy.source_timeout.idle_ms` | integer | `30000` | `100..=600000`; enforced per body chunk on both the background pump and the inline tee | | `policy.bandwidth_limit_bytes_per_sec` | integer \| null | `null` | When set, at least `65536` | -Values that are **not** configurable: the breaker opens after 5 consecutive counted failures inside a 30 s window, stays open for 30 s and then admits one probe (`breaker.rs`); the negative cache holds at most 100 000 keys per bucket with LRU eviction (`negative_cache.rs`); a background pull retries a retryable source failure at most 3 times with 1 s / 4 s / 16 s base delays plus up to 25 % jitter (`pull.rs`). The SDK's own retry policy is disabled on the source client, so one logical source call is exactly one wire request and the retry budget above is the only one. +Values that are **not** configurable: the breaker opens after 5 consecutive counted failures inside a 30 s window, stays open for 30 s and then admits one probe (`breaker.rs`); the negative cache holds at most 100 000 keys per bucket with LRU eviction (`negative_cache.rs`); a background pull retries a retryable source failure at most 3 times with 1 s / 4 s / 16 s base delays plus up to 25 % jitter (`pull.rs`). The SDK's own retry policy is disabled on the source client. Each SDK operation makes one wire request; an ambiguous HEAD 404 additionally probes the bucket, within the same configured first-byte budget. Validation also rejects two shapes outright: a source whose endpoint and bucket name **this** bucket on this deployment (`SelfReference`), and a source that matches one of the bucket's own replication targets (`ReplicationLoop`) — that pairing would amplify a write-back into a loop. @@ -160,6 +162,14 @@ No write, delete, ACL or versioning permission is required or used. Scope the po Behaviour a client can observe. The "Test" column names the case that pins it: `*_test.rs` files live under `crates/e2e_test/src/on_demand_migration/`, and the unit tests live next to the code in `rustfs/src/app/object/get.rs`, `head.rs` and `shared.rs`. +ODM merged continuation tokens use a NUL-prefixed JSON envelope inside the existing base64 encoding. NUL is not valid in a local object key, so a legitimate JSON-shaped key can never be mistaken for a merged cursor. Upgrade every node before using list-through, and restart any in-progress ODM listing issued by an older build: its unframed JSON tokens cannot be distinguished from legitimate local keys. Ordinary local listing tokens remain unchanged. Tokens issued by this build can still resume the local side after list-through is disabled. + +Source `HEAD` responses with status 404 require a successful bucket probe before being negative-cached. The source credential therefore needs permission for `HeadBucket` (S3 `ListBucket`); a prefix-restricted ListBucket policy can deny that probe, in which case the response is a source failure rather than a cached miss. A missing/inaccessible source bucket, a missing source version, or an ambiguous GET 404 is not proof that the requested key is absent. Conditional GET validators are checked against the actual source GET metadata as well as the advisory HEAD; a missing required validator fails with 424. Source LIST entries without a key or a non-negative size fail the page rather than fabricating an empty object. + +Write-back currently requires namespace locking enabled and exactly one pool with one erasure set. Other topologies fail write-back explicitly as `unsupported`: source reads remain available, but backfill cannot complete successfully or certify cutover. This restriction avoids relying on a set-local condition across distinct pool or lock domains; it does not restrict ordinary S3 writes. Full cross-pool migration requires a globally fenced commit protocol. + +On the supported topology, write-back uses a create-only check under the local storage commit lock for both single-part PUT and multipart completion. A client write that commits while ODM is reading the source is preserved. With `respect_local_delete_marker=true`, a concurrent versioned deletion is preserved too. An explicit `respect_local_delete_marker=false` still permits revival; an unversioned deletion has no tombstone and therefore cannot be distinguished from a key that has never existed locally. + | Situation | Behaviour | Test | |---|---|---| | GET miss, object at or below `inline_max_bytes` | One source GET, teed: the client streams while the same bytes are written locally. Later reads are local and carry no source marker | `get_basic_test.rs::get_miss_pulls_inline_and_serves_locally_afterwards`, `get.rs::odm_get_inline_streams_to_client_and_commits_the_same_bytes` | @@ -229,7 +239,7 @@ Five provenance keys are written on every pulled object under both internal pref | Concurrency limit | Local write amplification | `max_concurrent_pulls` permits shared by inline and background pulls | | Bounded queue | Unbounded memory on a burst | `pull_queue_capacity` waiting jobs; overflow is counted as `queue_full` and never fails a client response | | Bandwidth limit | Source and network saturation | `bandwidth_limit_bytes_per_sec` (minimum 64 KiB/s) on the source client | -| Retry budget | Transient source blips | Background pulls retry a retryable failure up to 3 times (1 s / 4 s / 16 s plus jitter). Inline pulls never retry: the bytes are already on their way to the client. The SDK retry policy on the source client is disabled (`RemoteS3RetryPolicy::Disabled`), so this is the only retry budget and one logical source call is exactly one wire request — replication targets keep the SDK's three attempts, declared on their own spec | +| Retry budget | Transient source blips | Background pulls retry a retryable failure up to 3 times (1 s / 4 s / 16 s plus jitter). Inline pulls never retry: the bytes are already on their way to the client. The SDK retry policy is disabled (`RemoteS3RetryPolicy::Disabled`); HEAD 404 also requires one bucket probe. Replication targets keep their separately declared three SDK attempts | | Idle timeout | A source that answers and then goes quiet mid-body | `source_timeout.idle_ms` per body chunk on both paths. The budget measures the source read, upstream of the inline tee, so a slow client is never mistaken for an idle source; when it fires the client stream ends in an error and the write-back is discarded | | Anti-loop marker | Migration chains between RustFS/MinIO deployments | Every source request carries `x-rustfs-source-proxy-request` and `x-minio-source-proxy-request`; a request carrying it is always answered locally | | Outbound endpoint policy | SSRF | See [outbound-connection-policy.md](outbound-connection-policy.md) | diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs index 23b7513f2..b8aa0d38e 100644 --- a/rustfs/src/app/bucket_list_through.rs +++ b/rustfs/src/app/bucket_list_through.rs @@ -551,6 +551,9 @@ mod tests { #[test] fn a_plain_local_token_is_passed_through_and_a_tampered_one_is_rejected() { + let json_key = r#"{"t":"odm-list","v":1,"local_done":true}"#; + assert!(decode_list_cursor(Some(json_key)).expect("valid local key").is_none()); + assert!(matches!(local_cursor(Some(json_key), None), LocalListCursor::Token(Some(local)) if local == json_key)); assert!( decode_list_cursor(Some("photos/a.jpg")) .expect("plain markers decode") diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index b9158afda..c1511ebba 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -4615,6 +4615,7 @@ fn odm_inline_client_body(primary: TeePrimary) -> StreamingBlob { async fn odm_get_passthrough( state: &Arc, source: &S, + headers: &HeaderMap, key: &str, range: Option<&HTTPRangeSpec>, backfill: Option, @@ -4623,6 +4624,9 @@ async fn odm_get_passthrough( Ok(get) => get, Err(err) => return OdmGetReply::Error(odm_get_source_failure(state, &err)), }; + if let Err(err) = odm_check_source_preconditions(headers, &get.head) { + return OdmGetReply::Error(err); + } let content_length = match odm_content_length(get.head.size) { Ok(length) => length, Err(err) => { @@ -4648,6 +4652,7 @@ async fn odm_get_passthrough( async fn odm_get_inline( state: &Arc, source: &S, + headers: &HeaderMap, key: &str, leader: PullLeader, request_context: Option, @@ -4676,6 +4681,12 @@ async fn odm_get_inline( body, content_range, } = get; + // HEAD and GET can observe different source versions. Validate the + // representation whose body will actually be returned and persisted. + if let Err(err) = odm_check_source_preconditions(headers, &head) { + leader.complete(Err(PullError::canceled("source GET did not satisfy request preconditions"))); + return OdmGetReply::Error(err); + } // The object outgrew the inline budget between HEAD and GET: followers // stream through on their own and the background pull stores it. if head.size > policy.inline_max_bytes { @@ -4758,19 +4769,19 @@ pub(super) async fn odm_get_from_source( let policy = &state.config().policy; if let Some(range) = range { let backfill = (policy.range_get == RangeGetPolicy::ServeAndBackfill).then_some(PullReason::RangeGet); - return odm_get_passthrough(state, source, key, Some(range), backfill).await; + return odm_get_passthrough(state, source, headers, key, Some(range), backfill).await; } if head.size > policy.inline_max_bytes { - return odm_get_passthrough(state, source, key, None, Some(PullReason::LargeObject)).await; + return odm_get_passthrough(state, source, headers, key, None, Some(PullReason::LargeObject)).await; } let slot = match state.acquire_pull_slot(key).await { Ok(slot) => slot, // The bucket state was torn down under this request: serve it // without queueing anything on the old state. - Err(_) => return odm_get_passthrough(state, source, key, None, None).await, + Err(_) => return odm_get_passthrough(state, source, headers, key, None, None).await, }; match slot { - PullSlot::Leader(leader) => odm_get_inline(state, source, key, leader, request_context).await, + PullSlot::Leader(leader) => odm_get_inline(state, source, headers, key, leader, request_context).await, PullSlot::Follower(follower) => { let first_byte = Duration::from_millis(policy.source_timeout.first_byte_ms); match tokio::time::timeout(first_byte, follower.wait()).await { @@ -4778,7 +4789,7 @@ pub(super) async fn odm_get_from_source( stats.record_request(OdmOp::Get, OdmOutcome::SourceHit); OdmGetReply::RetryLocal } - Ok(Err(_)) | Err(_) => odm_get_passthrough(state, source, key, None, None).await, + Ok(Err(_)) | Err(_) => odm_get_passthrough(state, source, headers, key, None, None).await, } } } @@ -5296,6 +5307,73 @@ mod on_demand_migration_tests { assert!(rt.write_back.puts().is_empty()); } + #[tokio::test] + async fn odm_get_rechecks_conditions_against_the_get_representation() { + for inline_max_bytes in [0, 1024] { + for range in [ + None, + Some(HTTPRangeSpec { + is_suffix_length: false, + start: 0, + end: 2, + }), + ] { + let rt = runtime( + "changed-source", + PolicyConfig { + inline_max_bytes, + ..Default::default() + }, + ) + .await; + let state = rt.state("changed-source"); + let before = source_head(b"before"); + let after = source_head(b"after!"); + let source = ScriptedSource::new(vec![Ok(before.clone())], vec![Ok((after, b"after!".to_vec(), None))]); + let mut headers = HeaderMap::new(); + headers.insert( + http::header::IF_MATCH, + HeaderValue::from_str(&format!("\"{}\"", before.etag.expect("etag"))).expect("header"), + ); + let error = failed(odm_get_from_source(&state, &source, &headers, KEY, range.as_ref(), None).await); + assert_eq!(error.code(), &S3ErrorCode::PreconditionFailed); + assert_eq!(source.get_calls(), 1); + assert_eq!(state.inflight_keys(), 0); + assert!(rt.write_back.puts().is_empty(), "a failed condition must not start write-back"); + } + } + } + + #[tokio::test] + async fn odm_get_missing_validators_cannot_bypass_a_condition() { + for inline_max_bytes in [0, 1024] { + let rt = runtime( + "missing-validator", + PolicyConfig { + inline_max_bytes, + ..Default::default() + }, + ) + .await; + let state = rt.state("missing-validator"); + let before = source_head(b"before"); + let after = SourceHead { + size: 6, + ..Default::default() + }; + let source = ScriptedSource::new(vec![Ok(before.clone())], vec![Ok((after, b"after!".to_vec(), None))]); + let mut headers = HeaderMap::new(); + headers.insert( + http::header::IF_MATCH, + HeaderValue::from_str(&format!("\"{}\"", before.etag.expect("etag"))).expect("header"), + ); + let error = failed(odm_get_from_source(&state, &source, &headers, KEY, None, None).await); + assert_eq!(error.status_code(), Some(StatusCode::FAILED_DEPENDENCY)); + assert_eq!(error.message(), Some("missing_source_validator")); + assert!(rt.write_back.puts().is_empty()); + } + } + #[tokio::test] async fn odm_get_source_not_found_is_404_and_negative_cached() { let rt = runtime("n", PolicyConfig::default()).await; diff --git a/rustfs/src/app/object/internal_put.rs b/rustfs/src/app/object/internal_put.rs index 4c54ddc3f..bfd01429f 100644 --- a/rustfs/src/app/object/internal_put.rs +++ b/rustfs/src/app/object/internal_put.rs @@ -57,6 +57,9 @@ pub(crate) struct InternalPutContext { pub(crate) expected_md5_hex: Option, /// ETag to store instead of the computed one. pub(crate) preserve_etag: Option, + /// Reject an existing current object under the storage commit lock. + pub(crate) if_absent: bool, + pub(crate) preserve_delete_marker: bool, pub(crate) content_headers: HashMap, pub(crate) user_metadata: HashMap, pub(crate) tags: Option, @@ -240,6 +243,8 @@ impl DefaultObjectUsecase { size, expected_md5_hex, preserve_etag, + if_absent, + preserve_delete_marker, content_headers, user_metadata, tags, @@ -252,7 +257,10 @@ impl DefaultObjectUsecase { }; let size = i64::try_from(size).map_err(|_| ApiError::invalid_request("internal put size exceeds the supported range"))?; - let headers = internal_put_headers(&content_headers)?; + let mut headers = internal_put_headers(&content_headers)?; + if if_absent { + headers.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*")); + } validate_internal_write_target(&key, &bucket, &headers).await?; remove_source_replication_bookkeeping(&mut internal_metadata); @@ -287,6 +295,7 @@ impl DefaultObjectUsecase { origin: PutObjectOrigin::Internal { principal_id, emit_events, + preserve_delete_marker, }, }; let committed = self @@ -527,10 +536,14 @@ impl DefaultObjectUsecase { .map_err(api_error_from_s3)?; let store = self.object_store().ok_or_else(not_initialized)?; - let headers = HeaderMap::new(); + let mut headers = HeaderMap::new(); + if ctx.if_absent { + headers.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*")); + } let mut opts = get_complete_multipart_upload_opts_with_replication_authorization(&headers, false).map_err(ApiError::from)?; opts.preserve_etag = ctx.preserve_etag.clone(); + opts.preserve_delete_marker = ctx.preserve_delete_marker; let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; opts.versioned = versioned; opts.version_suspended = BucketVersioningSys::prefix_suspended(&bucket, &key).await; @@ -747,6 +760,8 @@ mod tests { size: Some(body.len() as u64), expected_md5_hex: Some(md5_hex(body)), preserve_etag: None, + if_absent: false, + preserve_delete_marker: false, content_headers: HashMap::from([ ("Content-Type".to_string(), "text/plain".to_string()), ("Cache-Control".to_string(), "max-age=60".to_string()), diff --git a/rustfs/src/app/object/on_demand_migration_put.rs b/rustfs/src/app/object/on_demand_migration_put.rs index fc81009a1..685e60b38 100644 --- a/rustfs/src/app/object/on_demand_migration_put.rs +++ b/rustfs/src/app/object/on_demand_migration_put.rs @@ -66,6 +66,15 @@ impl OnDemandMigrationWriteBack { .object_store() .ok_or_else(|| WriteBackError::Local("object store is not initialized".to_string())) } + + fn require_atomic_write_back(&self) -> Result<(), WriteBackError> { + if !self.store()?.supports_atomic_create_only_write_back() { + return Err(WriteBackError::Unsupported( + "write-back requires namespace locking and exactly one pool with one erasure set".to_string(), + )); + } + Ok(()) + } } fn rfc3339(time: OffsetDateTime) -> String { @@ -161,6 +170,8 @@ pub(super) async fn write_back_context(request: &WriteBackRequest, single_part: size: Some(head.size), expected_md5_hex: single_part.then(|| expected_md5_hex(head)).flatten(), preserve_etag, + if_absent: true, + preserve_delete_marker: request.respect_delete_marker, content_headers: content_headers(head), user_metadata: head.user_metadata.clone(), tags: request.tags.as_ref().and_then(encode_tags), @@ -207,6 +218,7 @@ impl OdmWriteBack for OnDemandMigrationWriteBack { } async fn put_object(&self, request: &WriteBackRequest, body: WriteBackBody) -> Result { + self.require_atomic_write_back()?; let ctx = write_back_context(request, true).await; self.usecase() .internal_put_object(ctx, body) @@ -216,6 +228,7 @@ impl OdmWriteBack for OnDemandMigrationWriteBack { } async fn create_multipart_upload(&self, request: &WriteBackRequest) -> Result { + self.require_atomic_write_back()?; let ctx = write_back_context(request, false).await; self.usecase() .internal_create_multipart_upload(&ctx) @@ -249,6 +262,7 @@ impl OdmWriteBack for OnDemandMigrationWriteBack { upload_id: &str, parts: Vec, ) -> Result { + self.require_atomic_write_back()?; let ctx = write_back_context(request, false).await; let parts = parts .into_iter() @@ -334,6 +348,7 @@ mod tests { pulled_at: OffsetDateTime::from_unix_timestamp(1_756_800_000).expect("valid timestamp"), preserve_etag: true, emit_events: true, + respect_delete_marker: true, tags: Some(HashMap::from([ ("team".to_string(), "storage".to_string()), ("env".to_string(), "prod".to_string()), @@ -486,6 +501,33 @@ mod tests { assert!(!local.delete_marker); } + #[tokio::test] + #[serial_test::serial] + async fn write_back_rejects_unsupported_topology_before_any_mutation() { + let (_dir, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = "odm-unsupported"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket"); + let write_back = OnDemandMigrationWriteBack::new(); + let req = request(bucket, "key", source_head(b"source")); + assert!(matches!( + write_back.put_object(&req, body_stream(b"source")).await, + Err(WriteBackError::Unsupported(_)) + )); + assert!(matches!( + write_back.create_multipart_upload(&req).await, + Err(WriteBackError::Unsupported(_)) + )); + assert!(matches!( + write_back.complete_multipart_upload(&req, "no-session", Vec::new()).await, + Err(WriteBackError::Unsupported(_)) + )); + assert_nothing_left(&store, bucket, "key").await; + } + #[tokio::test] #[serial_test::serial] async fn write_back_integrity_failure_leaves_nothing_behind() { @@ -503,6 +545,133 @@ mod tests { assert_nothing_left(&store, &bucket, "wrong.bin").await; } + #[tokio::test] + #[serial_test::serial] + async fn write_back_commit_does_not_overwrite_a_concurrent_client_put() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + for versioned in [false, true] { + let (store, bucket) = write_back_test_bucket("odm-wb-race", versioned).await; + let source = b"old source bytes"; + let client = b"new client bytes"; + let req = request(&bucket, "race", source_head(source)); + let client_req = request(&bucket, "race", source_head(client)); + let mut client_ctx = write_back_context(&client_req, true).await; + client_ctx.if_absent = false; + let client_after = PutObjectCommitBarrier::install(&bucket, "race", PutObjectCommitPause::AfterNamespace); + let client_put = tokio::spawn(async move { + DefaultObjectUsecase::from_global() + .internal_put_object(client_ctx, body_stream(client)) + .await + }); + client_after.wait_until_paused().await; + let source_before = PutObjectCommitBarrier::install(&bucket, "race", PutObjectCommitPause::BeforeNamespace); + let write_back = OnDemandMigrationWriteBack::new(); + let (result, ()) = tokio::join!(write_back.put_object(&req, body_stream(source)), async { + source_before.wait_until_paused().await; + drop(source_before); + drop(client_after); + }); + let committed = client_put.await.expect("client task").expect("ordinary client write wins"); + assert!( + matches!(result, Err(WriteBackError::Local(ref error)) if error.contains("PreconditionFailed")), + "{result:?}" + ); + let stored = stored_object(&store, &bucket, "race").await; + assert_eq!(stored.etag, committed.etag); + assert_eq!(stored.version_id, committed.version_id); + assert_eq!(committed.version_id.is_some(), versioned); + assert_eq!(raw_object_bytes(&store, &bucket, "race").await, client); + } + } + + #[tokio::test] + #[serial_test::serial] + async fn write_back_multipart_completion_preserves_a_client_put_after_staging() { + let (store, bucket) = write_back_test_bucket("odm-mpu-race", false).await; + let write_back = OnDemandMigrationWriteBack::new(); + let req = request(&bucket, "race", source_head(b"source")); + let upload_id = write_back.create_multipart_upload(&req).await.expect("create"); + let part = write_back + .upload_part(&req, &upload_id, 1, 6, body_stream(b"source")) + .await + .expect("stage"); + let mut client_ctx = write_back_context(&request(&bucket, "race", source_head(b"client")), true).await; + client_ctx.if_absent = false; + let committed = DefaultObjectUsecase::from_global() + .internal_put_object(client_ctx, body_stream(b"client")) + .await + .expect("client put after staging"); + let result = write_back.complete_multipart_upload(&req, &upload_id, vec![part]).await; + assert!( + matches!(result, Err(WriteBackError::Local(ref error)) if error.contains("PreconditionFailed")), + "{result:?}" + ); + write_back + .abort_multipart_upload(&bucket, "race", &upload_id) + .await + .expect("abort rejected upload"); + let stored = stored_object(&store, &bucket, "race").await; + assert_eq!(stored.etag, committed.etag); + assert_eq!(stored.version_id, committed.version_id); + assert_eq!(raw_object_bytes(&store, &bucket, "race").await, b"client"); + } + + #[tokio::test] + #[serial_test::serial] + async fn write_back_preserves_delete_markers_unless_policy_allows_revival() { + for multipart in [false, true] { + let (store, bucket) = write_back_test_bucket("odm-wb-tombstone", true).await; + let write_back = OnDemandMigrationWriteBack::new(); + let mut req = request(&bucket, "deleted", source_head(b"source")); + let staged = if multipart { + let id = write_back.create_multipart_upload(&req).await.expect("create"); + let part = write_back + .upload_part(&req, &id, 1, 6, body_stream(b"source")) + .await + .expect("part"); + Some((id, part)) + } else { + None + }; + store + .delete_object( + &bucket, + "deleted", + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("delete marker"); + let marker = stored_object(&store, &bucket, "deleted").await; + assert!(marker.delete_marker); + let rejected = if let Some((id, part)) = staged { + let result = write_back.complete_multipart_upload(&req, &id, vec![part]).await; + write_back + .abort_multipart_upload(&bucket, "deleted", &id) + .await + .expect("abort"); + result + } else { + write_back.put_object(&req, body_stream(b"source")).await + }; + assert!( + matches!(rejected, Err(WriteBackError::Local(ref error)) if error.contains("PreconditionFailed")), + "{rejected:?}" + ); + let retained = stored_object(&store, &bucket, "deleted").await; + assert!(retained.delete_marker); + assert_eq!(retained.version_id, marker.version_id); + req.respect_delete_marker = false; + write_back + .put_object(&req, body_stream(b"source")) + .await + .expect("explicit revival policy"); + assert!(!stored_object(&store, &bucket, "deleted").await.delete_marker); + } + } + #[tokio::test] #[serial_test::serial] async fn write_back_truncated_stream_leaves_nothing_behind() { diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index 86d6e255a..443605791 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -949,7 +949,11 @@ pub(super) enum PutObjectOrigin<'a> { /// request and no credential: managed-SSE authorization treats the write /// as internal, and the creation event, when requested, names /// `principal_id` instead of an access key. - Internal { principal_id: &'static str, emit_events: bool }, + Internal { + principal_id: &'static str, + emit_events: bool, + preserve_delete_marker: bool, + }, } impl PutObjectOrigin<'_> { @@ -1603,6 +1607,12 @@ impl DefaultObjectUsecase { if let Some(etag) = preserve_etag { opts.preserve_etag = Some(etag); } + if let PutObjectOrigin::Internal { + preserve_delete_marker, .. + } = &origin + { + opts.preserve_delete_marker = *preserve_delete_marker; + } if let Some(quota_check) = quota_check.as_ref() { apply_quota_admission(&mut opts, quota_check)?; } @@ -1769,6 +1779,7 @@ impl DefaultObjectUsecase { PutObjectOrigin::Internal { principal_id, emit_events, + .. } => { let principal_id = *principal_id; let request_context = request_context::RequestContext::fallback(); diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs index ffc9e7db2..454cd9911 100644 --- a/rustfs/src/app/object/shared.rs +++ b/rustfs/src/app/object/shared.rs @@ -1014,11 +1014,44 @@ pub(crate) fn mark_on_demand_migration_list_local_only(headers: &mut HeaderMap) /// forwarded to the source: a 304/412 answered by the source would be /// indistinguishable from a source failure. pub(crate) fn odm_check_source_preconditions(headers: &HeaderMap, head: &SourceHead) -> S3Result<()> { + let if_match = headers + .get(http::header::IF_MATCH) + .and_then(|value| value.to_str().ok()) + .map(str::trim); + let if_none_match = headers + .get(http::header::IF_NONE_MATCH) + .and_then(|value| value.to_str().ok()) + .map(str::trim); + let needs_etag = if_match.is_some_and(|value| value != "*") || if_none_match.is_some_and(|value| value != "*"); + let needs_mtime = (!headers.contains_key(http::header::IF_MATCH) && headers.contains_key(http::header::IF_UNMODIFIED_SINCE)) + || (!headers.contains_key(http::header::IF_NONE_MATCH) && headers.contains_key(http::header::IF_MODIFIED_SINCE)); + if (needs_etag && head.etag.is_none()) || (needs_mtime && head.last_modified.is_none()) { + return Err(odm_source_unavailable_error("missing_source_validator")); + } let info = ObjectInfo { etag: head.etag.clone(), mod_time: head.last_modified.map(OffsetDateTime::from), ..Default::default() }; + // A successful source read establishes wildcard existence, but the + // remaining conditions must still run in their ordinary precedence. + if head.etag.is_none() && (if_match == Some("*") || if_none_match == Some("*")) { + let mut remaining = headers.clone(); + if if_match == Some("*") { + remaining.remove(http::header::IF_MATCH); + remaining.remove(http::header::IF_UNMODIFIED_SINCE); + } + if if_none_match == Some("*") { + remaining.remove(http::header::IF_NONE_MATCH); + remaining.remove(http::header::IF_MODIFIED_SINCE); + } + check_preconditions(&remaining, &info)?; + return if if_none_match == Some("*") { + Err(S3Error::new(S3ErrorCode::NotModified)) + } else { + Ok(()) + }; + } check_preconditions(headers, &info) } @@ -2075,8 +2108,42 @@ mod on_demand_migration_tests { .expect_err("modified since an earlier date is 412"); assert_eq!(err.code(), &S3ErrorCode::PreconditionFailed); - // A source without validators cannot fail a precondition. let bare = SourceHead::default(); - assert!(odm_check_source_preconditions(&headers_with(http::header::IF_MATCH, "\"other\""), &bare).is_ok()); + let err = + odm_check_source_preconditions(&headers_with(http::header::IF_MATCH, "\"other\""), &bare).expect_err("missing ETag"); + assert_eq!(err.status_code(), Some(http::StatusCode::FAILED_DEPENDENCY)); + assert!(odm_check_source_preconditions(&headers_with(http::header::IF_MATCH, "*"), &bare).is_ok()); + let err = + odm_check_source_preconditions(&headers_with(http::header::IF_NONE_MATCH, "*"), &bare).expect_err("source exists"); + assert_eq!(err.code(), &S3ErrorCode::NotModified); + let dated = SourceHead { + last_modified: head.last_modified, + ..Default::default() + }; + assert!(odm_check_source_preconditions(&headers_with(http::header::IF_MATCH, "*"), &dated).is_ok()); + let mut combined = headers_with(http::header::IF_NONE_MATCH, "*"); + combined.insert(http::header::IF_MATCH, HeaderValue::from_static("\"other\"")); + assert_eq!( + odm_check_source_preconditions(&combined, &dated) + .expect_err("specific ETag unavailable") + .status_code(), + Some(http::StatusCode::FAILED_DEPENDENCY) + ); + combined.remove(http::header::IF_MATCH); + combined.insert( + http::header::IF_UNMODIFIED_SINCE, + HeaderValue::from_static("Wed, 21 Oct 2015 07:28:00 GMT"), + ); + assert_eq!( + odm_check_source_preconditions(&combined, &dated) + .expect_err("unmodified-since fails before none-match") + .code(), + &S3ErrorCode::PreconditionFailed + ); + for header in [http::header::IF_MODIFIED_SINCE, http::header::IF_UNMODIFIED_SINCE] { + let err = odm_check_source_preconditions(&headers_with(header, "Wed, 21 Oct 2015 07:28:00 GMT"), &bare) + .expect_err("missing timestamp"); + assert_eq!(err.status_code(), Some(http::StatusCode::FAILED_DEPENDENCY)); + } } } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 111726af3..7278991c5 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -2131,9 +2131,9 @@ impl Node for NodeService { ) .map_err(|err| Status::failed_precondition(err.to_string()))?; } - let namespace_generation = store.scanner_namespace_mutation_generation(); let topology_digest = rustfs_scanner::scanner_topology_digest(store.as_ref()); let (data_movement_active, publication_blocked, movement_generation) = store.scanner_data_movement_activity().await; + let namespace_generation = store.scanner_namespace_mutation_generation(); let mut response = match request_protocol { SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION | SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => { previous_scanner_activity_response(namespace_generation, topology_digest, data_movement_active) @@ -6264,6 +6264,91 @@ mod tests { assert_eq!(unavailable.code(), tonic::Code::Unavailable); } + #[tokio::test] + async fn scanner_activity_samples_namespace_generation_after_waiting_for_movement_state() { + use crate::storage::storage_api::{ObjectOptions, PutObjReader, contract::object::ObjectIO as _}; + + let _ = rustfs_credentials::set_global_rpc_secret("scanner-activity-generation-test-secret".to_string()); + let _ = rustfs_credentials::init_global_action_credentials( + Some("TESTROOTACCESSKEY".to_string()), + Some("TESTROOTSECRET123".to_string()), + ); + let temp_dir = tempfile::tempdir().expect("scanner activity RPC test directory"); + let env = rustfs_test_utils::TestECStoreEnv::builder() + .base_dir(temp_dir.path()) + .build() + .await; + ObjectStore::new(Arc::clone(&env.ecstore)) + .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) + .await + .expect("seed IAM format"); + let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore)) + .await + .expect("build isolated IAM"); + let context = Arc::new(crate::runtime_sources::AppContext::with_default_interfaces( + Arc::clone(&env.ecstore), + iam, + Arc::new(KmsServiceManager::new()), + )); + let service = make_server_for_context(Some(context)); + let bucket = "scanner-activity-generation"; + env.make_bucket(bucket, false).await; + let generation_before = env.ecstore.scanner_namespace_mutation_generation(); + let mut request = Request::new(ScannerActivityRequest { + challenge: vec![7; 16].into(), + protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION, + acknowledge_instance_id: String::new(), + acknowledge_dirty_usage_generation: 0, + }); + let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref()) + .expect("scanner activity request should encode"); + set_tonic_canonical_body_digest(&mut request, &canonical).expect("digest metadata should encode"); + mark_v2_authenticated(&mut request); + + let pool_meta = env.ecstore.pool_meta.write().await; + drop( + env.ecstore + .decommission_cancelers + .try_write() + .expect("movement snapshot should not hold the cancelers before the RPC"), + ); + let mut activity = Box::pin(tokio::task::unconstrained(service.scanner_activity(request))); + assert!(futures::poll!(activity.as_mut()).is_pending()); + assert!( + env.ecstore.decommission_cancelers.try_write().is_err(), + "the RPC must hold the cancelers read guard while waiting for pool metadata" + ); + + // Select the existing set directly: ECStore pool selection reads the lock held by this test. + let mut reader = PutObjReader::from_vec(b"namespace changed during activity probe".to_vec()); + tokio::time::timeout( + Duration::from_secs(30), + env.ecstore.pools[0].disk_set[0].put_object( + bucket, + "object", + &mut reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ), + ) + .await + .expect("the namespace mutation must not wait for the RPC's pool lock") + .expect("the namespace mutation must complete while the RPC waits"); + let generation_after = env.ecstore.scanner_namespace_mutation_generation(); + assert!(generation_after > generation_before); + drop(pool_meta); + + let response = tokio::time::timeout(Duration::from_secs(30), activity) + .await + .expect("scanner activity RPC should resume after the pool lock is released") + .expect("authenticated scanner activity RPC should succeed") + .into_inner(); + assert_eq!(response.namespace_generation, generation_after); + assert_eq!(response.publication_blocked, Some(false)); + } + #[tokio::test] async fn test_scanner_dirty_usage_snapshot_requires_body_bound_auth_and_signs_a_consistent_view() { let _ = rustfs_credentials::set_global_rpc_secret("scanner-dirty-usage-snapshot-test-secret".to_string()); From 188f380b3b18af934c41e5ea364ab3b313cf6f8e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 22:06:30 +0800 Subject: [PATCH 39/40] feat(ecstore): add native azure blob and gcs migration sources (#7211) * feat(ecstore): add a native azure blob odm source backend * feat(ecstore): add a native gcs odm source backend and one backend contract * fix(ecstore): refuse an empty azure account key at client build * fix(ecstore): probe gcs sources with the listing permission * fix(app): drop a redundant match guard on the sse config lookup * fix(ecstore): drop stale rename commit duplicates from local.rs * test(ecstore): use the sanctioned placeholder key in the gcs fixture --- Cargo.lock | 1 + crates/ecstore/Cargo.toml | 1 + crates/ecstore/src/api/mod.rs | 15 +- .../src/bucket/on_demand_migration/azure.rs | 1046 +++++++++++++++++ .../on_demand_migration/backend_contract.rs | 172 +++ .../src/bucket/on_demand_migration/config.rs | 406 ++++++- .../src/bucket/on_demand_migration/gcs.rs | 506 ++++++++ .../src/bucket/on_demand_migration/mod.rs | 20 +- .../bucket/on_demand_migration/native_http.rs | 415 +++++++ .../src/bucket/on_demand_migration/pull.rs | 2 + .../on_demand_migration/source_client.rs | 226 +++- .../src/bucket/on_demand_migration/sys.rs | 33 +- .../on_demand_migration/test_http_fixture.rs | 120 ++ .../on_demand_migration/get_response.json | 2 +- .../on_demand_migration/set_request.json | 2 +- .../on_demand_migration/set_response.json | 2 +- crates/madmin/src/on_demand_migration.rs | 88 ++ docs/operations/on-demand-migration.md | 26 +- .../src/admin/handlers/on_demand_migration.rs | 6 + rustfs/src/admin/storage_api.rs | 1 + rustfs/src/app/object/get.rs | 2 + rustfs/src/app/object/head.rs | 3 + .../src/app/object/on_demand_migration_put.rs | 12 + 23 files changed, 3074 insertions(+), 33 deletions(-) create mode 100644 crates/ecstore/src/bucket/on_demand_migration/azure.rs create mode 100644 crates/ecstore/src/bucket/on_demand_migration/backend_contract.rs create mode 100644 crates/ecstore/src/bucket/on_demand_migration/gcs.rs create mode 100644 crates/ecstore/src/bucket/on_demand_migration/native_http.rs create mode 100644 crates/ecstore/src/bucket/on_demand_migration/test_http_fixture.rs diff --git a/Cargo.lock b/Cargo.lock index f4cc41aae..dc72ece63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9792,6 +9792,7 @@ dependencies = [ "path-absolutize", "pin-project-lite", "proptest", + "quick-xml", "rand 0.10.2", "ratelimit", "rcgen", diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index d36c9d85b..dd2063129 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -215,6 +215,7 @@ serde_urlencoded.workspace = true google-cloud-storage = { workspace = true } google-cloud-auth = { workspace = true } faster-hex = { workspace = true } +quick-xml = { workspace = true } ratelimit = { workspace = true } aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index fd66c897a..e752713ff 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -153,12 +153,13 @@ pub mod bucket { LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup, OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason, PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS, - SourceLatencySnapshot, source_client_spec, + SourceLatencySnapshot, source_backend_spec, source_client_spec, }; pub use crate::bucket::on_demand_migration::{ - ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION, - OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, - SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext, + AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, + ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, + Provider, RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, + ValidationContext, }; pub use crate::bucket::on_demand_migration::{ EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, @@ -185,9 +186,9 @@ pub mod bucket { } pub mod source_client { pub use crate::bucket::on_demand_migration::source_client::{ - SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, - SourceProbe, SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value, - resolve_path_style, + AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError, + SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceProbe, SourceProvider, SourceSse, + SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value, resolve_path_style, }; } } diff --git a/crates/ecstore/src/bucket/on_demand_migration/azure.rs b/crates/ecstore/src/bucket/on_demand_migration/azure.rs new file mode 100644 index 000000000..08e687797 --- /dev/null +++ b/crates/ecstore/src/bucket/on_demand_migration/azure.rs @@ -0,0 +1,1046 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Native Azure Blob source backend. +//! +//! Azure has no S3 API, so this backend speaks the Blob REST service directly: +//! Get Blob / Get Blob Properties for the read path, List Blobs for the listing, +//! Get Blob Tags for the tags and Get Container Properties for the probe. The +//! local key is the blob name inside the container named by `source.bucket`. +//! +//! Two authorization schemes are supported, matching the two forms an operator +//! can hold: a storage-account key signed per request with Shared Key, and a SAS +//! token appended to the request query. +//! +//! Azure's ETag is a concurrency token, not a digest of the bytes, so every head +//! this backend produces is marked [`SourceHead::etag_is_opaque`]: the write-back +//! path records the value for provenance and refuses to check content against it. +//! `Content-MD5` is the only Azure digest, it is optional per blob, and it is not +//! mapped onto the ETag slot precisely so that the two never get confused. +//! +//! The anti-loop `source-proxy-request` markers the S3 backend sends are omitted: +//! they mean something only to a RustFS or MinIO source, and Azure would have to +//! carry them through Shared Key canonicalization for no gain. + +use super::native_http::{ + NativeHeadFields, NativeHttp, header, native_source_head, parse_http_timestamp, read_text, response_body, +}; +use super::source_client::{ + AzureAuth, AzureSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, + SourceTimeouts, range_header_value, +}; +use crate::bucket::remote_s3_client::RemoteS3ClientError; +use crate::storage_api_contracts::range::HTTPRangeSpec; +use hmac::{Hmac, Mac, digest::KeyInit}; +use http::{HeaderMap, HeaderValue, Method}; +use quick_xml::Reader; +use quick_xml::events::Event; +use sha2::Sha256; +use std::collections::{BTreeMap, HashMap}; +use url::Url; + +type HmacSha256 = Hmac; + +/// Blob REST version this backend pins. Every response field it reads exists +/// from this version on, including blob tags and blob versioning. +const API_VERSION: &str = "2021-08-06"; +const HEADER_VERSION: &str = "x-ms-version"; +const HEADER_DATE: &str = "x-ms-date"; +const HEADER_ERROR_CODE: &str = "x-ms-error-code"; +const METADATA_PREFIX: &str = "x-ms-meta-"; +/// A List Blobs or Get Blob Tags response is small; refuse a source that +/// streams an unbounded document at us instead of buffering it. +const MAX_XML_BYTES: usize = 8 * 1024 * 1024; + +/// `Sun, 06 Nov 1994 08:49:37 GMT`, the only `x-ms-date` form Azure accepts. +const HTTP_DATE: &[time::format_description::BorrowedFormatItem<'static>] = + time::macros::format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT"); + +enum Credential { + /// Decoded storage-account key. + SharedKey(Vec), + /// SAS parameters, decoded once so re-encoding cannot double-escape them. + Sas(Vec<(String, String)>), +} + +pub struct AzureSourceBackend { + http: NativeHttp, + account: String, + container: String, + credential: Credential, +} + +impl AzureSourceBackend { + pub fn new( + endpoint: &str, + container: &str, + spec: &AzureSourceSpec, + timeouts: SourceTimeouts, + skip_tls_verify: bool, + ca_cert_pem: Option<&str>, + ) -> Result { + let credential = match &spec.auth { + AzureAuth::SharedKey(key) => { + let key = base64_simd::STANDARD + .decode_to_vec(key.as_bytes()) + .map_err(|_| RemoteS3ClientError::Credentials("azure account key is not base64"))?; + // HMAC accepts a zero-length key, so an absent one would sign + // every request with nothing rather than fail here. + if key.is_empty() { + return Err(RemoteS3ClientError::Credentials("azure account key is empty")); + } + Credential::SharedKey(key) + } + AzureAuth::Sas(sas) => { + let pairs: Vec<(String, String)> = url::form_urlencoded::parse(sas.trim_start_matches('?').as_bytes()) + .into_owned() + .collect(); + if pairs.is_empty() { + return Err(RemoteS3ClientError::Credentials("azure sas token has no parameters")); + } + Credential::Sas(pairs) + } + }; + Ok(Self { + http: NativeHttp::new(endpoint, timeouts, skip_tls_verify, ca_cert_pem)?, + account: spec.account.clone(), + container: container.to_string(), + credential, + }) + } + + /// URL of one blob in the container. The key is split so its `/` stay path + /// separators while every other character is percent-encoded. + fn blob_url(&self, key: &str) -> Result { + self.http.url(std::iter::once(self.container.as_str()).chain(key.split('/'))) + } + + fn container_url(&self) -> Result { + self.http.url(std::iter::once(self.container.as_str())) + } + + /// Builds a signed (or SAS-carrying) request. `headers` holds the + /// operation's own headers; the service headers and authorization are + /// added here so every request is authorized the same way. + fn request(&self, method: Method, mut url: Url, mut headers: HeaderMap) -> Result { + headers.insert(HEADER_VERSION, HeaderValue::from_static(API_VERSION)); + let now = time::OffsetDateTime::now_utc() + .format(HTTP_DATE) + .map_err(|err| SourceError::Other(format!("cannot render the request date: {err}")))?; + headers.insert( + HEADER_DATE, + HeaderValue::from_str(&now).map_err(|_| SourceError::Other("cannot render the request date".to_string()))?, + ); + + match &self.credential { + Credential::SharedKey(key) => { + let signature = shared_key_signature(key, &self.account, method.as_str(), &url, &headers)?; + headers.insert( + http::header::AUTHORIZATION, + HeaderValue::from_str(&format!("SharedKey {}:{signature}", self.account)) + .map_err(|_| SourceError::Other("cannot render the authorization header".to_string()))?, + ); + } + Credential::Sas(pairs) => { + url.query_pairs_mut().extend_pairs(pairs.iter().map(|(k, v)| (k, v))); + } + } + + let mut request = reqwest::Request::new(method, url); + *request.headers_mut() = headers; + Ok(request) + } + + /// Shared mapping for Get Blob and Get Blob Properties. + fn head_from_response(headers: &HeaderMap) -> Result { + // A customer-provided key means the service holds ciphertext it cannot + // decrypt for us; the same rule the S3 path applies to SSE-C. + if header(headers, "x-ms-encryption-key-sha256").is_some() { + return Err(SourceError::Unsupported( + "source blob uses a customer-provided encryption key; customer-key sources are not supported".to_string(), + )); + } + native_source_head( + headers, + METADATA_PREFIX, + NativeHeadFields { + etag: header(headers, "etag").map(str::to_string), + etag_is_opaque: true, + version_id: header(headers, "x-ms-version-id").map(str::to_string), + storage_class: header(headers, "x-ms-access-tier").map(str::to_string), + }, + ) + } +} + +#[async_trait::async_trait] +impl SourceBackend for AzureSourceBackend { + async fn head(&self, key: &str) -> Result { + let request = self.request(Method::HEAD, self.blob_url(key)?, HeaderMap::new())?; + let response = self.http.send(request, HEADER_ERROR_CODE).await?; + Self::head_from_response(response.headers()) + } + + async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result { + let mut headers = HeaderMap::new(); + if let Some(range) = range.map(range_header_value).transpose()? { + headers.insert( + http::header::RANGE, + HeaderValue::from_str(&range).map_err(|_| SourceError::Other("invalid range header".to_string()))?, + ); + } + let request = self.request(Method::GET, self.blob_url(key)?, headers)?; + let response = self.http.send(request, HEADER_ERROR_CODE).await?; + let head = Self::head_from_response(response.headers())?; + let content_range = header(response.headers(), "content-range").map(str::to_string); + Ok(SourceGet { + head, + body: response_body(response), + content_range, + }) + } + + async fn list(&self, request: &SourceListRequest<'_>) -> Result { + // Azure paginates with an opaque marker and has no "start after this + // key" form. Refuse rather than silently listing from the beginning. + if request.start_after.is_some() { + return Err(SourceError::Unsupported( + "azure sources cannot resume a listing from a key; use the continuation token".to_string(), + )); + } + let mut url = self.container_url()?; + { + let mut query = url.query_pairs_mut(); + query.append_pair("restype", "container"); + query.append_pair("comp", "list"); + if let Some(prefix) = request.prefix.filter(|prefix| !prefix.is_empty()) { + query.append_pair("prefix", prefix); + } + if let Some(delimiter) = request.delimiter.filter(|delimiter| !delimiter.is_empty()) { + query.append_pair("delimiter", delimiter); + } + if let Some(marker) = request.continuation_token.filter(|marker| !marker.is_empty()) { + query.append_pair("marker", marker); + } + if request.max_keys > 0 { + query.append_pair("maxresults", &request.max_keys.to_string()); + } + } + + let request = self.request(Method::GET, url, HeaderMap::new())?; + let response = self.http.send(request, HEADER_ERROR_CODE).await?; + let body = read_text(response, MAX_XML_BYTES).await?; + let listing = parse_list_blobs(&body)?; + + Ok(SourcePage { + objects: listing.objects, + common_prefixes: listing.prefixes, + is_truncated: listing.next_marker.is_some(), + next_continuation_token: listing.next_marker, + }) + } + + async fn tagging(&self, key: &str) -> Result, SourceError> { + let mut url = self.blob_url(key)?; + url.query_pairs_mut().append_pair("comp", "tags"); + let request = self.request(Method::GET, url, HeaderMap::new())?; + let response = self.http.send(request, HEADER_ERROR_CODE).await?; + let body = read_text(response, MAX_XML_BYTES).await?; + parse_blob_tags(&body) + } + + async fn probe(&self) -> Result<(), SourceError> { + let mut url = self.container_url()?; + url.query_pairs_mut().append_pair("restype", "container"); + let request = self.request(Method::HEAD, url, HeaderMap::new())?; + self.http.send(request, HEADER_ERROR_CODE).await?; + Ok(()) + } +} + +/// Shared Key signature over the canonical request. Only the fields this +/// backend ever sets are non-empty: `Range`, the `x-ms-*` headers and the +/// canonicalized resource. GET and HEAD carry no body, so every `Content-*` +/// slot stays empty. +fn shared_key_signature(key: &[u8], account: &str, method: &str, url: &Url, headers: &HeaderMap) -> Result { + let mut string_to_sign = String::with_capacity(256); + string_to_sign.push_str(method); + string_to_sign.push('\n'); + // Content-Encoding, Content-Language, Content-Length, Content-MD5, + // Content-Type, Date, If-Modified-Since, If-Match, If-None-Match, + // If-Unmodified-Since: all empty. `Date` stays empty because `x-ms-date` + // carries the timestamp and Azure then ignores this slot. + for _ in 0..10 { + string_to_sign.push('\n'); + } + string_to_sign.push_str(header(headers, "range").unwrap_or_default()); + string_to_sign.push('\n'); + + // Canonicalized headers: every `x-ms-*` header, lowercased and sorted. + let mut canonical_headers = BTreeMap::new(); + for (name, value) in headers { + let name = name.as_str(); + if let Some(rest) = name.strip_prefix("x-ms-") + && !rest.is_empty() + && let Ok(value) = value.to_str() + { + canonical_headers.insert(name.to_string(), value.trim().to_string()); + } + } + for (name, value) in &canonical_headers { + string_to_sign.push_str(name); + string_to_sign.push(':'); + string_to_sign.push_str(value); + string_to_sign.push('\n'); + } + + // Canonicalized resource: the account, the encoded path, then every query + // parameter lowercased and sorted, with repeated values joined by commas. + string_to_sign.push('/'); + string_to_sign.push_str(account); + string_to_sign.push_str(url.path()); + let mut canonical_query: BTreeMap> = BTreeMap::new(); + for (name, value) in url.query_pairs() { + canonical_query + .entry(name.to_ascii_lowercase()) + .or_default() + .push(value.into_owned()); + } + for (name, mut values) in canonical_query { + values.sort(); + string_to_sign.push('\n'); + string_to_sign.push_str(&name); + string_to_sign.push(':'); + string_to_sign.push_str(&values.join(",")); + } + + let mut mac = HmacSha256::new_from_slice(key) + .map_err(|_| SourceError::Other("azure account key has an unusable length".to_string()))?; + mac.update(string_to_sign.as_bytes()); + Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes())) +} + +#[derive(Debug)] +struct AzureListing { + objects: Vec, + prefixes: Vec, + /// `None` when the listing is complete; Azure marks the end with an empty + /// `NextMarker`. + next_marker: Option, +} + +#[derive(Default)] +struct BlobEntry { + name: String, + etag: Option, + size: u64, + last_modified: Option, + access_tier: Option, +} + +/// Parses one `List Blobs` page. +fn parse_list_blobs(xml: &str) -> Result { + let mut reader = xml_reader(xml); + let mut objects = Vec::new(); + let mut prefixes = Vec::new(); + let mut next_marker = None; + let mut blob: Option = None; + let mut in_blob_prefix = false; + // Open container elements. quick-xml reports a truncated document as a + // plain end of input, so a non-zero depth at EOF is the only signal that + // the page was cut short and must not be read as a complete listing. + let mut depth = 0_usize; + + loop { + match reader.read_event() { + Ok(Event::Start(start)) => { + let name = local_name(start.name().as_ref()); + match name.as_str() { + "blob" => { + depth += 1; + blob = Some(BlobEntry::default()); + } + "blobprefix" => { + depth += 1; + in_blob_prefix = true; + } + "properties" | "blobs" | "enumerationresults" => depth += 1, + _ => { + let end = start.to_end().into_owned(); + let text = leaf_text(&mut reader, end.name())?; + apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix); + } + } + } + Ok(Event::Empty(empty)) => { + let name = local_name(empty.name().as_ref()); + apply_list_field(&name, String::new(), &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix); + } + Ok(Event::End(end)) => match local_name(end.name().as_ref()).as_str() { + "blob" => { + depth = depth.saturating_sub(1); + if let Some(entry) = blob.take() { + objects.push(SourceObject { + key: entry.name, + etag: entry.etag, + size: entry.size, + last_modified: entry.last_modified, + storage_class: entry.access_tier, + // Azure ETags carry no part count; the listing + // never describes a composed object. + is_multipart_etag: false, + }); + } + } + "blobprefix" => { + depth = depth.saturating_sub(1); + in_blob_prefix = false; + } + "properties" | "blobs" | "enumerationresults" => depth = depth.saturating_sub(1), + _ => {} + }, + Ok(Event::Eof) => break, + Ok(_) => {} + Err(err) => return Err(SourceError::Other(format!("source listing is not valid XML: {err}"))), + } + } + if depth != 0 { + return Err(SourceError::Other("source listing ended before every element was closed".to_string())); + } + + Ok(AzureListing { + objects, + prefixes, + next_marker: next_marker.filter(|marker| !marker.is_empty()), + }) +} + +fn apply_list_field( + name: &str, + text: String, + blob: &mut Option, + prefixes: &mut Vec, + next_marker: &mut Option, + in_blob_prefix: bool, +) { + match name { + "name" => { + if in_blob_prefix { + prefixes.push(text); + } else if let Some(entry) = blob.as_mut() { + entry.name = text; + } + } + "nextmarker" => *next_marker = Some(text), + "etag" => { + if let Some(entry) = blob.as_mut() { + entry.etag = Some(text.trim().trim_matches('"').to_string()).filter(|etag| !etag.is_empty()); + } + } + "content-length" => { + if let Some(entry) = blob.as_mut() { + entry.size = text.trim().parse().unwrap_or(0); + } + } + "last-modified" => { + if let Some(entry) = blob.as_mut() { + entry.last_modified = parse_http_timestamp(text.trim()); + } + } + "accesstier" => { + if let Some(entry) = blob.as_mut() { + entry.access_tier = Some(text).filter(|tier| !tier.is_empty()); + } + } + _ => {} + } +} + +/// Parses a `Get Blob Tags` response. +fn parse_blob_tags(xml: &str) -> Result, SourceError> { + let mut reader = xml_reader(xml); + let mut tags = HashMap::new(); + let mut key = None; + let mut value = None; + let mut depth = 0_usize; + + loop { + match reader.read_event() { + Ok(Event::Start(start)) => { + let name = local_name(start.name().as_ref()); + match name.as_str() { + "tags" | "tagset" | "tag" => depth += 1, + _ => { + let end = start.to_end().into_owned(); + let text = leaf_text(&mut reader, end.name())?; + match name.as_str() { + "key" => key = Some(text), + "value" => value = Some(text), + _ => {} + } + } + } + } + Ok(Event::Empty(empty)) => match local_name(empty.name().as_ref()).as_str() { + "key" => key = Some(String::new()), + "value" => value = Some(String::new()), + _ => {} + }, + Ok(Event::End(end)) => { + let name = local_name(end.name().as_ref()); + if matches!(name.as_str(), "tags" | "tagset" | "tag") { + depth = depth.saturating_sub(1); + } + if name == "tag" + && let (Some(key), Some(value)) = (key.take(), value.take()) + { + tags.insert(key, value); + } + } + Ok(Event::Eof) => break, + Ok(_) => {} + Err(err) => return Err(SourceError::Other(format!("source tags are not valid XML: {err}"))), + } + } + if depth != 0 { + return Err(SourceError::Other("source tags ended before every element was closed".to_string())); + } + + Ok(tags) +} + +fn xml_reader(xml: &str) -> Reader<&[u8]> { + let mut reader = Reader::from_str(xml); + let config = reader.config_mut(); + config.trim_text_start = true; + config.trim_text_end = true; + reader +} + +/// Lowercased element name without its namespace prefix. +fn local_name(raw: &str) -> String { + raw.rsplit(':').next().unwrap_or(raw).to_ascii_lowercase() +} + +/// Text of a leaf element, consuming through its end tag. +fn leaf_text(reader: &mut Reader<&[u8]>, end: quick_xml::name::QName<'_>) -> Result { + let raw = reader + .read_text(end) + .map_err(|err| format!("source response is not valid XML: {err}")) + .and_then(|text| { + quick_xml::escape::unescape(text.as_ref()) + .map(|text| text.into_owned()) + .map_err(|err| format!("source response has invalid XML escapes: {err}")) + }); + raw.map_err(SourceError::Other) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract}; + use crate::bucket::on_demand_migration::source_client::SourceError; + use crate::bucket::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server}; + + const LIST_PAGE: &str = r#" + + photos/ + / + 2 + + + photos/a & b.jpg + + Wed, 21 Oct 2015 07:28:00 GMT + 0x8D2F1B0A1B2C3D4 + 42 + 1B2M2Y8AsgTpgAmY7PhCfg== + BlockBlob + Hot + + + + photos/b.jpg + + 7 + + + + photos/raw/ + + + 2!76!MDAwMDI0 +"#; + + const LAST_PAGE: &str = r#" +only.txt1"#; + + const TAGS: &str = r#" + + envprod + teamstorage & co +"#; + + #[test] + fn list_blobs_maps_entries_prefixes_and_the_marker() { + let listing = parse_list_blobs(LIST_PAGE).expect("page should parse"); + assert_eq!(listing.prefixes, vec!["photos/raw/"]); + assert_eq!(listing.next_marker.as_deref(), Some("2!76!MDAwMDI0")); + assert_eq!(listing.objects.len(), 2); + let first = &listing.objects[0]; + assert_eq!(first.key, "photos/a & b.jpg", "XML entities in a blob name are decoded"); + assert_eq!(first.etag.as_deref(), Some("0x8D2F1B0A1B2C3D4")); + assert_eq!(first.size, 42); + assert_eq!( + first.last_modified, + Some(std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_445_412_480)) + ); + assert_eq!(first.storage_class.as_deref(), Some("Hot")); + assert!(!first.is_multipart_etag); + assert_eq!(listing.objects[1].key, "photos/b.jpg"); + assert_eq!(listing.objects[1].size, 7); + assert!(listing.objects[1].etag.is_none()); + } + + #[test] + fn an_empty_next_marker_ends_the_listing() { + let listing = parse_list_blobs(LAST_PAGE).expect("page should parse"); + assert_eq!(listing.objects.len(), 1); + assert!(listing.next_marker.is_none(), "an empty NextMarker is not a cursor"); + } + + #[test] + fn malformed_listing_xml_is_an_error() { + for bad in [ + "", + "a", + "not xml at ").is_err(), "a truncated tag set must fail"); + } + + #[test] + fn blob_tags_parse_into_the_shared_tag_map() { + let tags = parse_blob_tags(TAGS).expect("tags should parse"); + assert_eq!( + tags, + HashMap::from([ + ("env".to_string(), "prod".to_string()), + ("team".to_string(), "storage & co".to_string()) + ]) + ); + assert!(parse_blob_tags("").expect("empty tag set").is_empty()); + } + + /// Signature fixture from a request this backend really builds: it pins the + /// canonical form so a change to the header set or the query canonicalization + /// cannot silently start producing signatures Azure rejects. + #[test] + fn shared_key_signs_the_canonical_request() { + let key = base64_simd::STANDARD.decode_to_vec(b"c2VjcmV0LWtleQ==").expect("test key"); + let mut headers = HeaderMap::new(); + headers.insert(HEADER_VERSION, HeaderValue::from_static(API_VERSION)); + headers.insert(HEADER_DATE, HeaderValue::from_static("Sun, 06 Nov 1994 08:49:37 GMT")); + headers.insert(http::header::RANGE, HeaderValue::from_static("bytes=10-14")); + let url = Url::parse("https://acct.blob.core.windows.net/legacy/photos/a.jpg").expect("url"); + + let signature = shared_key_signature(&key, "acct", "GET", &url, &headers).expect("signature"); + let expected = { + let string_to_sign = concat!( + "GET\n\n\n\n\n\n\n\n\n\n\n", + "bytes=10-14\n", + "x-ms-date:Sun, 06 Nov 1994 08:49:37 GMT\n", + "x-ms-version:2021-08-06\n", + "/acct/legacy/photos/a.jpg" + ); + let mut mac = HmacSha256::new_from_slice(&key).expect("hmac"); + mac.update(string_to_sign.as_bytes()); + base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()) + }; + assert_eq!(signature, expected); + } + + #[test] + fn shared_key_canonicalizes_query_parameters() { + let key = vec![1_u8; 32]; + let mut headers = HeaderMap::new(); + headers.insert(HEADER_VERSION, HeaderValue::from_static(API_VERSION)); + // Query order must not change the signature: Azure canonicalizes by + // lowercased parameter name. + let a = Url::parse("https://acct.blob.core.windows.net/legacy?restype=container&comp=list&prefix=p%2F").expect("url"); + let b = Url::parse("https://acct.blob.core.windows.net/legacy?prefix=p%2F&COMP=list&restype=container").expect("url"); + assert_eq!( + shared_key_signature(&key, "acct", "GET", &a, &headers).expect("a"), + shared_key_signature(&key, "acct", "GET", &b, &headers).expect("b") + ); + } + + fn backend(endpoint: &Url, credential: Credential) -> AzureSourceBackend { + AzureSourceBackend { + http: NativeHttp::for_test(endpoint.clone()), + account: "acct".to_string(), + container: "legacy".to_string(), + credential, + } + } + + fn blob_headers() -> Vec<(&'static str, String)> { + vec![ + ("ETag", "\"0x8D2F1B0A1B2C3D4\"".to_string()), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()), + ("Content-Type", "image/jpeg".to_string()), + ("Content-MD5", "1B2M2Y8AsgTpgAmY7PhCfg==".to_string()), + ("x-ms-meta-owner", "alice".to_string()), + ("x-ms-access-tier", "Cool".to_string()), + ("x-ms-version-id", "2026-01-01T00:00:00.0000000Z".to_string()), + ("x-ms-blob-type", "BlockBlob".to_string()), + ] + } + + #[test] + fn an_absent_or_malformed_account_key_is_refused_before_any_request() { + for key in ["", "not base64!"] { + let spec = AzureSourceSpec { + account: "acct".to_string(), + auth: AzureAuth::SharedKey(key.to_string()), + }; + let built = AzureSourceBackend::new( + "https://acct.blob.core.windows.net", + "legacy", + &spec, + SourceTimeouts::default(), + false, + None, + ); + assert!( + matches!(built, Err(RemoteS3ClientError::Credentials(_))), + "{key:?} must not build a client" + ); + } + let spec = AzureSourceSpec { + account: "acct".to_string(), + auth: AzureAuth::Sas(String::new()), + }; + assert!( + AzureSourceBackend::new( + "https://acct.blob.core.windows.net", + "legacy", + &spec, + SourceTimeouts::default(), + false, + None + ) + .is_err(), + "an empty SAS token carries no parameters" + ); + } + + #[tokio::test] + async fn head_signs_the_request_and_maps_azure_metadata() { + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, blob_headers(), String::new())]).await; + let backend = backend(&endpoint, Credential::SharedKey(b"0123456789abcdef0123456789abcdef".to_vec())); + + let head = backend.head("photos/a b.jpg").await.expect("HEAD should map"); + + let recorded = recorded.lock().expect("recorder lock").clone(); + assert_eq!(recorded.len(), 1); + assert_eq!(recorded[0].method, "HEAD"); + assert_eq!(recorded[0].target, "/legacy/photos/a%20b.jpg", "the blob name is path-encoded"); + assert_eq!(recorded[0].header("x-ms-version"), Some(API_VERSION)); + assert!(recorded[0].header("x-ms-date").is_some(), "a signed request must carry x-ms-date"); + assert!( + recorded[0] + .header("authorization") + .is_some_and(|value| value.starts_with("SharedKey acct:")), + "{:?}", + recorded[0].header("authorization") + ); + + assert_eq!(head.etag.as_deref(), Some("0x8D2F1B0A1B2C3D4")); + assert!(head.etag_is_opaque, "an Azure ETag is never a content digest"); + assert!(!head.is_multipart_etag); + assert_eq!(head.size, 0); + assert_eq!(head.content_type.as_deref(), Some("image/jpeg")); + assert_eq!(head.storage_class.as_deref(), Some("Cool")); + assert_eq!(head.version_id.as_deref(), Some("2026-01-01T00:00:00.0000000Z")); + assert_eq!(head.user_metadata, HashMap::from([("owner".to_string(), "alice".to_string())])); + assert!(head.sse.is_none()); + } + + #[tokio::test] + async fn sas_credentials_travel_in_the_query_and_never_sign() { + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, blob_headers(), String::new())]).await; + let backend = backend( + &endpoint, + Credential::Sas(vec![ + ("sv".to_string(), "2021-08-06".to_string()), + ("sig".to_string(), "a+b/c=".to_string()), + ]), + ); + + backend.head("a.txt").await.expect("HEAD should map"); + + let recorded = recorded.lock().expect("recorder lock").clone(); + assert!(recorded[0].header("authorization").is_none(), "a SAS request must not be signed"); + assert!(recorded[0].target.contains("sv=2021-08-06"), "{}", recorded[0].target); + assert!( + recorded[0].target.contains("sig=a%2Bb%2Fc%3D"), + "the SAS signature must be re-encoded exactly once: {}", + recorded[0].target + ); + } + + #[tokio::test] + async fn get_passes_the_range_through_and_streams_the_body() { + let mut headers = blob_headers(); + headers.push(("Content-Range", "bytes 10-14/100".to_string())); + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(206, headers, "hello".to_string())]).await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + + let range = HTTPRangeSpec { + is_suffix_length: false, + start: 10, + end: 14, + }; + let got = backend.get("a.txt", Some(&range)).await.expect("ranged GET should succeed"); + + let recorded = recorded.lock().expect("recorder lock").clone(); + assert_eq!(recorded[0].method, "GET"); + assert_eq!(recorded[0].header("range"), Some("bytes=10-14")); + assert_eq!(got.content_range.as_deref(), Some("bytes 10-14/100")); + assert_eq!(got.head.size, 5); + let body = got.body.collect().await.expect("body should stream").into_bytes(); + assert_eq!(body.as_ref(), b"hello"); + } + + #[tokio::test] + async fn customer_key_blobs_are_refused() { + let mut headers = blob_headers(); + headers.push(("x-ms-encryption-key-sha256", "abc".to_string())); + let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + + let err = backend.head("a.txt").await.expect_err("customer-key blobs are unsupported"); + assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}"); + assert_eq!(err.class_label(), "unsupported"); + assert!(!err.is_retryable()); + } + + #[tokio::test] + async fn list_requests_the_container_and_pages_with_the_marker() { + let (endpoint, recorded) = scripted_server(vec![ + ScriptedResponse::new(200, Vec::new(), LIST_PAGE.to_string()), + ScriptedResponse::new(200, Vec::new(), LAST_PAGE.to_string()), + ]) + .await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + + let page = backend + .list(&SourceListRequest { + prefix: Some("photos/"), + delimiter: Some("/"), + max_keys: 2, + ..Default::default() + }) + .await + .expect("first page should list"); + assert!(page.is_truncated); + assert_eq!(page.next_continuation_token.as_deref(), Some("2!76!MDAwMDI0")); + assert_eq!(page.common_prefixes, vec!["photos/raw/"]); + + let page = backend + .list(&SourceListRequest { + prefix: Some("photos/"), + continuation_token: page.next_continuation_token.as_deref(), + max_keys: 2, + ..Default::default() + }) + .await + .expect("second page should list"); + assert!(!page.is_truncated); + assert!(page.next_continuation_token.is_none()); + + let recorded = recorded.lock().expect("recorder lock").clone(); + for request in &recorded { + assert!(request.target.starts_with("/legacy?"), "{}", request.target); + assert!(request.target.contains("restype=container"), "{}", request.target); + assert!(request.target.contains("comp=list"), "{}", request.target); + assert!(request.target.contains("prefix=photos%2F"), "{}", request.target); + assert!(request.target.contains("maxresults=2"), "{}", request.target); + } + assert!(!recorded[0].target.contains("marker="), "{}", recorded[0].target); + assert!(recorded[1].target.contains("marker=2%2176%21MDAwMDI0"), "{}", recorded[1].target); + } + + #[tokio::test] + async fn list_refuses_a_start_after_cursor_before_sending() { + let (endpoint, recorded) = scripted_server(Vec::new()).await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + + let err = backend + .list(&SourceListRequest { + start_after: Some("a"), + max_keys: 1, + ..Default::default() + }) + .await + .expect_err("azure has no start-after form"); + assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}"); + assert!( + recorded.lock().expect("recorder lock").is_empty(), + "an unsupported request must never reach the source" + ); + } + + #[tokio::test] + async fn tagging_and_probe_address_the_right_resources() { + let (endpoint, recorded) = scripted_server(vec![ + ScriptedResponse::new(200, Vec::new(), TAGS.to_string()), + ScriptedResponse::new(200, Vec::new(), String::new()), + ]) + .await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + + let tags = backend.tagging("a.txt").await.expect("tags should parse"); + assert_eq!(tags.get("env").map(String::as_str), Some("prod")); + backend.probe().await.expect("probe should succeed"); + + let recorded = recorded.lock().expect("recorder lock").clone(); + assert_eq!(recorded[0].target, "/legacy/a.txt?comp=tags"); + assert_eq!(recorded[1].method, "HEAD"); + assert_eq!(recorded[1].target, "/legacy?restype=container"); + } + + #[tokio::test] + async fn azure_statuses_map_onto_the_shared_error_classes() { + for (status, code, expected, retryable) in [ + (404_u16, Some("BlobNotFound"), "not_found", false), + (403, Some("AuthorizationPermissionMismatch"), "access_denied", false), + (401, None, "access_denied", false), + (429, None, "throttled", true), + (503, Some("ServerBusy"), "throttled", true), + (500, None, "server_error", true), + ] { + let headers = code + .map(|code| vec![(HEADER_ERROR_CODE, code.to_string())]) + .unwrap_or_default(); + let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(status, headers, String::new())]).await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + let err = backend.head("a.txt").await.expect_err("{status} must fail"); + assert_eq!(err.class_label(), expected, "status {status} -> {err:?}"); + assert_eq!(err.is_retryable(), retryable, "status {status} -> {err:?}"); + } + } + + const CONTRACT_LIST_PAGE_ONE: &str = r#" + + + + dir/a.txt + + Wed, 21 Oct 2015 07:28:00 GMT + 0x8D2F1B0A1B2C3D4 + 5 + Hot + + + dir/sub/ + + cursor-1 +"#; + + const CONTRACT_LIST_PAGE_TWO: &str = r#" + + + + dir/b.txt + + Wed, 21 Oct 2015 07:28:00 GMT + 7 + + + + +"#; + + const CONTRACT_TAGS: &str = r#" +envprod"#; + + fn contract_blob_headers() -> Vec<(&'static str, String)> { + vec![ + ("ETag", "\"0x8D2F1B0A1B2C3D4\"".to_string()), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()), + ("Content-Type", "text/plain".to_string()), + ("x-ms-meta-owner", "alice".to_string()), + ("x-ms-access-tier", "Hot".to_string()), + ("x-ms-blob-type", "BlockBlob".to_string()), + ] + } + + #[tokio::test] + async fn azure_backend_satisfies_the_shared_backend_contract() { + let mut ranged = contract_blob_headers(); + ranged.push(("Content-Range", "bytes 1-3/5".to_string())); + // A HEAD reports the object size with no body, exactly as Azure does. + let mut head_only = contract_blob_headers(); + head_only.push(("Content-Length", "5".to_string())); + let (endpoint, _) = scripted_server(vec![ + ScriptedResponse::new(200, head_only, String::new()), + ScriptedResponse::new(200, contract_blob_headers(), "hello".to_string()), + ScriptedResponse::new(206, ranged, "ell".to_string()), + ScriptedResponse::new(200, Vec::new(), CONTRACT_LIST_PAGE_ONE.to_string()), + ScriptedResponse::new(200, Vec::new(), CONTRACT_LIST_PAGE_TWO.to_string()), + ScriptedResponse::new(200, Vec::new(), CONTRACT_TAGS.to_string()), + ScriptedResponse::new(200, Vec::new(), String::new()), + ScriptedResponse::new(404, vec![(HEADER_ERROR_CODE, "BlobNotFound".to_string())], String::new()), + ScriptedResponse::new( + 403, + vec![(HEADER_ERROR_CODE, "AuthorizationPermissionMismatch".to_string())], + String::new(), + ), + ]) + .await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + + assert_backend_contract( + &backend, + BackendCapabilities { + // Azure's ETag is a concurrency token; the contract requires it + // to be carried but never read as a digest. + etag_is_opaque: true, + // Azure paginates only with an opaque marker. + supports_start_after: false, + supports_tagging: true, + }, + ) + .await; + } + + #[tokio::test] + async fn transport_failures_never_render_the_request_url() { + // Nothing is listening on the reserved port, so the connect fails and + // the error must not carry the SAS-bearing URL. + let backend = backend( + &Url::parse("http://127.0.0.1:1").expect("endpoint"), + Credential::Sas(vec![("sig".to_string(), "top-secret-signature".to_string())]), + ); + let err = backend.head("a.txt").await.expect_err("a closed port must fail"); + let rendered = err.to_string(); + assert!(!rendered.contains("top-secret-signature"), "{rendered}"); + assert!(!rendered.contains("127.0.0.1"), "{rendered}"); + } +} diff --git a/crates/ecstore/src/bucket/on_demand_migration/backend_contract.rs b/crates/ecstore/src/bucket/on_demand_migration/backend_contract.rs new file mode 100644 index 000000000..a8e1b1337 --- /dev/null +++ b/crates/ecstore/src/bucket/on_demand_migration/backend_contract.rs @@ -0,0 +1,172 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! One contract every [`SourceBackend`] implementation must satisfy. +//! +//! The migration pipeline talks to a source only through the trait, so a new +//! provider is correct exactly when it answers the same questions the same way: +//! the same head fields, the same range semantics, the same page shape, the +//! same error classes. Each backend supplies a fixture that answers this fixed +//! corpus in its own dialect and then runs [`assert_backend_contract`], so a +//! provider-specific mapping bug shows up as a contract failure rather than as +//! a surprise in the pull pipeline. +//! +//! Backends differ in two documented ways, declared through +//! [`BackendCapabilities`]: whether the provider's ETag is a content digest, +//! and whether the provider can resume a listing from a key. + +use super::source_client::{SourceBackend, SourceError, SourceListRequest}; +use crate::storage_api_contracts::range::HTTPRangeSpec; +use std::collections::HashMap; + +/// The single object every fixture serves. +pub(super) const OBJECT_KEY: &str = "dir/a.txt"; +pub(super) const OBJECT_BODY: &[u8] = b"hello"; +/// MD5 of [`OBJECT_BODY`]; the ETag of the object on a digest provider. +pub(super) const OBJECT_MD5: &str = "5d41402abc4b2a76b9719d911017c592"; +/// The second key the fixture's listing returns, on its second page. +pub(super) const SECOND_KEY: &str = "dir/b.txt"; +pub(super) const COMMON_PREFIX: &str = "dir/sub/"; +pub(super) const LIST_CURSOR: &str = "cursor-1"; +/// A key the fixture answers with the provider's "no such object". +pub(super) const MISSING_KEY: &str = "missing"; +/// A key the fixture answers with the provider's "not authorized". +pub(super) const FORBIDDEN_KEY: &str = "secret"; + +/// Where backends are allowed to differ. +#[derive(Clone, Copy, Debug)] +pub(super) struct BackendCapabilities { + /// The provider's ETag is an opaque token, not a digest of the bytes. + pub(super) etag_is_opaque: bool, + /// The provider can resume a listing from a key rather than only from an + /// opaque cursor. + pub(super) supports_start_after: bool, + /// The provider has an object-tagging concept at all. GCS does not, and + /// answers with an empty map instead of failing a pull. + pub(super) supports_tagging: bool, +} + +/// Drives `backend` through the shared corpus. Fixtures are scripted in +/// request order, so the call order here is part of the contract. +pub(super) async fn assert_backend_contract(backend: &dyn SourceBackend, caps: BackendCapabilities) { + // 1. HEAD maps the object's shared fields. + let head = backend.head(OBJECT_KEY).await.expect("HEAD of the fixture object"); + assert_eq!(head.size, OBJECT_BODY.len() as u64, "HEAD reports the object size"); + assert_eq!(head.content_type.as_deref(), Some("text/plain")); + assert_eq!( + head.user_metadata, + HashMap::from([("owner".to_string(), "alice".to_string())]), + "user metadata is keyed without the provider prefix" + ); + assert!(head.storage_class.is_some(), "the provider's tier is recorded"); + assert!(head.last_modified.is_some(), "the provider's timestamp is parsed"); + assert!(head.sse.is_none(), "the fixture object is not server-side encrypted"); + assert!(!head.is_multipart_etag); + assert_eq!(head.etag_is_opaque, caps.etag_is_opaque); + match caps.etag_is_opaque { + false => assert_eq!(head.etag.as_deref(), Some(OBJECT_MD5), "a digest ETag is mapped verbatim"), + true => assert!(head.etag.is_some(), "an opaque ETag is still recorded"), + } + + // 2. An unranged GET streams the whole object and reports no range. + let got = backend.get(OBJECT_KEY, None).await.expect("unranged GET"); + assert_eq!(got.head.size, OBJECT_BODY.len() as u64); + assert!(got.content_range.is_none(), "an unranged GET has no content-range"); + assert_eq!(got.head.etag_is_opaque, caps.etag_is_opaque, "GET and HEAD agree about the ETag"); + let body = got.body.collect().await.expect("body streams").into_bytes(); + assert_eq!(body.as_ref(), OBJECT_BODY); + + // 3. A ranged GET returns exactly the requested interval, and `size` is + // the length of the returned bytes rather than of the object. + let range = HTTPRangeSpec { + is_suffix_length: false, + start: 1, + end: 3, + }; + let got = backend.get(OBJECT_KEY, Some(&range)).await.expect("ranged GET"); + assert_eq!(got.head.size, 3, "a ranged GET reports the range length"); + assert_eq!(got.content_range.as_deref(), Some("bytes 1-3/5")); + let body = got.body.collect().await.expect("body streams").into_bytes(); + assert_eq!(body.as_ref(), &OBJECT_BODY[1..=3]); + + // 4. A delimiter listing rolls prefixes up and hands back a cursor. + let page = backend + .list(&SourceListRequest { + prefix: Some("dir/"), + delimiter: Some("/"), + max_keys: 2, + ..Default::default() + }) + .await + .expect("first listing page"); + assert_eq!(page.objects.len(), 1, "the first page holds one object"); + assert_eq!(page.objects[0].key, OBJECT_KEY, "listing keys are in the source namespace"); + assert_eq!(page.objects[0].size, OBJECT_BODY.len() as u64); + assert!(page.objects[0].last_modified.is_some()); + assert_eq!(page.common_prefixes, vec![COMMON_PREFIX.to_string()]); + assert!(page.is_truncated); + assert_eq!(page.next_continuation_token.as_deref(), Some(LIST_CURSOR)); + + // 5. The cursor is passed back verbatim and the last page ends the walk. + let page = backend + .list(&SourceListRequest { + prefix: Some("dir/"), + delimiter: Some("/"), + continuation_token: Some(LIST_CURSOR), + max_keys: 2, + ..Default::default() + }) + .await + .expect("second listing page"); + assert_eq!(page.objects.len(), 1); + assert_eq!(page.objects[0].key, SECOND_KEY); + assert!(!page.is_truncated); + assert!(page.next_continuation_token.is_none(), "a complete listing carries no cursor"); + + // 6. Tags come back as a flat map, empty on a provider without tags. + let tags = backend.tagging(OBJECT_KEY).await.expect("object tags"); + match caps.supports_tagging { + true => assert_eq!(tags, HashMap::from([("env".to_string(), "prod".to_string())])), + false => assert!(tags.is_empty(), "a provider without tags reports none: {tags:?}"), + } + + // 7. The probe confirms the bucket or container answers. + backend.probe().await.expect("probe of the fixture bucket"); + + // 8. A missing object is `NotFound`, and never retried. + let err = backend.head(MISSING_KEY).await.expect_err("a missing object must fail"); + assert!(matches!(err, SourceError::NotFound), "{err:?}"); + assert_eq!(err.class_label(), "not_found"); + assert!(!err.is_retryable()); + + // 9. A denied object is `AccessDenied`, and never retried. + let err = backend.head(FORBIDDEN_KEY).await.expect_err("a denied object must fail"); + assert!(matches!(err, SourceError::AccessDenied), "{err:?}"); + assert_eq!(err.class_label(), "access_denied"); + assert!(!err.is_retryable()); + + // 10. A provider without a key cursor must refuse one instead of listing + // from the wrong position. This issues no request either way. + if !caps.supports_start_after { + let err = backend + .list(&SourceListRequest { + start_after: Some(OBJECT_KEY), + max_keys: 1, + ..Default::default() + }) + .await + .expect_err("a backend without a key cursor must refuse start_after"); + assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}"); + } +} diff --git a/crates/ecstore/src/bucket/on_demand_migration/config.rs b/crates/ecstore/src/bucket/on_demand_migration/config.rs index 759762d26..d00f08c2c 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/config.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/config.rs @@ -30,6 +30,10 @@ pub const ON_DEMAND_MIGRATION_CONFIG_VERSION: u32 = 1; const REDACTED: &str = "REDACTED"; const AUTO_REGION: &str = "auto"; const AUTO_REGION_FALLBACK: &str = "us-east-1"; +/// Public Azure Blob host suffix; the account name is the first label. +pub const AZURE_BLOB_SUFFIX: &str = "blob.core.windows.net"; +/// Public Google Cloud Storage endpoint for the native provider. +pub const GCS_DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com"; const KIB: u64 = 1024; const MIB: u64 = 1024 * KIB; @@ -75,14 +79,25 @@ pub struct SourceConfig { pub bucket: String, #[serde(default)] pub path_style: PathStyle, - /// `None` means anonymous access to a public source bucket. + /// `None` means anonymous access to a public source bucket. Only the + /// SigV4 providers read it; `azure` and `gcs_native` carry their own + /// credentials in `azure` / `gcs`. #[serde(default)] pub credentials: Option, #[serde(default)] pub tls: TlsConfig, + /// Required for [`Provider::Azure`] and rejected for every other + /// provider. + #[serde(default)] + pub azure: Option, + /// Required for [`Provider::GcsNative`] and rejected for every other + /// provider. [`Provider::Gcs`] keeps using `credentials` because it + /// speaks the S3 interoperability API. + #[serde(default)] + pub gcs: Option, } -/// Source vendor family. `azure` is deliberately absent from this version. +/// Source vendor family. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum Provider { @@ -94,6 +109,12 @@ pub enum Provider { R2, /// GCS XML interoperability API with HMAC keys. Gcs, + /// Native Azure Blob service; parameters in `source.azure`. + Azure, + /// Native GCS JSON API with a service-account key; parameters in + /// `source.gcs`. + #[serde(rename = "gcs_native")] + GcsNative, } impl Provider { @@ -105,13 +126,22 @@ impl Provider { Provider::Rustfs => "rustfs", Provider::R2 => "r2", Provider::Gcs => "gcs", + Provider::Azure => "azure", + Provider::GcsNative => "gcs_native", } } + /// Providers that do not speak S3 and therefore ignore `region`, + /// `path_style` and `credentials`. + pub fn is_native(&self) -> bool { + matches!(self, Provider::Azure | Provider::GcsNative) + } + /// Providers whose SDKs accept `region = "auto"`; RustFS maps it to - /// `us-east-1` for signing. + /// `us-east-1` for signing. The native providers never sign with a + /// region, so they accept it as well. fn accepts_auto_region(&self) -> bool { - matches!(self, Provider::R2 | Provider::Minio | Provider::Rustfs) + matches!(self, Provider::R2 | Provider::Minio | Provider::Rustfs) || self.is_native() } } @@ -164,6 +194,73 @@ impl fmt::Debug for SourceCredentials { } } +/// Native Azure Blob source parameters. The container is `source.bucket`, +/// so a config never carries two names for the same container. Exactly one +/// of `account_key` and `sas_token` must be set: the account key signs with +/// Shared Key, the SAS token is appended to every request URL. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AzureSourceConfig { + /// Storage account name; also derives the default `blob.core.windows.net` + /// endpoint when `source.endpoint` is absent. + pub account: String, + /// Base64 shared key of the storage account. + #[serde(default)] + pub account_key: Option, + /// SAS query string without the leading `?`. + #[serde(default)] + pub sas_token: Option, +} + +impl AzureSourceConfig { + /// A copy safe to return to admin clients or log: both secrets are + /// replaced by `REDACTED`, and whether each is set stays visible. + pub fn redacted(&self) -> Self { + Self { + account: self.account.clone(), + account_key: self.account_key.as_ref().map(|_| REDACTED.to_string()), + sas_token: self.sas_token.as_ref().map(|_| REDACTED.to_string()), + } + } +} + +impl fmt::Debug for AzureSourceConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AzureSourceConfig") + .field("account", &self.account) + .field("account_key", &self.account_key.as_ref().map(|_| REDACTED)) + .field("sas_token", &self.sas_token.as_ref().map(|_| REDACTED)) + .finish() + } +} + +/// Native Google Cloud Storage source parameters. The bucket is +/// `source.bucket`; only the service-account key lives here. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GcsSourceConfig { + /// Service-account key JSON, verbatim as downloaded from Google Cloud. + pub service_account_json: String, +} + +impl GcsSourceConfig { + /// A copy safe to return to admin clients or log: the whole key JSON is + /// a secret (it embeds the private key), so it is replaced wholesale. + pub fn redacted(&self) -> Self { + Self { + service_account_json: REDACTED.to_string(), + } + } +} + +impl fmt::Debug for GcsSourceConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GcsSourceConfig") + .field("service_account_json", &REDACTED) + .finish() + } +} + #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct TlsConfig { @@ -354,6 +451,14 @@ pub enum OnDemandMigrationConfigError { InvalidBucket(&'static str), #[error("source credentials field {0} must not be empty")] EmptyCredential(&'static str), + #[error("source.{0} is required for provider {1}")] + MissingProviderBlock(&'static str, Provider), + #[error("source.{0} is not valid for provider {1}")] + UnexpectedProviderBlock(&'static str, Provider), + /// Carries only the reason: the block holds account keys, SAS tokens and + /// service-account JSON, so no value of it is ever echoed. + #[error("source.{0} is invalid: {1}")] + InvalidProviderBlock(&'static str, &'static str), #[error("source tls.ca_cert_pem is not a PEM certificate")] InvalidCaCert, #[error("filter.{0} must be null or a non-empty string")] @@ -388,6 +493,8 @@ impl OnDemandMigrationConfig { pub fn redacted(&self) -> Self { let mut copy = self.clone(); copy.source.credentials = self.source.credentials.as_ref().map(SourceCredentials::redacted); + copy.source.azure = self.source.azure.as_ref().map(AzureSourceConfig::redacted); + copy.source.gcs = self.source.gcs.as_ref().map(GcsSourceConfig::redacted); copy } @@ -433,6 +540,12 @@ impl SourceConfig { match (&self.endpoint, self.provider) { (Some(endpoint), _) => endpoint.clone(), (None, Provider::Aws) => format!("https://s3.{}.amazonaws.com", self.region), + (None, Provider::Azure) => self + .azure + .as_ref() + .map(|azure| format!("https://{}.{AZURE_BLOB_SUFFIX}", azure.account)) + .unwrap_or_default(), + (None, Provider::GcsNative) => GCS_DEFAULT_ENDPOINT.to_string(), (None, _) => String::new(), } } @@ -448,6 +561,8 @@ impl SourceConfig { } fn validate(&self) -> Result<(), OnDemandMigrationConfigError> { + self.validate_provider_block()?; + if self.region.is_empty() { return Err(OnDemandMigrationConfigError::EmptyRegion); } @@ -466,6 +581,9 @@ impl SourceConfig { )); } } + // Both native providers derive a fixed endpoint; Azure's is built + // from the account name, already checked by `validate_provider_block`. + None if self.provider.is_native() => {} None => return Err(OnDemandMigrationConfigError::MissingEndpoint(self.provider)), } @@ -496,6 +614,84 @@ impl SourceConfig { Ok(()) } + + /// The provider-specific block must be present for exactly its own + /// provider: a stray `azure` block on an `s3` source would otherwise be + /// accepted, stored, and silently ignored by the client builder. + fn validate_provider_block(&self) -> Result<(), OnDemandMigrationConfigError> { + let missing = OnDemandMigrationConfigError::MissingProviderBlock; + let unexpected = OnDemandMigrationConfigError::UnexpectedProviderBlock; + let invalid = OnDemandMigrationConfigError::InvalidProviderBlock; + + if self.provider != Provider::Azure && self.azure.is_some() { + return Err(unexpected("azure", self.provider)); + } + if self.provider != Provider::GcsNative && self.gcs.is_some() { + return Err(unexpected("gcs", self.provider)); + } + + match self.provider { + Provider::Azure => { + let azure = self.azure.as_ref().ok_or(missing("azure", self.provider))?; + if azure.account.is_empty() { + return Err(invalid("azure", "account must not be empty")); + } + // The account feeds a hostname when the endpoint is derived: + // keep it to label characters so it cannot rewrite the host. + if !azure.account.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') { + return Err(invalid("azure", "account contains characters outside [A-Za-z0-9-]")); + } + match (azure.account_key.as_deref(), azure.sas_token.as_deref()) { + (Some(_), Some(_)) => return Err(invalid("azure", "account_key and sas_token are mutually exclusive")), + (None, None) => return Err(invalid("azure", "one of account_key and sas_token is required")), + (Some(key), None) => { + if key.is_empty() { + return Err(invalid("azure", "account_key must not be empty")); + } + // Decoded here so a mistyped key fails at the admin + // boundary instead of on the first source request. + if base64_simd::STANDARD.decode_to_vec(key.as_bytes()).is_err() { + return Err(invalid("azure", "account_key is not base64")); + } + } + (None, Some(sas)) => { + if sas.is_empty() { + return Err(invalid("azure", "sas_token must not be empty")); + } + if sas.starts_with('?') { + return Err(invalid("azure", "sas_token must not start with '?'")); + } + if sas.chars().any(char::is_whitespace) { + return Err(invalid("azure", "sas_token must not contain whitespace")); + } + } + } + } + Provider::GcsNative => { + let gcs = self.gcs.as_ref().ok_or(missing("gcs", self.provider))?; + let key: serde_json::Value = serde_json::from_str(&gcs.service_account_json) + .map_err(|_| invalid("gcs", "service_account_json is not valid JSON"))?; + let Some(object) = key.as_object() else { + return Err(invalid("gcs", "service_account_json is not a JSON object")); + }; + if object.get("type").and_then(serde_json::Value::as_str) != Some("service_account") { + return Err(invalid("gcs", "service_account_json is not a service_account key")); + } + for field in ["client_email", "private_key"] { + if object + .get(field) + .and_then(serde_json::Value::as_str) + .is_none_or(str::is_empty) + { + return Err(invalid("gcs", "service_account_json is missing client_email or private_key")); + } + } + } + Provider::S3 | Provider::Aws | Provider::Minio | Provider::Rustfs | Provider::R2 | Provider::Gcs => {} + } + + Ok(()) + } } fn validate_endpoint(endpoint: &str) -> Result<(), OnDemandMigrationConfigError> { @@ -699,7 +895,15 @@ mod tests { ), ( "provider enum", - r#"{"source":{"provider":"azure","endpoint":"https://h","region":"r","bucket":"b"}}"#, + r#"{"source":{"provider":"swift","endpoint":"https://h","region":"r","bucket":"b"}}"#, + ), + ( + "azure block", + r#"{"source":{"provider":"azure","region":"auto","bucket":"b","azure":{"account":"acct","account_key":"a2V5","extra":1}}}"#, + ), + ( + "gcs block", + r#"{"source":{"provider":"gcs_native","region":"auto","bucket":"b","gcs":{"service_account_json":"{}","extra":1}}}"#, ), ] { let err = OnDemandMigrationConfig::from_json(json.as_bytes()).expect_err(label); @@ -820,9 +1024,201 @@ mod tests { "{provider}" ); } + // The native providers never sign with a region, so "auto" is the + // honest value to write for them. + for cfg in [azure_cfg(), gcs_native_cfg()] { + assert_eq!(cfg.source.region, "auto"); + cfg.validate(empty_ctx()) + .unwrap_or_else(|err| panic!("{}: {err}", cfg.source.provider)); + } assert_eq!(sample().source.effective_region(), "us-west-1"); } + const SERVICE_ACCOUNT_JSON: &str = r#"{"type":"service_account","project_id":"p","client_email":"a@b.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\nsecret\n-----END PRIVATE KEY-----"}"#; + + fn azure_cfg() -> OnDemandMigrationConfig { + let mut cfg = sample(); + cfg.source.provider = Provider::Azure; + cfg.source.endpoint = None; + cfg.source.region = "auto".to_string(); + cfg.source.credentials = None; + cfg.source.azure = Some(AzureSourceConfig { + account: "legacyaccount".to_string(), + account_key: Some("c2VjcmV0LWtleQ==".to_string()), + sas_token: None, + }); + cfg + } + + fn gcs_native_cfg() -> OnDemandMigrationConfig { + let mut cfg = sample(); + cfg.source.provider = Provider::GcsNative; + cfg.source.endpoint = None; + cfg.source.region = "auto".to_string(); + cfg.source.credentials = None; + cfg.source.gcs = Some(GcsSourceConfig { + service_account_json: SERVICE_ACCOUNT_JSON.to_string(), + }); + cfg + } + + #[test] + fn native_providers_derive_their_endpoint_and_round_trip_on_the_wire() { + let azure = azure_cfg(); + assert_eq!(azure.source.effective_endpoint(), "https://legacyaccount.blob.core.windows.net"); + let gcs = gcs_native_cfg(); + assert_eq!(gcs.source.effective_endpoint(), "https://storage.googleapis.com"); + + for cfg in [azure_cfg(), gcs_native_cfg()] { + let json = cfg.to_json().expect("config must serialize"); + assert_eq!(OnDemandMigrationConfig::from_json(&json).expect("config must parse"), cfg); + } + // The wire labels are part of the admin contract. + assert!( + String::from_utf8(azure_cfg().to_json().expect("json")) + .expect("utf8") + .contains(r#""provider":"azure""#) + ); + assert!( + String::from_utf8(gcs_native_cfg().to_json().expect("json")) + .expect("utf8") + .contains(r#""provider":"gcs_native""#) + ); + } + + #[test] + fn an_explicit_endpoint_overrides_the_derived_native_one() { + // Azurite and fake-gcs-server are addressed this way. + let mut cfg = azure_cfg(); + cfg.source.endpoint = Some("http://azurite.example.com:10000".to_string()); + cfg.validate(empty_ctx()).expect("an explicit native endpoint is allowed"); + assert_eq!(cfg.source.effective_endpoint(), "http://azurite.example.com:10000"); + + cfg.source.endpoint = Some("http://azurite.example.com:10000/devstoreaccount1".to_string()); + assert!( + matches!(cfg.validate(empty_ctx()), Err(OnDemandMigrationConfigError::InvalidEndpoint(_))), + "a native endpoint is still an origin" + ); + } + + #[test] + fn a_provider_block_belongs_to_exactly_its_own_provider() { + let mut cfg = sample(); + cfg.source.azure = azure_cfg().source.azure; + assert_eq!( + cfg.validate(empty_ctx()), + Err(OnDemandMigrationConfigError::UnexpectedProviderBlock("azure", Provider::S3)) + ); + + let mut cfg = sample(); + cfg.source.gcs = gcs_native_cfg().source.gcs; + assert_eq!( + cfg.validate(empty_ctx()), + Err(OnDemandMigrationConfigError::UnexpectedProviderBlock("gcs", Provider::S3)) + ); + + let mut cfg = azure_cfg(); + cfg.source.azure = None; + assert_eq!( + cfg.validate(empty_ctx()), + Err(OnDemandMigrationConfigError::MissingProviderBlock("azure", Provider::Azure)) + ); + + let mut cfg = gcs_native_cfg(); + cfg.source.gcs = None; + assert_eq!( + cfg.validate(empty_ctx()), + Err(OnDemandMigrationConfigError::MissingProviderBlock("gcs", Provider::GcsNative)) + ); + } + + #[test] + fn azure_block_rules() { + let with = |account: &str, key: Option<&str>, sas: Option<&str>| { + let mut cfg = azure_cfg(); + cfg.source.azure = Some(AzureSourceConfig { + account: account.to_string(), + account_key: key.map(str::to_string), + sas_token: sas.map(str::to_string), + }); + cfg.validate(empty_ctx()) + }; + + with("legacyaccount", None, Some("sv=2021-08-06&sig=abc%3D")).expect("a SAS token is a complete credential"); + with("legacyaccount", Some("c2VjcmV0LWtleQ=="), None).expect("an account key is a complete credential"); + + for (label, result) in [ + ("empty account", with("", Some("c2VjcmV0LWtleQ=="), None)), + // The account becomes the first label of the derived hostname. + ("account with a dot", with("legacy.account", Some("c2VjcmV0LWtleQ=="), None)), + ("account with a slash", with("legacy/account", Some("c2VjcmV0LWtleQ=="), None)), + ("no credential", with("legacyaccount", None, None)), + ("both credentials", with("legacyaccount", Some("c2VjcmV0LWtleQ=="), Some("sv=1"))), + ("empty key", with("legacyaccount", Some(""), None)), + ("key that is not base64", with("legacyaccount", Some("not base64!"), None)), + ("empty sas", with("legacyaccount", None, Some(""))), + ("sas with a leading question mark", with("legacyaccount", None, Some("?sv=1"))), + ("sas with whitespace", with("legacyaccount", None, Some("sv=1 &sig=a"))), + ] { + assert!( + matches!(result, Err(OnDemandMigrationConfigError::InvalidProviderBlock("azure", _))), + "{label}: {result:?}" + ); + } + } + + #[test] + fn gcs_native_block_requires_a_usable_service_account_key() { + let with = |json: &str| { + let mut cfg = gcs_native_cfg(); + cfg.source.gcs = Some(GcsSourceConfig { + service_account_json: json.to_string(), + }); + cfg.validate(empty_ctx()) + }; + + with(SERVICE_ACCOUNT_JSON).expect("a service-account key is accepted"); + for (label, json) in [ + ("empty", ""), + ("not json", "not json"), + ("not an object", "[]"), + ("wrong type", r#"{"type":"authorized_user","client_email":"a@b","private_key":"k"}"#), + ("no private key", r#"{"type":"service_account","client_email":"a@b"}"#), + ("empty client email", r#"{"type":"service_account","client_email":"","private_key":"k"}"#), + ] { + let result = with(json); + assert!( + matches!(result, Err(OnDemandMigrationConfigError::InvalidProviderBlock("gcs", _))), + "{label}: {result:?}" + ); + } + } + + #[test] + fn native_secrets_never_survive_redaction_or_debug() { + let mut azure = azure_cfg(); + azure.source.azure.as_mut().expect("block").sas_token = Some("sv=2021-08-06&sig=top-secret".to_string()); + azure.source.azure.as_mut().expect("block").account_key = None; + let gcs = gcs_native_cfg(); + + for rendered in [ + format!("{:?}", azure.redacted()), + format!("{azure:?}"), + String::from_utf8(azure.redacted().to_json().expect("json")).expect("utf8"), + ] { + assert!(!rendered.contains("top-secret"), "{rendered}"); + assert!(rendered.contains("legacyaccount"), "the account name is not a secret: {rendered}"); + } + for rendered in [ + format!("{:?}", gcs.redacted()), + format!("{gcs:?}"), + String::from_utf8(gcs.redacted().to_json().expect("json")).expect("utf8"), + ] { + assert!(!rendered.contains("PRIVATE KEY-----"), "{rendered}"); + assert!(!rendered.contains("gserviceaccount"), "{rendered}"); + } + } + #[test] fn bucket_rules() { let mut cfg = sample(); diff --git a/crates/ecstore/src/bucket/on_demand_migration/gcs.rs b/crates/ecstore/src/bucket/on_demand_migration/gcs.rs new file mode 100644 index 000000000..707e1cc82 --- /dev/null +++ b/crates/ecstore/src/bucket/on_demand_migration/gcs.rs @@ -0,0 +1,506 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Native Google Cloud Storage source backend. +//! +//! The `gcs` provider already reaches GCS through its S3 interoperability API, +//! which needs an HMAC key pair. This backend is the other half: it authorizes +//! with a service-account key, the credential most GCS projects actually issue, +//! by minting OAuth tokens through the shared `google-cloud-auth` credential +//! machinery the tier layer already uses. +//! +//! Two GCS surfaces are involved, each for the half it describes best. The read +//! path uses the XML API (`/{bucket}/{object}`), whose responses carry +//! `x-goog-meta-*` user metadata and the `x-goog-hash` digest in one round trip. +//! Listing uses the JSON API (`objects.list`), whose `pageToken` maps directly +//! onto the shared page cursor and whose `prefixes` are the delimiter roll-up. +//! Both accept the same bearer token. +//! +//! Every call this backend makes needs only `storage.objects.get` and +//! `storage.objects.list`, the two permissions of the `objectViewer` role, so a +//! key scoped to exactly the migration's needs works. +//! +//! `x-goog-hash` carries a base64 MD5 for every non-composite object; it is +//! converted to hex and becomes the head's ETag, so a pulled object is checked +//! against the digest GCS itself computed. A composite object has no MD5, and +//! its ETag is then marked opaque rather than checked. + +use super::native_http::{ + NativeHeadFields, NativeHttp, base64_md5_to_hex, header, native_source_head, parse_http_timestamp, read_text, response_body, +}; +use super::source_client::{ + GcsSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, + SourceTimeouts, range_header_value, +}; +use crate::bucket::remote_s3_client::RemoteS3ClientError; +use crate::storage_api_contracts::range::HTTPRangeSpec; +use google_cloud_auth::credentials::service_account::{AccessSpecifier, Builder as ServiceAccountBuilder}; +use google_cloud_auth::credentials::{CacheableResource, Credentials}; +use http::{HeaderMap, HeaderValue, Method}; +use serde::Deserialize; +use std::collections::HashMap; +use url::Url; + +/// Read-only object scope: this backend never writes to the source. +const READ_ONLY_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_only"; +const METADATA_PREFIX: &str = "x-goog-meta-"; +/// GCS reports its error code in the response body, not a header; the shared +/// transport takes a header name, so it is given one that never matches and +/// classification falls back to the status. +const NO_ERROR_CODE_HEADER: &str = "x-goog-unused-error-code"; +/// One `objects.list` page is small; refuse an unbounded document. +const MAX_JSON_BYTES: usize = 8 * 1024 * 1024; + +pub struct GcsNativeSourceBackend { + http: NativeHttp, + bucket: String, + credentials: Credentials, +} + +impl GcsNativeSourceBackend { + pub fn new( + endpoint: &str, + bucket: &str, + spec: &GcsSourceSpec, + timeouts: SourceTimeouts, + skip_tls_verify: bool, + ca_cert_pem: Option<&str>, + ) -> Result { + let key: serde_json::Value = serde_json::from_str(&spec.service_account_json) + .map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not valid JSON"))?; + let credentials = ServiceAccountBuilder::new(key) + .with_access_specifier(AccessSpecifier::from_scopes([READ_ONLY_SCOPE])) + .build() + .map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not usable"))?; + Ok(Self { + http: NativeHttp::new(endpoint, timeouts, skip_tls_verify, ca_cert_pem)?, + bucket: bucket.to_string(), + credentials, + }) + } + + /// Authorization headers for one request. A credential failure is reported + /// as `AccessDenied` with no message: the renderer of a credential error + /// has the key material in scope, and the class is what callers act on. + async fn auth_headers(&self) -> Result { + match self.credentials.headers(http::Extensions::new()).await { + Ok(CacheableResource::New { data, .. }) => Ok(data), + // Only returned when the caller passes an entity tag, which this + // backend never does; an empty set is still the honest answer. + Ok(CacheableResource::NotModified) => Ok(HeaderMap::new()), + Err(_) => Err(SourceError::AccessDenied), + } + } + + /// XML API URL of one object; `/` in the key stay path separators. + fn object_url(&self, key: &str) -> Result { + self.http.url(std::iter::once(self.bucket.as_str()).chain(key.split('/'))) + } + + /// JSON API URL of the bucket's object collection. + fn objects_url(&self) -> Result { + self.http.url(["storage", "v1", "b", self.bucket.as_str(), "o"]) + } + + async fn request(&self, method: Method, url: Url, mut headers: HeaderMap) -> Result { + for (name, value) in self.auth_headers().await? { + if let Some(name) = name { + headers.insert(name, value); + } + } + let mut request = reqwest::Request::new(method, url); + *request.headers_mut() = headers; + Ok(request) + } + + /// Shared mapping for the XML API's HEAD and GET responses. + fn head_from_response(headers: &HeaderMap) -> Result { + if header(headers, "x-goog-encryption-key-sha256").is_some() { + return Err(SourceError::Unsupported( + "source object uses a customer-supplied encryption key; customer-key sources are not supported".to_string(), + )); + } + // `x-goog-hash` lists digests as `name=base64`, comma separated, and may + // repeat across header lines. Only the MD5 describes the whole object. + let md5 = headers + .get_all("x-goog-hash") + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .filter_map(|digest| digest.trim().strip_prefix("md5=")) + .find_map(base64_md5_to_hex); + + let (etag, etag_is_opaque) = match md5 { + Some(md5) => (Some(md5), false), + // A composite object has no MD5; its ETag describes the composition + // rather than the bytes, so it is provenance only. + None => (header(headers, "etag").map(str::to_string), true), + }; + native_source_head( + headers, + METADATA_PREFIX, + NativeHeadFields { + etag, + etag_is_opaque, + version_id: header(headers, "x-goog-generation").map(str::to_string), + storage_class: header(headers, "x-goog-storage-class").map(str::to_string), + }, + ) + } +} + +#[async_trait::async_trait] +impl SourceBackend for GcsNativeSourceBackend { + async fn head(&self, key: &str) -> Result { + let request = self.request(Method::HEAD, self.object_url(key)?, HeaderMap::new()).await?; + let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?; + Self::head_from_response(response.headers()) + } + + async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result { + let mut headers = HeaderMap::new(); + if let Some(range) = range.map(range_header_value).transpose()? { + headers.insert( + http::header::RANGE, + HeaderValue::from_str(&range).map_err(|_| SourceError::Other("invalid range header".to_string()))?, + ); + } + let request = self.request(Method::GET, self.object_url(key)?, headers).await?; + let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?; + let head = Self::head_from_response(response.headers())?; + let content_range = header(response.headers(), "content-range").map(str::to_string); + Ok(SourceGet { + head, + body: response_body(response), + content_range, + }) + } + + async fn list(&self, request: &SourceListRequest<'_>) -> Result { + // `objects.list` offers `startOffset`, which is inclusive, so it cannot + // express "resume after this key" without silently repeating it. + if request.start_after.is_some() { + return Err(SourceError::Unsupported( + "gcs sources cannot resume a listing from a key; use the continuation token".to_string(), + )); + } + let mut url = self.objects_url()?; + { + let mut query = url.query_pairs_mut(); + if let Some(prefix) = request.prefix.filter(|prefix| !prefix.is_empty()) { + query.append_pair("prefix", prefix); + } + if let Some(delimiter) = request.delimiter.filter(|delimiter| !delimiter.is_empty()) { + query.append_pair("delimiter", delimiter); + } + if let Some(token) = request.continuation_token.filter(|token| !token.is_empty()) { + query.append_pair("pageToken", token); + } + if request.max_keys > 0 { + query.append_pair("maxResults", &request.max_keys.to_string()); + } + } + + let request = self.request(Method::GET, url, HeaderMap::new()).await?; + let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?; + let body = read_text(response, MAX_JSON_BYTES).await?; + parse_objects_list(&body) + } + + /// GCS has no object tagging API; user metadata is already carried by the + /// head mapping. An empty map keeps `policy.copy_tags` from failing a pull + /// over a concept the provider does not have. + async fn tagging(&self, _key: &str) -> Result, SourceError> { + Ok(HashMap::new()) + } + + /// A one-object listing, not `buckets.get`: the migration pipeline only + /// ever needs `storage.objects.list` and `storage.objects.get`, and a key + /// scoped to exactly those (the `objectViewer` role) cannot read the bucket + /// resource. Probing with `buckets.get` would reject a correct key. + async fn probe(&self) -> Result<(), SourceError> { + let mut url = self.objects_url()?; + url.query_pairs_mut().append_pair("maxResults", "1"); + let request = self.request(Method::GET, url, HeaderMap::new()).await?; + let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?; + read_text(response, MAX_JSON_BYTES) + .await + .and_then(|body| parse_objects_list(&body))?; + Ok(()) + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ObjectsList { + #[serde(default)] + items: Vec, + #[serde(default)] + prefixes: Vec, + #[serde(default)] + next_page_token: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ListedObject { + name: String, + /// GCS renders the size as a decimal string, not a JSON number. + #[serde(default)] + size: Option, + #[serde(default)] + updated: Option, + #[serde(default)] + md5_hash: Option, + #[serde(default)] + etag: Option, + #[serde(default)] + storage_class: Option, +} + +fn parse_objects_list(body: &str) -> Result { + let listing: ObjectsList = + serde_json::from_str(body).map_err(|err| SourceError::Other(format!("source listing is not valid JSON: {err}")))?; + let next_continuation_token = listing.next_page_token.filter(|token| !token.is_empty()); + let objects = listing + .items + .into_iter() + .map(|item| { + let etag = item + .md5_hash + .as_deref() + .and_then(base64_md5_to_hex) + .or_else(|| item.etag.map(|etag| etag.trim_matches('"').to_string())) + .filter(|etag| !etag.is_empty()); + SourceObject { + key: item.name, + etag, + size: item.size.and_then(|size| size.parse().ok()).unwrap_or(0), + last_modified: item.updated.as_deref().and_then(parse_http_timestamp), + storage_class: item.storage_class, + // GCS never encodes a part count in a digest or an ETag. + is_multipart_etag: false, + } + }) + .collect(); + + Ok(SourcePage { + objects, + common_prefixes: listing.prefixes, + is_truncated: next_continuation_token.is_some(), + next_continuation_token, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract}; + use crate::bucket::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server}; + use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder; + + const LIST_PAGE_ONE: &str = r#"{ + "kind": "storage#objects", + "nextPageToken": "cursor-1", + "prefixes": ["dir/sub/"], + "items": [ + { + "name": "dir/a.txt", + "size": "5", + "updated": "2015-10-21T07:28:00.000Z", + "md5Hash": "XUFAKrxLKna5cZ2REBfFkg==", + "etag": "CJizy9Wq0McCEAE=", + "storageClass": "STANDARD" + } + ] + }"#; + + const LIST_PAGE_TWO: &str = r#"{ + "kind": "storage#objects", + "items": [ + { + "name": "dir/b.txt", + "size": "7", + "updated": "2015-10-21T07:28:00.000Z", + "etag": "\"CJizy9Wq0McCEAI=\"" + } + ] + }"#; + + fn backend(endpoint: &Url) -> GcsNativeSourceBackend { + GcsNativeSourceBackend { + http: NativeHttp::for_test(endpoint.clone()), + bucket: "legacy".to_string(), + // Anonymous credentials add no headers, so the fixture sees exactly + // the request this backend builds. + credentials: AnonymousBuilder::new().build(), + } + } + + fn object_headers() -> Vec<(&'static str, String)> { + vec![ + ("Content-Type", "text/plain".to_string()), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()), + ("ETag", "\"CJizy9Wq0McCEAE=\"".to_string()), + ("x-goog-hash", "crc32c=AAAAAA==,md5=XUFAKrxLKna5cZ2REBfFkg==".to_string()), + ("x-goog-meta-owner", "alice".to_string()), + ("x-goog-storage-class", "STANDARD".to_string()), + ("x-goog-generation", "1445412480000000".to_string()), + ] + } + + #[test] + fn objects_list_maps_items_prefixes_and_the_page_token() { + let page = parse_objects_list(LIST_PAGE_ONE).expect("page should parse"); + assert_eq!(page.common_prefixes, vec!["dir/sub/"]); + assert!(page.is_truncated); + assert_eq!(page.next_continuation_token.as_deref(), Some("cursor-1")); + assert_eq!(page.objects.len(), 1); + assert_eq!(page.objects[0].key, "dir/a.txt"); + assert_eq!(page.objects[0].size, 5, "the string size is parsed"); + assert_eq!( + page.objects[0].etag.as_deref(), + Some("5d41402abc4b2a76b9719d911017c592"), + "the base64 md5Hash becomes a hex ETag" + ); + assert_eq!(page.objects[0].storage_class.as_deref(), Some("STANDARD")); + assert!(page.objects[0].last_modified.is_some(), "RFC 3339 `updated` is parsed"); + + let page = parse_objects_list(LIST_PAGE_TWO).expect("page should parse"); + assert!(!page.is_truncated); + assert!(page.next_continuation_token.is_none()); + assert_eq!( + page.objects[0].etag.as_deref(), + Some("CJizy9Wq0McCEAI="), + "without md5Hash the raw etag is carried" + ); + + assert!(parse_objects_list("not json").is_err()); + } + + #[tokio::test] + async fn head_prefers_the_goog_hash_md5_over_the_etag() { + let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, object_headers(), String::new())]).await; + let head = backend(&endpoint).head("dir/a b.txt").await.expect("HEAD should map"); + + let recorded = recorded.lock().expect("recorder lock").clone(); + assert_eq!(recorded[0].method, "HEAD"); + assert_eq!(recorded[0].target, "/legacy/dir/a%20b.txt", "the XML API addresses the object by path"); + assert_eq!( + head.etag.as_deref(), + Some("5d41402abc4b2a76b9719d911017c592"), + "the x-goog-hash md5 is the content digest" + ); + assert!(!head.etag_is_opaque, "a GCS md5 may be checked against the pulled bytes"); + assert_eq!(head.user_metadata, HashMap::from([("owner".to_string(), "alice".to_string())])); + assert_eq!(head.version_id.as_deref(), Some("1445412480000000")); + assert_eq!(head.storage_class.as_deref(), Some("STANDARD")); + } + + #[tokio::test] + async fn a_composite_object_without_an_md5_keeps_an_opaque_etag() { + let headers = object_headers() + .into_iter() + .map(|(name, value)| { + if name == "x-goog-hash" { + (name, "crc32c=AAAAAA==".to_string()) + } else { + (name, value) + } + }) + .collect(); + let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await; + let head = backend(&endpoint).head("composed").await.expect("HEAD should map"); + assert_eq!(head.etag.as_deref(), Some("CJizy9Wq0McCEAE=")); + assert!(head.etag_is_opaque, "a composite ETag describes the composition, not the bytes"); + } + + #[tokio::test] + async fn customer_supplied_key_objects_are_refused() { + let mut headers = object_headers(); + headers.push(("x-goog-encryption-key-sha256", "abc".to_string())); + let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await; + let err = backend(&endpoint) + .head("a.txt") + .await + .expect_err("CSEK objects are unsupported"); + assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}"); + } + + #[tokio::test] + async fn list_and_probe_address_the_json_api() { + let (endpoint, recorded) = scripted_server(vec![ + ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()), + ScriptedResponse::new(200, Vec::new(), "{}".to_string()), + ]) + .await; + let backend = backend(&endpoint); + + backend + .list(&SourceListRequest { + prefix: Some("dir/"), + delimiter: Some("/"), + continuation_token: Some("cursor-0"), + max_keys: 2, + ..Default::default() + }) + .await + .expect("listing should succeed"); + backend.probe().await.expect("probe should succeed"); + + let recorded = recorded.lock().expect("recorder lock").clone(); + assert!(recorded[0].target.starts_with("/storage/v1/b/legacy/o?"), "{}", recorded[0].target); + assert!(recorded[0].target.contains("prefix=dir%2F"), "{}", recorded[0].target); + assert!(recorded[0].target.contains("delimiter=%2F"), "{}", recorded[0].target); + assert!(recorded[0].target.contains("pageToken=cursor-0"), "{}", recorded[0].target); + assert!(recorded[0].target.contains("maxResults=2"), "{}", recorded[0].target); + assert_eq!( + recorded[1].target, "/storage/v1/b/legacy/o?maxResults=1", + "the probe uses the listing permission the pipeline already needs" + ); + } + + #[tokio::test] + async fn gcs_native_backend_satisfies_the_shared_backend_contract() { + let mut ranged = object_headers(); + ranged.push(("Content-Range", "bytes 1-3/5".to_string())); + // A HEAD reports the object size with no body, exactly as GCS does. + let mut head_only = object_headers(); + head_only.push(("Content-Length", "5".to_string())); + let (endpoint, _) = scripted_server(vec![ + ScriptedResponse::new(200, head_only, String::new()), + ScriptedResponse::new(200, object_headers(), "hello".to_string()), + ScriptedResponse::new(206, ranged, "ell".to_string()), + ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()), + ScriptedResponse::new(200, Vec::new(), LIST_PAGE_TWO.to_string()), + // GCS has no tagging call, so the contract's tag step issues no + // request; the probe is the next one on the wire. + ScriptedResponse::new(200, Vec::new(), "{}".to_string()), + ScriptedResponse::new(404, Vec::new(), String::new()), + ScriptedResponse::new(403, Vec::new(), String::new()), + ]) + .await; + + assert_backend_contract( + &backend(&endpoint), + BackendCapabilities { + etag_is_opaque: false, + supports_start_after: false, + // GCS objects have no tags; the contract's tag step is skipped. + supports_tagging: false, + }, + ) + .await; + } +} diff --git a/crates/ecstore/src/bucket/on_demand_migration/mod.rs b/crates/ecstore/src/bucket/on_demand_migration/mod.rs index 0ca4a29ce..6147f1f94 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/mod.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/mod.rs @@ -19,25 +19,36 @@ //! client, and the per-node runtime (`sys`) that turns configs into live //! clients guarded by a breaker, a negative cache, singleflight and a pull //! concurrency limit (rustfs/backlog#2147). +//! +//! A source is reached through one `SourceBackend`: the S3 dialect for every +//! S3-compatible provider, and a native backend for the providers that have no +//! S3 API (`azure`, `gcs_native`). +pub mod azure; +#[cfg(test)] +mod backend_contract; pub mod backfill; pub mod breaker; pub mod config; +pub mod gcs; pub mod list_through; +mod native_http; pub mod negative_cache; pub mod pull; pub mod source_client; pub mod stats; pub mod sys; +#[cfg(test)] +mod test_http_fixture; pub use breaker::{ BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker, BreakerState, BreakerTransition, BreakerVerdict, }; pub use config::{ - ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION, - OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig, - SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext, + AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, + ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, + RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext, }; pub use list_through::{ FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, @@ -57,5 +68,6 @@ pub use stats::{ }; pub use sys::{ ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError, - OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_client_spec, + OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_backend_spec, + source_client_spec, }; diff --git a/crates/ecstore/src/bucket/on_demand_migration/native_http.rs b/crates/ecstore/src/bucket/on_demand_migration/native_http.rs new file mode 100644 index 000000000..881db697d --- /dev/null +++ b/crates/ecstore/src/bucket/on_demand_migration/native_http.rs @@ -0,0 +1,415 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared HTTP transport for the on-demand migration source backends that do +//! not speak S3 (Azure Blob, native GCS). +//! +//! The S3 backend rides the AWS SDK; these providers have no SigV4 dialect, so +//! they talk plain HTTP through one `reqwest` client that carries the same +//! connect/read timeouts and TLS policy the operator configured for the source. +//! Redirects are refused: the endpoint passed the outbound policy gate once, and +//! following a source-chosen `Location` would leave that gate behind. +//! +//! Errors never render the request URL. A SAS token lives in the query string, +//! so a `reqwest` error rendered with its URL would print the credential into +//! the log line and the admin response. + +use super::source_client::{SourceError, SourceHead, SourceTimeouts, USER_AGENT_SUFFIX, classify_status, is_multipart_etag}; +use crate::bucket::remote_s3_client::{RemoteS3ClientError, validate_remote_endpoint, validate_target_ca_pem}; +use aws_sdk_s3::primitives::ByteStream; +use aws_smithy_types::body::SdkBody; +use futures::StreamExt; +use http::HeaderMap; +use std::collections::HashMap; +use std::time::SystemTime; +use time::OffsetDateTime; +use time::format_description::well_known::{Rfc2822, Rfc3339}; +use url::Url; + +/// Origin the native backends are allowed to address, plus the HTTP client +/// that reaches it. +pub(super) struct NativeHttp { + client: reqwest::Client, + endpoint: Url, +} + +impl NativeHttp { + /// `endpoint` must be a bare `scheme://host[:port]` origin; it is checked + /// against the outbound policy exactly like an S3 source endpoint. + pub(super) fn new( + endpoint: &str, + timeouts: SourceTimeouts, + skip_tls_verify: bool, + ca_cert_pem: Option<&str>, + ) -> Result { + let endpoint = Url::parse(endpoint.trim()).map_err(|err| RemoteS3ClientError::InvalidEndpoint(err.to_string()))?; + if !matches!(endpoint.scheme(), "http" | "https") { + return Err(RemoteS3ClientError::InvalidEndpoint(format!( + "unsupported scheme {}; expected http or https", + endpoint.scheme() + ))); + } + if endpoint.host_str().is_none_or(str::is_empty) { + return Err(RemoteS3ClientError::InvalidEndpoint("endpoint has no host".to_string())); + } + if !endpoint.username().is_empty() || endpoint.password().is_some() { + return Err(RemoteS3ClientError::InvalidEndpoint("endpoint must not carry userinfo".to_string())); + } + if !matches!(endpoint.path(), "" | "/") || endpoint.query().is_some() || endpoint.fragment().is_some() { + return Err(RemoteS3ClientError::InvalidEndpoint( + "endpoint must be an origin without path, query or fragment".to_string(), + )); + } + validate_remote_endpoint(&endpoint).map_err(RemoteS3ClientError::EndpointNotAllowed)?; + + let mut builder = reqwest::Client::builder() + .connect_timeout(timeouts.connect) + .read_timeout(timeouts.read) + .redirect(reqwest::redirect::Policy::none()) + .user_agent(USER_AGENT_SUFFIX); + if skip_tls_verify { + builder = builder.danger_accept_invalid_certs(true); + } else if let Some(pem) = ca_cert_pem.map(str::trim).filter(|pem| !pem.is_empty()) { + // Reject a malformed bundle the same way the S3 path does, so the + // operator sees "invalid CA PEM" instead of a TLS handshake failure. + validate_target_ca_pem(pem)?; + let certificate = reqwest::Certificate::from_pem(pem.as_bytes()) + .map_err(|err| RemoteS3ClientError::InvalidCaPem(err.to_string()))?; + builder = builder.add_root_certificate(certificate); + } + + let client = builder + .build() + .map_err(|err| RemoteS3ClientError::InvalidEndpoint(format!("http client cannot be built: {err}")))?; + Ok(Self { client, endpoint }) + } + + #[cfg(test)] + pub(super) fn for_test(endpoint: Url) -> Self { + Self { + client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("test http client should build"), + endpoint, + } + } + + /// A URL under the endpoint origin. `segments` are percent-encoded as + /// path segments, so a key containing `?`, `#` or a space cannot rewrite + /// the request target. + pub(super) fn url<'a>(&self, segments: impl IntoIterator) -> Result { + let mut url = self.endpoint.clone(); + { + let mut path = url + .path_segments_mut() + .map_err(|_| SourceError::Other("source endpoint cannot carry a path".to_string()))?; + path.clear(); + path.extend(segments); + } + Ok(url) + } + + /// Sends the request and returns the response only for a 2xx status. + /// Non-2xx statuses are classified from the status and the provider's own + /// error-code header; response bodies are not read, so no provider message + /// can smuggle credentials or markup into a log line. + pub(super) async fn send( + &self, + request: reqwest::Request, + error_code_header: &str, + ) -> Result { + let response = self.client.execute(request).await.map_err(classify_transport_error)?; + let status = response.status(); + if status.is_success() { + return Ok(response); + } + let code = response + .headers() + .get(error_code_header) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + Err(classify_status( + status.as_u16(), + None, + match &code { + Some(code) => format!("source returned HTTP {status} ({code})"), + None => format!("source returned HTTP {status}"), + }, + )) + } +} + +/// Renders a transport failure without the request URL: a SAS token or a +/// signed query would otherwise reach logs and admin responses. +pub(super) fn classify_transport_error(err: reqwest::Error) -> SourceError { + let is_timeout = err.is_timeout(); + let is_connect = err.is_connect(); + let message = err.without_url().to_string(); + if is_timeout { + SourceError::Timeout + } else if is_connect { + SourceError::Connect(message) + } else { + SourceError::Other(message) + } +} + +/// Streams the response body without buffering it. +pub(super) fn response_body(response: reqwest::Response) -> ByteStream { + let stream = response.bytes_stream().map(|chunk| { + chunk + .map(http_body::Frame::data) + .map_err(|err| std::io::Error::other(err.without_url().to_string())) + }); + ByteStream::new(SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream))) +} + +/// Reads a bounded response body as UTF-8, for the XML and JSON listings. +pub(super) async fn read_text(response: reqwest::Response, max_bytes: usize) -> Result { + let mut body = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(classify_transport_error)?; + if body.len().saturating_add(chunk.len()) > max_bytes { + return Err(SourceError::Other("source listing response exceeded the size limit".to_string())); + } + body.extend_from_slice(&chunk); + } + String::from_utf8(body).map_err(|_| SourceError::Other("source listing response is not valid UTF-8".to_string())) +} + +/// Base64 digest (`Content-MD5`, `md5Hash`, `x-goog-hash`) as lowercase hex. +/// `None` when the value is not a 16-byte digest, so a CRC32C never passes as +/// an MD5. +pub(super) fn base64_md5_to_hex(value: &str) -> Option { + let raw = base64_simd::STANDARD.decode_to_vec(value.trim().as_bytes()).ok()?; + (raw.len() == 16).then(|| faster_hex::hex_string(&raw)) +} + +pub(super) fn header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + headers.get(name).and_then(|value| value.to_str().ok()).map(str::trim) +} + +fn header_string(headers: &HeaderMap, name: &str) -> Option { + header(headers, name).filter(|value| !value.is_empty()).map(str::to_string) +} + +/// `Last-Modified` and friends arrive as an HTTP date; the JSON dialects use +/// RFC 3339 for the same field, so both are accepted. +pub(super) fn parse_http_timestamp(value: &str) -> Option { + OffsetDateTime::parse(value, &Rfc2822) + .or_else(|_| OffsetDateTime::parse(value, &Rfc3339)) + .ok() + .map(SystemTime::from) +} + +/// Provider-specific fields the shared header mapping cannot infer. +pub(super) struct NativeHeadFields { + pub(super) etag: Option, + /// The ETag is an opaque token rather than a digest of the bytes. + pub(super) etag_is_opaque: bool, + pub(super) version_id: Option, + pub(super) storage_class: Option, +} + +/// Maps a HEAD or GET response onto [`SourceHead`]. `metadata_prefix` is the +/// provider's user-metadata header prefix (`x-ms-meta-`, `x-goog-meta-`); the +/// stored shape drops it, matching the `x-amz-meta-` handling of the S3 path. +pub(super) fn native_source_head( + headers: &HeaderMap, + metadata_prefix: &str, + fields: NativeHeadFields, +) -> Result { + let size = header(headers, "content-length") + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| SourceError::Other("source response has no valid content-length".to_string()))?; + + let mut user_metadata = HashMap::new(); + for (name, value) in headers { + let name = name.as_str(); + if let Some(key) = name.strip_prefix(metadata_prefix) + && !key.is_empty() + && let Ok(value) = value.to_str() + { + user_metadata.insert(key.to_string(), value.to_string()); + } + } + + let etag = fields + .etag + .map(|etag| etag.trim().trim_matches('"').to_string()) + .filter(|etag| !etag.is_empty()); + // An opaque ETag never encodes a part count, so the multipart flag stays + // false for it however the provider happens to spell the token. + let is_multipart_etag = !fields.etag_is_opaque && etag.as_deref().is_some_and(is_multipart_etag); + + Ok(SourceHead { + etag, + size, + last_modified: header(headers, "last-modified").and_then(parse_http_timestamp), + content_type: header_string(headers, "content-type"), + content_encoding: header_string(headers, "content-encoding"), + content_disposition: header_string(headers, "content-disposition"), + content_language: header_string(headers, "content-language"), + cache_control: header_string(headers, "cache-control"), + expires: header_string(headers, "expires"), + user_metadata, + version_id: fields.version_id, + storage_class: fields.storage_class, + // Neither native provider hands back ciphertext: a customer-key object + // is refused by the backend before it reaches this mapping, and the + // service-managed encryption is transparent to the reader. + sse: None, + is_multipart_etag, + etag_is_opaque: fields.etag_is_opaque, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use http::HeaderValue; + + fn headers(pairs: &[(&str, &str)]) -> HeaderMap { + let mut headers = HeaderMap::new(); + for (name, value) in pairs { + headers.insert( + http::HeaderName::from_bytes(name.as_bytes()).expect("test header name"), + HeaderValue::from_str(value).expect("test header value"), + ); + } + headers + } + + fn fields() -> NativeHeadFields { + NativeHeadFields { + etag: None, + etag_is_opaque: false, + version_id: None, + storage_class: None, + } + } + + #[test] + fn native_source_head_maps_content_headers_and_prefixed_metadata() { + let headers = headers(&[ + ("content-length", "1234"), + ("content-type", "text/plain"), + ("content-encoding", "gzip"), + ("content-language", "en"), + ("content-disposition", "attachment"), + ("cache-control", "max-age=60"), + ("expires", "Thu, 01 Jan 2026 00:00:00 GMT"), + ("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ("x-ms-meta-owner", "alice"), + ("x-goog-meta-owner", "not-mine"), + ]); + let head = native_source_head( + &headers, + "x-ms-meta-", + NativeHeadFields { + etag: Some("\"0x8DCE1D2\"".to_string()), + etag_is_opaque: true, + version_id: Some("2026-01-01T00:00:00.0000000Z".to_string()), + storage_class: Some("Hot".to_string()), + }, + ) + .expect("head should map"); + + assert_eq!(head.size, 1234); + assert_eq!(head.content_type.as_deref(), Some("text/plain")); + assert_eq!(head.content_encoding.as_deref(), Some("gzip")); + assert_eq!(head.content_language.as_deref(), Some("en")); + assert_eq!(head.content_disposition.as_deref(), Some("attachment")); + assert_eq!(head.cache_control.as_deref(), Some("max-age=60")); + assert_eq!(head.expires.as_deref(), Some("Thu, 01 Jan 2026 00:00:00 GMT")); + assert_eq!( + head.last_modified, + Some(SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_445_412_480)), + "HTTP-date Last-Modified must parse" + ); + assert_eq!( + head.user_metadata, + HashMap::from([("owner".to_string(), "alice".to_string())]), + "only the provider's own metadata prefix is read" + ); + assert_eq!(head.etag.as_deref(), Some("0x8DCE1D2"), "quotes are stripped, the token is kept"); + assert!(head.etag_is_opaque); + assert!(!head.is_multipart_etag); + assert_eq!(head.storage_class.as_deref(), Some("Hot")); + assert!(head.sse.is_none()); + } + + #[test] + fn native_source_head_requires_a_content_length() { + let err = native_source_head(&headers(&[("content-type", "text/plain")]), "x-ms-meta-", fields()) + .expect_err("a response without content-length is unusable"); + assert!(matches!(err, SourceError::Other(_)), "{err:?}"); + } + + #[test] + fn opaque_etag_never_reads_as_a_multipart_etag() { + // A digest-shaped ETag keeps the S3 reading; the same string marked + // opaque must not be split into "digest-partcount". + for (opaque, expected) in [(false, true), (true, false)] { + let head = native_source_head( + &headers(&[("content-length", "1")]), + "x-ms-meta-", + NativeHeadFields { + etag: Some("d41d8cd98f00b204e9800998ecf8427e-3".to_string()), + etag_is_opaque: opaque, + ..fields() + }, + ) + .expect("head should map"); + assert_eq!(head.is_multipart_etag, expected, "opaque = {opaque}"); + } + } + + #[test] + fn base64_md5_converts_only_sixteen_byte_digests() { + assert_eq!( + base64_md5_to_hex("1B2M2Y8AsgTpgAmY7PhCfg==").as_deref(), + Some("d41d8cd98f00b204e9800998ecf8427e") + ); + assert_eq!(base64_md5_to_hex("not base64!").as_deref(), None); + // A CRC32C digest is four bytes: it must not pass as an MD5. + assert_eq!(base64_md5_to_hex("AAAAAA==").as_deref(), None); + } + + #[test] + fn native_http_rejects_endpoints_that_are_not_bare_origins() { + for bad in [ + "ftp://source.example.com", + "https://user:pw@source.example.com", + "https://source.example.com/container", + "https://source.example.com/?x=1", + "not a url", + ] { + assert!( + NativeHttp::new(bad, SourceTimeouts::default(), false, None).is_err(), + "{bad} must be rejected" + ); + } + } + + #[test] + fn native_http_percent_encodes_every_path_segment() { + let http = NativeHttp::for_test(Url::parse("https://acct.blob.core.windows.net").expect("origin")); + let url = http.url(["container", "dir", "a b?c#d.txt"]).expect("url should build"); + assert_eq!(url.as_str(), "https://acct.blob.core.windows.net/container/dir/a%20b%3Fc%23d.txt"); + assert_eq!(url.query(), None, "a key with '?' must not become a query"); + } +} diff --git a/crates/ecstore/src/bucket/on_demand_migration/pull.rs b/crates/ecstore/src/bucket/on_demand_migration/pull.rs index df3dd6462..8d18bdbdd 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/pull.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/pull.rs @@ -1118,6 +1118,8 @@ mod tests { session_token: None, }), tls: TlsConfig::default(), + azure: None, + gcs: None, }, filter: FilterConfig::default(), policy: PolicyConfig::default(), diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs index a97cf5a0a..618fff00e 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs @@ -25,6 +25,8 @@ //! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never //! forwarded: v1 rejects SSE-C source objects outright. +use super::azure::AzureSourceBackend; +use super::gcs::GcsNativeSourceBackend; use super::list_through::{ListPageError, validate_list_page}; use crate::bucket::remote_s3_client::{ PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config, @@ -65,6 +67,10 @@ pub enum SourceProvider { /// Generic S3-compatible service. #[default] S3, + /// Native Azure Blob service; not an S3 dialect. + Azure, + /// Native GCS JSON API with a service-account key; not an S3 dialect. + GcsNative, } impl SourceProvider { @@ -76,6 +82,8 @@ impl SourceProvider { "minio" => Some(Self::Minio), "rustfs" => Some(Self::Rustfs), "s3" => Some(Self::S3), + "azure" => Some(Self::Azure), + "gcs_native" => Some(Self::GcsNative), _ => None, } } @@ -88,6 +96,8 @@ impl SourceProvider { Self::Minio => "minio", Self::Rustfs => "rustfs", Self::S3 => "s3", + Self::Azure => "azure", + Self::GcsNative => "gcs_native", } } @@ -159,6 +169,69 @@ pub struct SourceClientSpec { /// Bytes per second the pull pipeline may consume from this source; /// `None` means unlimited. Enforced by the consumer, not by this client. pub bandwidth_limit: Option, + /// Which [`SourceBackend`] to build. The S3 variant reads `region`, + /// `path_style` and `credentials`; the native variants ignore all three + /// and carry their own credentials. + pub backend: SourceBackendSpec, +} + +/// Provider-specific half of [`SourceClientSpec`]. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum SourceBackendSpec { + #[default] + S3, + Azure(AzureSourceSpec), + Gcs(GcsSourceSpec), +} + +/// Native Azure Blob parameters. The container is [`SourceClientSpec::bucket`]. +#[derive(Clone, PartialEq, Eq)] +pub struct AzureSourceSpec { + pub account: String, + pub auth: AzureAuth, +} + +impl fmt::Debug for AzureSourceSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AzureSourceSpec") + .field("account", &self.account) + .field("auth", &self.auth) + .finish() + } +} + +/// How Azure requests are authorized. +#[derive(Clone, PartialEq, Eq)] +pub enum AzureAuth { + /// Base64 storage-account key, signed per request with Shared Key. + SharedKey(String), + /// SAS query string without the leading `?`, appended to every URL. + Sas(String), +} + +impl fmt::Debug for AzureAuth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Both variants are secrets; only the scheme may be rendered. + f.write_str(match self { + Self::SharedKey(_) => "SharedKey(REDACTED)", + Self::Sas(_) => "Sas(REDACTED)", + }) + } +} + +/// Native GCS parameters. The bucket is [`SourceClientSpec::bucket`]. +#[derive(Clone, PartialEq, Eq)] +pub struct GcsSourceSpec { + /// Service-account key JSON. + pub service_account_json: String, +} + +impl fmt::Debug for GcsSourceSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GcsSourceSpec") + .field("service_account_json", &"REDACTED") + .finish() + } } impl SourceClientSpec { @@ -272,7 +345,7 @@ const ACCESS_DENIED_CODES: &[&str] = &[ "InvalidToken", ]; -fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError { +pub(super) fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError { if let Some(code) = code { if THROTTLE_CODES.contains(&code) { return SourceError::Throttled; @@ -344,6 +417,11 @@ pub struct SourceHead { pub storage_class: Option, pub sse: Option, pub is_multipart_etag: bool, + /// The provider's ETag is not derived from the object bytes (Azure + /// stamps an opaque concurrency token). Such an ETag is recorded for + /// provenance but must never be read as a content digest, so the + /// write-back path refuses to use it as the expected MD5. + pub etag_is_opaque: bool, } /// Per-operation fields shared by HEAD and GET outputs. @@ -365,7 +443,7 @@ struct HeadParts { sse_customer_algorithm: Option, } -fn normalize_etag(etag: Option) -> Option { +pub(super) fn normalize_etag(etag: Option) -> Option { etag.map(|etag| etag.trim().trim_matches('"').to_string()) .filter(|etag| !etag.is_empty()) } @@ -414,6 +492,7 @@ fn source_head(parts: HeadParts) -> Result { storage_class: parts.storage_class, sse, is_multipart_etag, + etag_is_opaque: false, }) } @@ -624,9 +703,48 @@ impl fmt::Debug for SourceClient { impl SourceClient { pub async fn new(spec: &SourceClientSpec) -> Result { - let endpoint = spec.endpoint_spec()?; - let config = build_remote_s3_config(&endpoint).await?; - Ok(Self::from_config_builder(config, endpoint.endpoint_url(), spec)) + match &spec.backend { + SourceBackendSpec::S3 => { + let endpoint = spec.endpoint_spec()?; + let config = build_remote_s3_config(&endpoint).await?; + Ok(Self::from_config_builder(config, endpoint.endpoint_url(), spec)) + } + SourceBackendSpec::Azure(azure) => { + let backend = AzureSourceBackend::new( + &spec.endpoint, + &spec.bucket, + azure, + spec.timeouts, + spec.skip_tls_verify, + spec.ca_cert_pem.as_deref(), + )?; + Ok(Self::from_backend(Box::new(backend), spec)) + } + SourceBackendSpec::Gcs(gcs) => { + let backend = GcsNativeSourceBackend::new( + &spec.endpoint, + &spec.bucket, + gcs, + spec.timeouts, + spec.skip_tls_verify, + spec.ca_cert_pem.as_deref(), + )?; + Ok(Self::from_backend(Box::new(backend), spec)) + } + } + } + + /// Wraps a ready backend in the prefix-mapping client. The endpoint is + /// kept only for `Debug` and admin status. + fn from_backend(backend: Box, spec: &SourceClientSpec) -> Self { + Self { + backend, + endpoint: spec.endpoint.clone(), + bucket: spec.bucket.clone(), + source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()), + timeouts: spec.timeouts, + bandwidth_limit: spec.bandwidth_limit, + } } /// `config` must come from [`SourceClientSpec::endpoint_spec`], which is @@ -871,6 +989,7 @@ fn s3_source_object(object: SdkObject) -> Result { #[cfg(test)] mod tests { use super::*; + use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, OBJECT_MD5, assert_backend_contract}; use aws_smithy_runtime_api::client::http::{HttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn}; use aws_smithy_runtime_api::client::orchestrator::HttpRequest; use aws_smithy_runtime_api::client::result::ConnectorError; @@ -988,6 +1107,7 @@ mod tests { retry: RemoteS3RetryPolicy::Disabled, timeouts: SourceTimeouts::default(), bandwidth_limit: NonZeroU64::new(1_000_000), + backend: SourceBackendSpec::S3, } } @@ -1615,7 +1735,101 @@ mod tests { assert_eq!(resolve_path_style(PathStyle::VirtualHost, Minio, "10.0.0.1"), PathStyle::VirtualHost); assert_eq!(resolve_path_style(PathStyle::Path, Aws, "s3.amazonaws.com"), PathStyle::Path); assert_eq!(SourceProvider::from_label(" AWS "), Some(Aws)); - assert_eq!(SourceProvider::from_label("azure"), None); + assert_eq!(SourceProvider::from_label(" Azure "), Some(Azure)); + assert_eq!(SourceProvider::from_label("gcs_native"), Some(GcsNative)); + assert_eq!(SourceProvider::from_label("swift"), None); + } + + const CONTRACT_LIST_PAGE_ONE: &str = r#" + + source-bucket + true + cursor-1 + + dir/a.txt + 2015-10-21T07:28:00.000Z + "5d41402abc4b2a76b9719d911017c592" + 5 + STANDARD + + dir/sub/ +"#; + + const CONTRACT_LIST_PAGE_TWO: &str = r#" + + source-bucket + false + + dir/b.txt + 2015-10-21T07:28:00.000Z + "7d41402abc4b2a76b9719d911017c592" + 7 + +"#; + + const CONTRACT_TAGGING: &str = r#" + + envprod +"#; + + fn contract_object_headers(content_length: u64) -> Vec<(&'static str, String)> { + vec![ + ("etag", format!("\"{OBJECT_MD5}\"")), + ("content-length", content_length.to_string()), + ("content-type", "text/plain".to_string()), + ("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()), + ("x-amz-meta-owner", "alice".to_string()), + ("x-amz-storage-class", "STANDARD".to_string()), + ] + } + + /// The S3 backend behind the scripted connector, without the prefix-mapping + /// client on top: the contract is a property of the backend itself. + async fn scripted_s3_backend(responses: Vec) -> S3SourceBackend { + let spec = spec(None); + let connector = SharedHttpConnector::new(ScriptedConnector { + requests: Arc::new(Mutex::new(Vec::new())), + responses: Arc::new(Mutex::new(responses.into_iter().collect())), + }); + let http_client = http_client_fn(move |_settings, _components| connector.clone()); + let endpoint = spec.endpoint_spec().expect("test spec endpoint should parse"); + let config = build_remote_s3_config(&endpoint) + .await + .expect("test spec should build") + .http_client(http_client) + .interceptor(SourceProxyMarkerInterceptor::new()); + S3SourceBackend { + client: S3Client::from_conf(config.build()), + bucket: spec.bucket.clone(), + } + } + + #[tokio::test] + async fn s3_backend_satisfies_the_shared_backend_contract() { + let mut ranged = contract_object_headers(3); + ranged.push(("content-range", "bytes 1-3/5".to_string())); + let backend = scripted_s3_backend(vec![ + ok(contract_object_headers(5), ""), + ok(contract_object_headers(5), "hello"), + ok(ranged, "ell"), + ok(Vec::new(), CONTRACT_LIST_PAGE_ONE), + ok(Vec::new(), CONTRACT_LIST_PAGE_TWO), + ok(Vec::new(), CONTRACT_TAGGING), + ok(Vec::new(), ""), + status(404, ""), + status(403, ACCESS_DENIED_BODY), + ]) + .await; + + assert_backend_contract( + &backend, + BackendCapabilities { + etag_is_opaque: false, + supports_start_after: true, + supports_tagging: true, + }, + ) + .await; } fn prefix_client(prefix: Option) -> SourceClient { diff --git a/crates/ecstore/src/bucket/on_demand_migration/sys.rs b/crates/ecstore/src/bucket/on_demand_migration/sys.rs index 16cb8ea65..6c348749e 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/sys.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/sys.rs @@ -47,7 +47,10 @@ use super::config::{ use super::list_through::{SOURCE_LIST_RATE_PER_SEC, SourceListRateLimiter}; use super::negative_cache::NegativeCache; use super::pull::{OdmWriteBack, PullQueue}; -use super::source_client::{SourceClient, SourceClientSpec, SourceError, SourceProvider, SourceTimeouts}; +use super::source_client::{ + AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError, SourceProvider, + SourceTimeouts, +}; use super::stats::{GaugeGuard, OdmStats, OdmStatsSnapshot, PullFailureReason}; use crate::bucket::remote_s3_client::{ PathStyle as ClientPathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy, @@ -619,6 +622,7 @@ pub fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClientSpec // load on a source that is already failing. retry: RemoteS3RetryPolicy::Disabled, bandwidth_limit: policy.bandwidth_limit_bytes_per_sec.and_then(NonZeroU64::new), + backend: source_backend_spec(source), } } @@ -630,6 +634,31 @@ fn source_provider(provider: Provider) -> SourceProvider { Provider::Rustfs => SourceProvider::Rustfs, Provider::R2 => SourceProvider::R2, Provider::Gcs => SourceProvider::Gcs, + Provider::Azure => SourceProvider::Azure, + Provider::GcsNative => SourceProvider::GcsNative, + } +} + +/// Which backend the client builds. A native provider whose block is missing +/// falls back to the S3 spec, where the builder reports the missing +/// credentials: the config layer already refuses to store that shape, so this +/// only covers a config written by an older or hand-edited build. +pub fn source_backend_spec(source: &SourceConfig) -> SourceBackendSpec { + match (source.provider, source.azure.as_ref(), source.gcs.as_ref()) { + (Provider::Azure, Some(azure), _) => SourceBackendSpec::Azure(AzureSourceSpec { + account: azure.account.clone(), + auth: match (&azure.account_key, &azure.sas_token) { + (Some(key), _) => AzureAuth::SharedKey(key.clone()), + (None, Some(sas)) => AzureAuth::Sas(sas.clone()), + // Refused by `SourceConfig::validate`; an empty shared key + // fails closed at the builder rather than signing with none. + (None, None) => AzureAuth::SharedKey(String::new()), + }, + }), + (Provider::GcsNative, _, Some(gcs)) => SourceBackendSpec::Gcs(GcsSourceSpec { + service_account_json: gcs.service_account_json.clone(), + }), + _ => SourceBackendSpec::S3, } } @@ -929,6 +958,8 @@ mod tests { session_token: None, }), tls: TlsConfig::default(), + azure: None, + gcs: None, }, filter: FilterConfig { prefix: prefix.map(str::to_string), diff --git a/crates/ecstore/src/bucket/on_demand_migration/test_http_fixture.rs b/crates/ecstore/src/bucket/on_demand_migration/test_http_fixture.rs new file mode 100644 index 000000000..eb6dda014 --- /dev/null +++ b/crates/ecstore/src/bucket/on_demand_migration/test_http_fixture.rs @@ -0,0 +1,120 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Scripted HTTP server for the native source backends' tests. +//! +//! The S3 backend can be driven through the SDK's own connector; the native +//! backends talk to a real socket, so their tests need a server that answers a +//! fixed script and records what it was asked. Every response closes its +//! connection, which keeps one request on one socket and makes the script order +//! exactly the request order. + +use std::sync::{Arc, Mutex}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use url::Url; + +pub(super) struct ScriptedResponse { + status: u16, + headers: Vec<(&'static str, String)>, + body: String, +} + +impl ScriptedResponse { + pub(super) fn new(status: u16, headers: Vec<(&'static str, String)>, body: String) -> Self { + Self { status, headers, body } + } +} + +#[derive(Clone, Debug)] +pub(super) struct RecordedRequest { + pub(super) method: String, + /// Request target as it appeared on the wire: path plus query. + pub(super) target: String, + pub(super) headers: Vec<(String, String)>, +} + +impl RecordedRequest { + pub(super) fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } +} + +pub(super) type Recorder = Arc>>; + +/// Binds a loopback listener that answers `responses` in order and returns its +/// origin plus the recorder. The task ends once the script is exhausted. +pub(super) async fn scripted_server(responses: Vec) -> (Url, Recorder) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("fixture listener should bind"); + let port = listener.local_addr().expect("fixture address").port(); + let recorder: Recorder = Arc::new(Mutex::new(Vec::new())); + let sink = Arc::clone(&recorder); + + tokio::spawn(async move { + for response in responses { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let mut request = Vec::new(); + let mut buffer = [0_u8; 2048]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + match stream.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => request.extend_from_slice(&buffer[..read]), + } + } + let text = String::from_utf8_lossy(&request).into_owned(); + let mut lines = text.lines(); + let start = lines.next().unwrap_or_default().to_string(); + let mut parts = start.split_whitespace(); + sink.lock().expect("recorder lock").push(RecordedRequest { + method: parts.next().unwrap_or_default().to_string(), + target: parts.next().unwrap_or_default().to_string(), + headers: lines + .take_while(|line| !line.is_empty()) + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.trim().to_string(), value.trim().to_string())) + .collect(), + }); + + // A scripted HEAD declares the object size in its own headers while + // carrying no body, so an explicit `Content-Length` wins over the + // body length. + let declares_length = response + .headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("content-length")); + let mut rendered = match declares_length { + true => format!("HTTP/1.1 {} Scripted\r\nConnection: close\r\n", response.status), + false => format!( + "HTTP/1.1 {} Scripted\r\nContent-Length: {}\r\nConnection: close\r\n", + response.status, + response.body.len() + ), + }; + for (name, value) in response.headers { + rendered.push_str(&format!("{name}: {value}\r\n")); + } + rendered.push_str("\r\n"); + rendered.push_str(&response.body); + let _ = stream.write_all(rendered.as_bytes()).await; + let _ = stream.flush().await; + } + }); + + (Url::parse(&format!("http://127.0.0.1:{port}")).expect("fixture endpoint"), recorder) +} diff --git a/crates/madmin/fixtures/on_demand_migration/get_response.json b/crates/madmin/fixtures/on_demand_migration/get_response.json index aff3a808b..4101403c3 100644 --- a/crates/madmin/fixtures/on_demand_migration/get_response.json +++ b/crates/madmin/fixtures/on_demand_migration/get_response.json @@ -1 +1 @@ -{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"} +{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"} diff --git a/crates/madmin/fixtures/on_demand_migration/set_request.json b/crates/madmin/fixtures/on_demand_migration/set_request.json index e5b7fb03a..a67d359da 100644 --- a/crates/madmin/fixtures/on_demand_migration/set_request.json +++ b/crates/madmin/fixtures/on_demand_migration/set_request.json @@ -1 +1 @@ -{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}} +{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}} diff --git a/crates/madmin/fixtures/on_demand_migration/set_response.json b/crates/madmin/fixtures/on_demand_migration/set_response.json index 81bf69669..2ea089aa3 100644 --- a/crates/madmin/fixtures/on_demand_migration/set_response.json +++ b/crates/madmin/fixtures/on_demand_migration/set_response.json @@ -1 +1 @@ -{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}} +{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}} diff --git a/crates/madmin/src/on_demand_migration.rs b/crates/madmin/src/on_demand_migration.rs index 941ab0233..0fce8af6c 100644 --- a/crates/madmin/src/on_demand_migration.rs +++ b/crates/madmin/src/on_demand_migration.rs @@ -78,10 +78,18 @@ pub struct OnDemandMigrationSource { #[serde(default)] pub path_style: OnDemandMigrationPathStyle, /// `None` means anonymous access to a public source bucket. + /// `None` means anonymous access to a public source bucket. The native + /// providers carry their credentials in `azure` / `gcs` instead. #[serde(default)] pub credentials: Option, #[serde(default)] pub tls: OnDemandMigrationTls, + /// Required for `azure` and rejected for every other provider. + #[serde(default)] + pub azure: Option, + /// Required for `gcs_native` and rejected for every other provider. + #[serde(default)] + pub gcs: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -92,7 +100,49 @@ pub enum OnDemandMigrationProvider { Minio, Rustfs, R2, + /// GCS XML interoperability API with HMAC keys. Gcs, + /// Native Azure Blob service. + Azure, + /// Native GCS JSON API with a service-account key. + #[serde(rename = "gcs_native")] + GcsNative, +} + +/// Native Azure Blob parameters. The container is `source.bucket`; exactly one +/// of `account_key` and `sas_token` is set. Responses carry both as `REDACTED`. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnDemandMigrationAzure { + pub account: String, + #[serde(default)] + pub account_key: Option, + #[serde(default)] + pub sas_token: Option, +} + +impl fmt::Debug for OnDemandMigrationAzure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OnDemandMigrationAzure") + .field("account", &self.account) + .field("account_key", &self.account_key.as_ref().map(|_| "REDACTED")) + .field("sas_token", &self.sas_token.as_ref().map(|_| "REDACTED")) + .finish() + } +} + +/// Native GCS parameters. The bucket is `source.bucket`; the key JSON embeds a +/// private key, so responses carry it as `REDACTED`. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnDemandMigrationGcs { + pub service_account_json: String, +} + +impl fmt::Debug for OnDemandMigrationGcs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OnDemandMigrationGcs") + .field("service_account_json", &"REDACTED") + .finish() + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] @@ -806,6 +856,8 @@ mod tests { session_token: None, }), tls: OnDemandMigrationTls::default(), + azure: None, + gcs: None, }); let mut expected: OnDemandMigrationConfig = serde_json::from_str(SET_REQUEST_FIXTURE.trim()).expect("fixture"); expected.filter.source_prefix = None; @@ -821,6 +873,42 @@ mod tests { assert!(minimal.source.credentials.is_none()); } + #[test] + fn native_provider_documents_round_trip_and_hide_their_secrets() { + for (label, json) in [ + ( + "azure", + r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"},"gcs":null}"#, + ), + ( + "gcs_native", + r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#, + ), + ] { + let source: OnDemandMigrationSource = serde_json::from_str(json).unwrap_or_else(|err| panic!("{label}: {err}")); + assert_eq!( + serde_json::to_string(&source).expect("re-encodes"), + json, + "{label} must reproduce the server wire shape byte for byte" + ); + } + + let azure = OnDemandMigrationAzure { + account: "legacyaccount".to_string(), + account_key: Some("c2VjcmV0".to_string()), + sas_token: Some("sig=topsecret".to_string()), + }; + let rendered = format!("{azure:?}"); + assert!(rendered.contains("legacyaccount")); + assert!(!rendered.contains("c2VjcmV0"), "{rendered}"); + assert!(!rendered.contains("topsecret"), "{rendered}"); + + let gcs = OnDemandMigrationGcs { + service_account_json: r#"{"private_key":"-----BEGIN PRIVATE KEY-----"}"#.to_string(), + }; + assert!(!format!("{gcs:?}").contains("PRIVATE KEY"), "{gcs:?}"); + } + #[test] fn credentials_debug_never_prints_secrets() { let credentials = OnDemandMigrationCredentials { diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md index 8155c6f11..d253e33a2 100644 --- a/docs/operations/on-demand-migration.md +++ b/docs/operations/on-demand-migration.md @@ -103,14 +103,20 @@ The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unkno |---|---|---|---| | `version` | integer | `1` | Must be `1` | | `enabled` | bool | `true` | `false` keeps the config but stops all source traffic | -| `source.provider` | `s3` \| `aws` \| `minio` \| `rustfs` \| `r2` \| `gcs` | — (required) | Drives endpoint and addressing defaults | -| `source.endpoint` | string \| null | — | `http(s)://host[:port]`, no path, query, fragment or userinfo. Required for every provider except `aws`, where it is derived from `region` | -| `source.region` | string | — (required) | Non-empty. `auto` is accepted only for `r2`, `minio`, `rustfs` and is signed as `us-east-1` | -| `source.bucket` | string | — (required) | Non-empty, no `/` and no whitespace | +| `source.provider` | `s3` \| `aws` \| `minio` \| `rustfs` \| `r2` \| `gcs` \| `azure` \| `gcs_native` | — (required) | Drives endpoint and addressing defaults, and which backend the client builds: every value but `azure` and `gcs_native` speaks S3 | +| `source.endpoint` | string \| null | — | `http(s)://host[:port]`, no path, query, fragment or userinfo. Required except for `aws` (derived from `region`), `azure` (derived as `https://.blob.core.windows.net`) and `gcs_native` (`https://storage.googleapis.com`). Set it explicitly to point at Azurite or fake-gcs-server, subject to the same outbound policy as any other source endpoint | +| `source.region` | string | — (required) | Non-empty. `auto` is accepted for `r2`, `minio`, `rustfs` and for the native providers, and is signed as `us-east-1`. `azure` and `gcs_native` never sign with a region, so `auto` is the honest value there | +| `source.bucket` | string | — (required) | Non-empty, no `/` and no whitespace. For `azure` this is the container name, for `gcs_native` the bucket name; the provider block never repeats it | | `source.path_style` | `auto` \| `path` \| `virtual` | `auto` | `auto` resolves to path-style for IP-literal or `localhost` endpoints and for `s3`/`minio`/`rustfs`; virtual-host for `aws`/`gcs`/`r2` | -| `source.credentials` | object \| null | `null` | `null` means anonymous, which the client builder does not support yet: the admin `PUT` refuses it with `InvalidArgument`, and a config that reached the metadata another way resolves as unavailable. `access_key` and `secret_key` must be non-empty; `session_token` is optional but must be non-empty when present | +| `source.credentials` | object \| null | `null` | Read only by the S3 providers; `azure` and `gcs_native` must leave it `null` and carry their credentials in their own block. `null` means anonymous, which the client builder does not support yet: the admin `PUT` refuses it with `InvalidArgument`, and a config that reached the metadata another way resolves as unavailable. `access_key` and `secret_key` must be non-empty; `session_token` is optional but must be non-empty when present | | `source.tls.skip_verify` | bool | `false` | Disables certificate verification for the source connection | | `source.tls.ca_cert_pem` | string \| null | `null` | Must contain `-----BEGIN CERTIFICATE-----` | +| `source.azure` | object \| null | `null` | Required for `provider = "azure"` and rejected for every other provider | +| `source.azure.account` | string | — (required) | Storage account name; `[A-Za-z0-9-]` only, because it becomes the first label of the derived host | +| `source.azure.account_key` | string \| null | `null` | Base64 storage-account key, signed per request with Shared Key. Mutually exclusive with `sas_token`; exactly one of the two is required | +| `source.azure.sas_token` | string \| null | `null` | SAS query string without the leading `?` and without whitespace, appended to every request URL | +| `source.gcs` | object \| null | `null` | Required for `provider = "gcs_native"` and rejected for every other provider | +| `source.gcs.service_account_json` | string | — (required) | Service-account key JSON; must parse and carry `type: service_account`, `client_email` and `private_key`. Tokens are minted read-only (`devstorage.read_only`) | | `filter.prefix` | string \| null | `null` | Null or non-empty. Only local keys with this prefix consult the source | | `filter.source_prefix` | string \| null | `null` | Null or non-empty. Prepended to the local key to form the source key | | `policy.head` | `proxy` \| `local_only` | `proxy` | `local_only` answers a HEAD miss with 404 and no source traffic | @@ -119,7 +125,7 @@ The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unkno | `policy.list_through` | bool | `false` | Merges the source listing into `ListObjectsV2` so clients see the whole namespace during the migration. Off by default: it puts the source in the path of every listing | | `policy.respect_local_delete_marker` | bool | `true` | A local delete marker is the final answer; only a versioned bucket can produce one | | `policy.preserve_etag` | bool | `true` | Keeps the source ETag on the stored object unless the bucket encrypts by default | -| `policy.copy_tags` | bool | `false` | Copies source object tags; needs `s3:GetObjectTagging` and costs one extra source call per inline pull | +| `policy.copy_tags` | bool | `false` | Copies source object tags; needs `s3:GetObjectTagging` and costs one extra source call per inline pull. `azure` reads blob tags instead; `gcs_native` has no tags and always finds none | | `policy.emit_events` | bool | `true` | Whether a write-back emits `ObjectCreated` notifications | | `policy.negative_cache_ttl_secs` | integer | `30` | `0..=3600`; `0` disables the negative cache | | `policy.inline_max_bytes` | integer | `16777216` (16 MiB) | `0..=268435456` (256 MiB). At or below this size a GET miss is teed inline; above it the response streams through and a background pull stores the object | @@ -145,8 +151,14 @@ Validation also rejects two shapes outright: a source whose endpoint and bucket | `rustfs` | Required | Path-style | `auto` allowed | A RustFS source answers the migration request locally thanks to the anti-loop marker | `real_source_test.rs` in the `e2e-nightly` lane | | `r2` | `https://.r2.cloudflarestorage.com` | Virtual-host | `auto` allowed (signed as `us-east-1`) | | `cloud-source (r2)`, only while `ODM_INTEROP_R2_*` are configured; no difference recorded yet | | `gcs` | `https://storage.googleapis.com` | Virtual-host | Real region required | Uses the GCS XML interoperability API with an HMAC key pair, not a service-account JSON key | `cloud-source (gcs)`, only while `ODM_INTEROP_GCS_HMAC_*` are configured; no difference recorded yet | +| `azure` | Optional; derived as `https://.blob.core.windows.net` | Native Blob REST, not S3 | Unused; write `auto` | Needs `source.azure`; the container is `source.bucket`. Reads need `Read` on the blob and `List` on the container, plus `Tags` when `policy.copy_tags` is on | None yet: no interop job covers Azure | +| `gcs_native` | Optional; derived as `https://storage.googleapis.com` | Native GCS API, not S3 | Unused; write `auto` | Needs `source.gcs`. Reads use the XML API for objects and `objects.list` for listings, both with an OAuth token minted from the service-account key; the key needs `storage.objects.get` and `storage.objects.list` | None yet: no interop job covers native GCS | -Azure Blob has no preset; a native provider is deferred (rustfs/backlog#2166). +Every backend answers the same trait contract, pinned by `backend_contract.rs` in `crates/ecstore/src/bucket/on_demand_migration/`, and the three differences that contract allows are the ones documented here. + +`azure` differs in two of them. Its ETag is a concurrency token rather than a digest of the bytes, so it is stored as `odm-source-etag` provenance and never used as the expected MD5 of a pulled object — the write-back integrity check falls back to the local digest. And its listing paginates only with an opaque marker: there is no "start after this key" form, so a caller that asks for one gets `Unsupported` instead of a listing that silently starts over. + +`gcs_native` differs in the other two. Its listing also has no exclusive "start after" form (`startOffset` is inclusive), so it refuses one the same way. And GCS has no object tagging at all: `policy.copy_tags` finds no tags rather than failing the pull, because GCS custom metadata is already carried by the head mapping. Its ETag is normally usable: the `x-goog-hash` MD5 is converted to hex and checked against the pulled bytes, except on a composite object, which has no MD5 and whose ETag is then treated as opaque. The "Interop evidence" column names the job in `.github/workflows/on-demand-migration-interop.yml` (rustfs/backlog#2167) that last exercised the preset against a real implementation, and is where a provider difference belongs once the lane finds one. That lane is report-only and scheduled: it runs `crates/e2e_test/src/on_demand_migration/interop_test.rs` — the same case bodies as the merge-gate suite, with the source injected through `RUSTFS_ODM_INTEROP_*` — against a pinned MinIO container, and against each cloud provider whose repository secrets are configured. A provider without secrets is skipped with a note in the run summary rather than failing, so "no difference recorded yet" means exactly that and not "verified clean"; see [ci-gates.md](../testing/ci-gates.md) for the row. diff --git a/rustfs/src/admin/handlers/on_demand_migration.rs b/rustfs/src/admin/handlers/on_demand_migration.rs index 5fdf3b2f4..103e6ee8d 100644 --- a/rustfs/src/admin/handlers/on_demand_migration.rs +++ b/rustfs/src/admin/handlers/on_demand_migration.rs @@ -46,6 +46,7 @@ use crate::admin::storage_api::bucket::on_demand_migration::source_client::{ }; use crate::admin::storage_api::bucket::on_demand_migration::{ OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext, + source_backend_spec, }; use crate::admin::storage_api::bucket::remote_s3_client::{ PathStyle as RemotePathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy, @@ -585,6 +586,8 @@ fn source_provider(config: &OnDemandMigrationConfig) -> SourceProvider { Provider::Rustfs => SourceProvider::Rustfs, Provider::R2 => SourceProvider::R2, Provider::Gcs => SourceProvider::Gcs, + Provider::Azure => SourceProvider::Azure, + Provider::GcsNative => SourceProvider::GcsNative, } } @@ -621,6 +624,9 @@ pub(crate) fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClie // a flapping source behind a success and triple the probe's cost. retry: RemoteS3RetryPolicy::Disabled, bandwidth_limit: config.policy.bandwidth_limit_bytes_per_sec.and_then(NonZeroU64::new), + // One mapping serves the probe and the runtime, so an admin probe + // always exercises the backend the runtime will build. + backend: source_backend_spec(source), } } diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 03adac373..ea7ae56b5 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -292,6 +292,7 @@ pub(crate) mod on_demand_migration { pub(crate) type PathStyle = super::ecstore_bucket::on_demand_migration::PathStyle; pub(crate) type Provider = super::ecstore_bucket::on_demand_migration::Provider; pub(crate) type ValidationContext<'a> = super::ecstore_bucket::on_demand_migration::ValidationContext<'a>; + pub(crate) use super::ecstore_bucket::on_demand_migration::source_backend_spec; pub(crate) mod backfill { pub(crate) type BackfillCheckpoint = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillCheckpoint; diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index c1511ebba..65039350c 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -4832,6 +4832,8 @@ mod on_demand_migration_tests { session_token: None, }), tls: TlsConfig::default(), + azure: None, + gcs: None, }, filter: FilterConfig { prefix: None, diff --git a/rustfs/src/app/object/head.rs b/rustfs/src/app/object/head.rs index 73fd09b39..6d877af8a 100644 --- a/rustfs/src/app/object/head.rs +++ b/rustfs/src/app/object/head.rs @@ -665,6 +665,8 @@ mod tests { session_token: None, }), tls: TlsConfig::default(), + azure: None, + gcs: None, }, filter: FilterConfig { prefix: None, @@ -743,6 +745,7 @@ mod tests { }, ), is_multipart_etag: true, + etag_is_opaque: false, } } diff --git a/rustfs/src/app/object/on_demand_migration_put.rs b/rustfs/src/app/object/on_demand_migration_put.rs index 685e60b38..7edea71f8 100644 --- a/rustfs/src/app/object/on_demand_migration_put.rs +++ b/rustfs/src/app/object/on_demand_migration_put.rs @@ -124,6 +124,12 @@ pub(super) fn expected_md5_hex(head: &SourceHead) -> Option { if head.sse.is_some() { return None; } + // Azure stamps an opaque concurrency token in the ETag slot. It is + // recorded as provenance, but reading it as a digest would compare the + // pulled bytes against a value that never described them. + if head.etag_is_opaque { + return None; + } let etag = head.etag.as_deref()?; if etag.len() != 32 || is_multipart_etag(etag) || !etag.bytes().all(|byte| byte.is_ascii_hexdigit()) { return None; @@ -1043,6 +1049,12 @@ mod tests { head.sse = None; head.etag = None; assert_eq!(expected_md5_hex(&head), None); + + // An Azure ETag can be any string the service chooses; even one that + // happens to look like an MD5 must not be checked against the bytes. + let mut head = source_head(b"abc"); + head.etag_is_opaque = true; + assert_eq!(expected_md5_hex(&head), None, "opaque provider ETag"); } #[test] From d8580ec970de8acc0428d3207adc1e7fb623f167 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 22:50:00 +0800 Subject: [PATCH 40/40] test(e2e): prove operations overlap data movement --- .../concurrent_data_movement_test.rs | 16 ++- crates/e2e_test/src/distributed/harness.rs | 131 +++++++++++++++++- .../s3_during_data_movement_test.rs | 19 ++- 3 files changed, 154 insertions(+), 12 deletions(-) diff --git a/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs b/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs index 730d84d07..66b30f5de 100644 --- a/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs +++ b/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs @@ -13,9 +13,9 @@ // limitations under the License. use super::harness::{ - DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, payload_for, put_inventory_retrying, - retrying_get_equals, retrying_put, start_decommission, unique_bucket, wait_for_decommission_active, - wait_for_decommission_complete, + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress, + decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission, + unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress, }; use crate::common::init_logging; use std::sync::Arc; @@ -33,10 +33,9 @@ async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects dist.expand_to_four_pools().await?; start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; - wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; let clients = Arc::new(dist.clients()?); - let barrier = Arc::new(Barrier::new(16)); + let barrier = Arc::new(Barrier::new(17)); let mut handles = Vec::new(); for idx in 0..16 { let clients = clients.clone(); @@ -52,10 +51,17 @@ async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects })); } + wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + barrier.wait().await; + let mut live_objects = Vec::new(); for handle in handles { live_objects.push(handle.await??); } + let status = decommission_status_json(&dist.cluster).await?; + if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? { + return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into()); + } wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; diff --git a/crates/e2e_test/src/distributed/harness.rs b/crates/e2e_test/src/distributed/harness.rs index f5cf72cce..f610beaa4 100644 --- a/crates/e2e_test/src/distributed/harness.rs +++ b/crates/e2e_test/src/distributed/harness.rs @@ -723,6 +723,21 @@ pub(crate) fn decommission_active(status: &serde_json::Value, pool_id: usize) -> Ok(queued || status_text.eq_ignore_ascii_case("running") || pool_status.eq_ignore_ascii_case("decommissioning")) } +pub(crate) fn decommission_running_with_progress(status: &serde_json::Value, pool_id: usize) -> TestResult { + let pool = pool_entry(status, pool_id).ok_or_else(|| format!("pool {pool_id} missing from decommission status: {status}"))?; + if let Some(reason) = decommission_failure(pool) { + return Err(format!("{reason}: {pool}").into()); + } + let info = pool + .get("decommissionInfo") + .ok_or_else(|| format!("pool {pool_id} has no decommissionInfo: {pool}"))?; + let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or(""); + let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or(""); + let running = status_text.eq_ignore_ascii_case("running") || pool_status.eq_ignore_ascii_case("decommissioning"); + let progressed = nonzero_u64(info.get("objectsDecommissioned")) || nonzero_u64(info.get("bytesDecommissioned")); + Ok(running && progressed) +} + pub(crate) fn decommission_complete(status: &serde_json::Value, pool_id: usize) -> TestResult { let pool = pool_entry(status, pool_id).ok_or_else(|| format!("pool {pool_id} missing from decommission status: {status}"))?; if let Some(reason) = decommission_failure(pool) { @@ -735,7 +750,7 @@ pub(crate) fn decommission_complete(status: &serde_json::Value, pool_id: usize) let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or(""); let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or(""); let terminal = status_text.eq_ignore_ascii_case("complete") && pool_status.eq_ignore_ascii_case("decommissioned"); - let moved_data = nonzero_u64(info.get("objectsDecommissioned")) || nonzero_u64(info.get("bytesDecommissioned")); + let moved_data = nonzero_u64(info.get("objectsDecommissioned")) && nonzero_u64(info.get("bytesDecommissioned")); Ok(complete && terminal && moved_data) } @@ -747,6 +762,27 @@ pub(crate) async fn wait_for_decommission_active( wait_for_decommission_state(cluster, pool_id, timeout, "active", decommission_active).await } +pub(crate) async fn wait_for_decommission_running_with_progress( + cluster: &RustFSTestClusterEnvironment, + pool_id: usize, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = decommission_status_json(cluster).await?; + if decommission_running_with_progress(&status, pool_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!( + "decommission did not become active with non-zero progress within {timeout:?}; last status: {status}" + ) + .into()); + } + sleep(Duration::from_millis(100)).await; + } +} + pub(crate) async fn wait_for_decommission_complete( cluster: &RustFSTestClusterEnvironment, pool_id: usize, @@ -866,6 +902,20 @@ pub(crate) fn rebalance_active(status: &serde_json::Value, expected_id: &str) -> })) } +pub(crate) fn rebalance_running_with_progress(status: &serde_json::Value, expected_id: &str) -> TestResult { + Ok(validate_rebalance_status(status, expected_id)?.iter().any(|pool| { + let started = pool + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| value.eq_ignore_ascii_case("started")); + let progress = pool.get("progress"); + started + && (nonzero_u64(progress.and_then(|value| value.get("objects"))) + || nonzero_u64(progress.and_then(|value| value.get("versions"))) + || nonzero_u64(progress.and_then(|value| value.get("bytes")))) + })) +} + pub(crate) fn rebalance_complete(status: &serde_json::Value, expected_id: &str) -> TestResult { let pools = validate_rebalance_status(status, expected_id)?; let completed: Vec<&serde_json::Value> = pools @@ -908,6 +958,27 @@ pub(crate) async fn wait_for_rebalance_active( } } +pub(crate) async fn wait_for_rebalance_running_with_progress( + cluster: &RustFSTestClusterEnvironment, + expected_id: &str, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = rebalance_status_json(cluster).await?; + if rebalance_running_with_progress(&status, expected_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!( + "rebalance did not become active with non-zero progress within {timeout:?}; last status: {status}" + ) + .into()); + } + sleep(Duration::from_millis(100)).await; + } +} + pub(crate) async fn wait_for_rebalance_complete( cluster: &RustFSTestClusterEnvironment, expected_id: &str, @@ -1072,8 +1143,45 @@ fn decommission_complete_requires_terminal_status_and_clean_counters() { { "id": 1, "status": "none", "poolStatus": "active" } ] }); - assert!(decommission_complete(&status, 0).unwrap()); + assert!(decommission_complete(&status, 0).expect("complete fixture should be accepted")); assert!(decommission_complete(&status, 1).is_err()); + + for missing_counter in ["objectsDecommissioned", "bytesDecommissioned"] { + let mut one_sided = status.clone(); + one_sided["pools"][0]["decommissionInfo"][missing_counter] = serde_json::json!(0); + assert!( + !decommission_complete(&one_sided, 0).expect("one-sided progress fixture should be readable"), + "completion must require both movement counters; zeroed {missing_counter}" + ); + } +} + +#[test] +fn decommission_overlap_requires_running_state_and_progress() { + let mut status = serde_json::json!({ + "pools": [{ + "id": 0, + "status": "queued", + "poolStatus": "active", + "decommissionInfo": { + "queued": true, + "objectsDecommissioned": 1, + "bytesDecommissioned": 1024 + } + }] + }); + assert!( + !decommission_running_with_progress(&status, 0).expect("queued fixture should be readable"), + "queued work is not temporal overlap" + ); + status["pools"][0]["status"] = serde_json::json!("running"); + assert!(decommission_running_with_progress(&status, 0).expect("running fixture should be readable")); + status["pools"][0]["decommissionInfo"]["objectsDecommissioned"] = serde_json::json!(0); + status["pools"][0]["decommissionInfo"]["bytesDecommissioned"] = serde_json::json!(0); + assert!( + !decommission_running_with_progress(&status, 0).expect("zero-progress fixture should be readable"), + "running state alone does not prove movement started" + ); } #[test] @@ -1084,6 +1192,25 @@ fn rebalance_active_treats_started_as_in_progress() { assert!(!rebalance_active(&done, "run-1").unwrap()); } +#[test] +fn rebalance_overlap_requires_started_state_and_progress() { + let mut status = serde_json::json!({ + "id": "run-1", + "pools": [{ "id": 0, "status": "Started", "stopping": false, "progress": { "objects": 0, "bytes": 0 } }] + }); + assert!( + !rebalance_running_with_progress(&status, "run-1").expect("zero-progress fixture should be readable"), + "started state alone does not prove movement" + ); + status["pools"][0]["progress"]["objects"] = serde_json::json!(1); + assert!(rebalance_running_with_progress(&status, "run-1").expect("progress fixture should be readable")); + status["pools"][0]["status"] = serde_json::json!("Completed"); + assert!( + !rebalance_running_with_progress(&status, "run-1").expect("completed fixture should be readable"), + "completed movement is not temporal overlap" + ); +} + #[test] fn rebalance_complete_accepts_non_participating_pools_but_requires_progress() { let completed = serde_json::json!({ diff --git a/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs b/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs index 896b25924..66c835bbd 100644 --- a/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs +++ b/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs @@ -13,9 +13,10 @@ // limitations under the License. use super::harness::{ - DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, put_inventory_retrying, retrying_get_equals, - retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active, - wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete, + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress, + decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json, + retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete, + wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress, }; use crate::common::init_logging; use std::time::Duration; @@ -31,7 +32,7 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu dist.expand_to_four_pools().await?; start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; - wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; let live = dist.client(2)?; retrying_put( &live, @@ -57,12 +58,16 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu .any(|object| object.key() == Some("during-decommission.bin")), "list during decommission missed the newly written key" ); + let status = decommission_status_json(&dist.cluster).await?; + if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? { + return Err(format!("decommission did not remain active across the S3 operations: {status}").into()); + } wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; assert_inventory(&live, &bucket, &inventory).await?; let rebalance_id = start_rebalance(&dist.cluster).await?; - wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?; + wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?; retrying_put( &live, &bucket, @@ -79,6 +84,10 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu Duration::from_secs(30), ) .await?; + let status = rebalance_status_json(&dist.cluster).await?; + if !rebalance_running_with_progress(&status, &rebalance_id)? { + return Err(format!("rebalance did not remain active across the S3 operations: {status}").into()); + } wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?; assert_inventory(&dist.client(1)?, &bucket, &inventory).await?; Ok(())